Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

consolidate: purge jobids without files before starting the consolidation #1107

Merged
merged 7 commits into from Mar 9, 2022
Merged

consolidate: purge jobids without files before starting the consolidation #1107

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8e01d55
postgresql: print PQerrorMessage()
pstorz Jan 25, 2022
cc57f79
systemtests: ai-consolidate-test purge empty jobs
pstorz Jan 25, 2022
b948dcd
cats: refactor functionality from UA to DB
arogge Mar 1, 2022
cdf1ec1
dird: purge empty jobs as part of consolidation
pstorz Feb 14, 2022
dea12b2
bareos-check-sources: update copyright year
pstorz Mar 8, 2022
d12739a
update CHANGELOG.md
pstorz Mar 8, 2022
8c9f518
sql_get.cc: add missing <algorithm> include
pstorz Mar 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Expand Up @@ -16,6 +16,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:

### Changed
- webui: add timeline chart by jobs [PR #1063]
- Consolidation now purges candidate jobs with no files instead of ignoring them [PR #1107]

### Removed

Expand Down
19 changes: 14 additions & 5 deletions core/src/cats/cats.h
View file
Open in desktop
Expand Up @@ -3,7 +3,7 @@

Copyright (C) 2000-2012 Free Software Foundation Europe e.V.
Copyright (C) 2011-2016 Planets Communications B.V.
Copyright (C) 2013-2021 Bareos GmbH & Co. KG
Copyright (C) 2013-2022 Bareos GmbH & Co. KG

This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -488,6 +488,11 @@ typedef struct sql_field {
} SQL_FIELD;
#endif

class BareosSqlError : public std::runtime_error {
public:
BareosSqlError(const char* what) : std::runtime_error(what) {}
};

class BareosDb : public BareosDbQueryEnum {
protected:
brwlock_t lock_; /**< Transaction lock */
Expand Down Expand Up @@ -651,6 +656,8 @@ class BareosDb : public BareosDbQueryEnum {
bool DeletePoolRecord(JobControlRecord* jcr, PoolDbRecord* pool_dbr);
bool DeleteMediaRecord(JobControlRecord* jcr, MediaDbRecord* mr);
bool PurgeMediaRecord(JobControlRecord* jcr, MediaDbRecord* mr);
void PurgeFiles(const char* jobids);
void PurgeJobs(const char* jobids);

/* sql_find.c */

Expand Down Expand Up @@ -739,6 +746,7 @@ class BareosDb : public BareosDbQueryEnum {
bool AccurateGetJobids(JobControlRecord* jcr,
JobDbRecord* jr,
db_list_ctx* jobids);
db_list_ctx FilterZeroFileJobs(db_list_ctx& jobids);
bool GetUsedBaseJobids(JobControlRecord* jcr,
const char* jobids,
db_list_ctx* result);
Expand Down Expand Up @@ -905,6 +913,7 @@ class BareosDb : public BareosDbQueryEnum {
bool MarkFileRecord(JobControlRecord* jcr, FileId_t FileId, JobId_t JobId);
void MakeInchangerUnique(JobControlRecord* jcr, MediaDbRecord* mr);
int UpdateStats(JobControlRecord* jcr, utime_t age);
void UpgradeCopies(const char* jobids);

/* Low level methods */
bool MatchDatabase(const char* db_driver,
Expand Down Expand Up @@ -1014,8 +1023,8 @@ struct SqlPoolEntry {
int id = 0; /**< Unique ID, connection numbering can have holes and the pool
is not sorted on it */
int reference_count = 0; /**< Reference count for this entry */
time_t last_update = 0; /**< When was this connection last updated either used
or put back on the pool */
time_t last_update = 0; /**< When was this connection last updated either
used or put back on the pool */
BareosDb* db_handle = nullptr; /**< Connection handle to the database */
dlink<SqlPoolEntry> link; /**< list management */
};
Expand All @@ -1031,8 +1040,8 @@ struct SqlPoolDescriptor {
int max_connections
= 0; /**< Maximum number of connections in the connection pool
*/
int increment_connections = 0; /**< Increase/Decrease the number of connection
in the pool with this value */
int increment_connections = 0; /**< Increase/Decrease the number of
connection in the pool with this value */
int idle_timeout = 0; /**< Number of seconds to wait before tearing down a
connection */
int validate_timeout = 0; /**< Number of seconds after which an idle
Expand Down
8 changes: 5 additions & 3 deletions core/src/cats/postgresql.cc
View file
Open in desktop
Expand Up @@ -3,7 +3,7 @@

Copyright (C) 2003-2011 Free Software Foundation Europe e.V.
Copyright (C) 2011-2016 Planets Communications B.V.
Copyright (C) 2013-2021 Bareos GmbH & Co. KG
Copyright (C) 2013-2022 Bareos GmbH & Co. KG

This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -634,8 +634,10 @@ bool BareosDbPostgresql::SqlQueryWithoutHandler(const char* query, int flags)
retval = true;
break;
case PGRES_FATAL_ERROR:
Dmsg1(50, "Result status fatal: %s\n", query);
if (exit_on_fatal_) { Emsg0(M_ERROR_TERM, 0, "Fatal database error\n"); }
Dmsg1(50, "Result status fatal: %s, %s\n", query, sql_strerror());
if (exit_on_fatal_) {
Emsg1(M_ERROR_TERM, 0, "Fatal database error: %s\n", sql_strerror());
}

if (try_reconnect_ && !transaction_) {
/*
Expand Down
48 changes: 47 additions & 1 deletion core/src/cats/sql_delete.cc
View file
Open in desktop
Expand Up @@ -3,7 +3,7 @@

Copyright (C) 2000-2006 Free Software Foundation Europe e.V.
Copyright (C) 2011-2016 Planets Communications B.V.
Copyright (C) 2013-2019 Bareos GmbH & Co. KG
Copyright (C) 2013-2022 Bareos GmbH & Co. KG

This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -234,4 +234,50 @@ bool BareosDb::PurgeMediaRecord(JobControlRecord* jcr, MediaDbRecord* mr)
DbUnlock(this);
return retval;
}

void BareosDb::PurgeFiles(const char* jobids)
{
PoolMem query(PM_MESSAGE);

Mmsg(query, "DELETE FROM File WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());

Mmsg(query, "DELETE FROM BaseFiles WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());

Mmsg(query, "UPDATE Job SET PurgedFiles=1 WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());
}

void BareosDb::PurgeJobs(const char* jobids)
{
PoolMem query(PM_MESSAGE);

/* Delete (or purge) records associated with the job */
PurgeFiles(jobids);

Mmsg(query, "DELETE FROM JobMedia WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());

Mmsg(query, "DELETE FROM Log WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());

Mmsg(query, "DELETE FROM RestoreObject WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());

Mmsg(query, "DELETE FROM PathVisibility WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());

Mmsg(query, "DELETE FROM NDMPJobEnvironment WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());

Mmsg(query, "DELETE FROM JobStats WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());

UpgradeCopies(jobids);

/* Now remove the Job record itself */
Mmsg(query, "DELETE FROM Job WHERE JobId IN (%s)", jobids);
SqlQuery(query.c_str());
}
#endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL || HAVE_INGRES */
23 changes: 22 additions & 1 deletion core/src/cats/sql_get.cc
View file
Open in desktop
Expand Up @@ -3,7 +3,7 @@

Copyright (C) 2000-2012 Free Software Foundation Europe e.V.
Copyright (C) 2011-2016 Planets Communications B.V.
Copyright (C) 2013-2021 Bareos GmbH & Co. KG
Copyright (C) 2013-2022 Bareos GmbH & Co. KG

This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -1320,6 +1320,27 @@ bool BareosDb::GetUsedBaseJobids(JobControlRecord* jcr,
return SqlQueryWithHandler(query.c_str(), DbListHandler, result);
}

/*
* Remove the jobs that have JobFiles == 0 from the supplied jobid list
* and return them in another list
*/
db_list_ctx BareosDb::FilterZeroFileJobs(db_list_ctx& jobids)
{
std::string query{"SELECT JobId FROM Job WHERE JobFiles = 0 AND JobId IN ("};
query += jobids.Join(",") + ")";

db_list_ctx zero_file_jobs;
if (!SqlQueryWithHandler(query.c_str(), DbListHandler, &zero_file_jobs)) {
throw new BareosSqlError(sql_strerror());
}
for (auto& remove_jobid : zero_file_jobs) {
jobids.erase(std::remove(jobids.begin(), jobids.end(), remove_jobid),
jobids.end());
}

return zero_file_jobs;
}

/**
* The decision do change an incr/diff was done before
* Full : do nothing
Expand Down
42 changes: 41 additions & 1 deletion core/src/cats/sql_update.cc
View file
Open in desktop
Expand Up @@ -3,7 +3,7 @@

Copyright (C) 2000-2012 Free Software Foundation Europe e.V.
Copyright (C) 2011-2016 Planets Communications B.V.
Copyright (C) 2013-2019 Bareos GmbH & Co. KG
Copyright (C) 2013-2022 Bareos GmbH & Co. KG

This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -569,5 +569,45 @@ bool BareosDb::UpdateNdmpLevelMapping(JobControlRecord* jcr,

return retval;
}

/**
* Change the type of the next copy job to backup.
* We need to upgrade the next copy of a normal job,
* and also upgrade the next copy when the normal job
* already have been purged.
*
* JobId: 1 PriorJobId: 0 (original)
* JobId: 2 PriorJobId: 1 (first copy)
* JobId: 3 PriorJobId: 1 (second copy)
*
* JobId: 2 PriorJobId: 1 (first copy, now regular backup)
* JobId: 3 PriorJobId: 1 (second copy)
*
* => Search through PriorJobId in jobid and
* PriorJobId in PriorJobId (jobid)
*/
void BareosDb::UpgradeCopies(const char* jobids)
{
PoolMem query(PM_MESSAGE);

DbLock(this);

/* Do it in two times for mysql */
FillQuery(query, BareosDb::SQL_QUERY::uap_upgrade_copies_oldest_job,
JT_JOB_COPY, jobids, jobids);

SqlQuery(query.c_str());

/* Now upgrade first copy to Backup */
Mmsg(query,
"UPDATE Job SET Type='B' " /* JT_JOB_COPY => JT_BACKUP */
"WHERE JobId IN ( SELECT JobId FROM cpy_tmp )");

SqlQuery(query.c_str());

Mmsg(query, "DROP TABLE cpy_tmp");
SqlQuery(query.c_str());
DbUnlock(this);
}
#endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL || HAVE_INGRES || \
HAVE_DBI */
16 changes: 12 additions & 4 deletions core/src/dird/consolidate.cc
View file
Open in desktop
@@ -1,7 +1,7 @@
/*
BAREOS® - Backup Archiving REcovery Open Sourced

Copyright (C) 2016-2020 Bareos GmbH & Co. KG
Copyright (C) 2016-2022 Bareos GmbH & Co. KG

This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -177,13 +177,21 @@ bool DoConsolidate(JobControlRecord* jcr)
Dmsg1(10, "consolidate candidates: %s.\n",
jobids_ctx.GetAsString().c_str());

db_list_ctx zero_file_jobs = jcr->db->FilterZeroFileJobs(jobids_ctx);
if (zero_file_jobs.size() > 0) {
Jmsg(jcr, M_INFO, 0, "%s: purging empty jobids %s\n",
job->resource_name_, zero_file_jobs.Join(", ").c_str());
jcr->db->PurgeJobs(zero_file_jobs.GetAsString().c_str());
}
incrementals_total = jobids_ctx.size() - 1;
/**
* Consolidation of zero or one job does not make sense, we leave it like
* it is
* Consolidation of zero or one job does not make sense, we leave it
* like it is
*/
if (incrementals_total < 1) {
Jmsg(jcr, M_INFO, 0,
_("%s: less than two jobs to consolidate found, doing nothing.\n"),
_("%s: less than two jobs to consolidate found, doing "
"nothing.\n"),
job->resource_name_);
continue;
}
Expand Down
81 changes: 4 additions & 77 deletions core/src/dird/ua_purge.cc
View file
Open in desktop
Expand Up @@ -3,7 +3,7 @@

Copyright (C) 2002-2012 Free Software Foundation Europe e.V.
Copyright (C) 2011-2016 Planets Communications B.V.
Copyright (C) 2013-2021 Bareos GmbH & Co. KG
Copyright (C) 2013-2022 Bareos GmbH & Co. KG

This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -321,25 +321,7 @@ static bool PurgeJobsFromClient(UaContext* ua, ClientResource* client)
// Remove File records from a list of JobIds
void PurgeFilesFromJobs(UaContext* ua, const char* jobs)
{
PoolMem query(PM_MESSAGE);

Mmsg(query, "DELETE FROM File WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Delete File sql=%s\n", query.c_str());

Mmsg(query, "DELETE FROM BaseFiles WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Delete BaseFiles sql=%s\n", query.c_str());

/*
* Now mark Job as having files purged. This is necessary to
* avoid having too many Jobs to process in future prunings. If
* we don't do this, the number of JobId's in our in memory list
* could grow very large.
*/
Mmsg(query, "UPDATE Job SET PurgedFiles=1 WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Mark purged sql=%s\n", query.c_str());
ua->db->PurgeFiles(jobs);
}

/**
Expand Down Expand Up @@ -447,68 +429,13 @@ static bool PurgeQuotaFromClient(UaContext* ua, ClientResource* client)
*/
void UpgradeCopies(UaContext* ua, const char* jobs)
{
PoolMem query(PM_MESSAGE);

DbLock(ua->db);

/* Do it in two times for mysql */
ua->db->FillQuery(query, BareosDb::SQL_QUERY::uap_upgrade_copies_oldest_job,
JT_JOB_COPY, jobs, jobs);

ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Upgrade copies Log sql=%s\n", query.c_str());

/* Now upgrade first copy to Backup */
Mmsg(query,
"UPDATE Job SET Type='B' " /* JT_JOB_COPY => JT_BACKUP */
"WHERE JobId IN ( SELECT JobId FROM cpy_tmp )");

ua->db->SqlQuery(query.c_str());

Mmsg(query, "DROP TABLE cpy_tmp");
ua->db->SqlQuery(query.c_str());

DbUnlock(ua->db);
ua->db->UpgradeCopies(jobs);
}

// Remove all records from catalog for a list of JobIds
void PurgeJobsFromCatalog(UaContext* ua, const char* jobs)
{
PoolMem query(PM_MESSAGE);

/* Delete (or purge) records associated with the job */
PurgeFilesFromJobs(ua, jobs);

Mmsg(query, "DELETE FROM JobMedia WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Delete JobMedia sql=%s\n", query.c_str());

Mmsg(query, "DELETE FROM Log WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Delete Log sql=%s\n", query.c_str());

Mmsg(query, "DELETE FROM RestoreObject WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Delete RestoreObject sql=%s\n", query.c_str());

Mmsg(query, "DELETE FROM PathVisibility WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Delete PathVisibility sql=%s\n", query.c_str());

Mmsg(query, "DELETE FROM NDMPJobEnvironment WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Delete NDMPJobEnvironment sql=%s\n", query.c_str());

Mmsg(query, "DELETE FROM JobStats WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Delete JobStats sql=%s\n", query.c_str());

UpgradeCopies(ua, jobs);

/* Now remove the Job record itself */
Mmsg(query, "DELETE FROM Job WHERE JobId IN (%s)", jobs);
ua->db->SqlQuery(query.c_str());
Dmsg1(050, "Delete Job sql=%s\n", query.c_str());
ua->db->PurgeJobs(jobs);
}

void PurgeFilesFromVolume(UaContext* ua, MediaDbRecord* mr) {
Expand Down