Skip to content

Commit

Permalink
core: register xxHash128
Browse files Browse the repository at this point in the history 
Registers the xxHash128 checksum in every place configurable hashes are
done.
  • Loading branch information
@arogge
arogge committed Feb 24, 2023
1 parent 414c48e commit 8801b21
Show file tree
Hide file tree
Showing 17 changed files with 162 additions and 183 deletions.
66 changes: 25 additions & 41 deletions core/src/dird/catreq.cc
View file
Expand Up @@ -3,7 +3,7 @@
Copyright (C) 2001-2012 Free Software Foundation Europe e.V.
Copyright (C) 2011-2016 Planets Communications B.V.
Copyright (C) 2013-2022 Bareos GmbH & Co. KG
Copyright (C) 2013-2023 Bareos GmbH & Co. KG
This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -167,28 +167,22 @@ void CatalogRequest(JobControlRecord* jcr, BareosSocket* bs)
UnbashSpaces(mr.VolumeName);
if (jcr->db->GetMediaRecord(jcr, &mr)) {
const char* reason = NULL; /* detailed reason for rejection */
/*
* If we are reading, accept any volume (reason == NULL)
/* If we are reading, accept any volume (reason == NULL)
* If we are writing, check if the Volume is valid
* for this job, and do a recycle if necessary
*/
* for this job, and do a recycle if necessary */
if (writing) {
/*
* SD wants to write this Volume, so make
/* SD wants to write this Volume, so make
* sure it is suitable for this job, i.e.
* Pool matches, and it is either Append or Recycle
* and Media Type matches and Pool allows any volume.
*/
* and Media Type matches and Pool allows any volume. */
if (mr.PoolId != jcr->dir_impl->jr.PoolId) {
reason = _("not in Pool");
} else if (!bstrcmp(mr.MediaType,
jcr->dir_impl->res.write_storage->media_type)) {
reason = _("not correct MediaType");
} else {
/*
* Now try recycling if necessary
* reason set non-NULL if we cannot use it
*/
/* Now try recycling if necessary
* reason set non-NULL if we cannot use it */
CheckIfVolumeValidOrRecyclable(jcr, &mr, &reason);
}
}
Expand All @@ -215,11 +209,9 @@ void CatalogRequest(JobControlRecord* jcr, BareosSocket* bs)
&sdmr.VolStatus, &sdmr.Slot, &label, &sdmr.InChanger,
&sdmr.VolReadTime, &sdmr.VolWriteTime, &VolFirstWritten)
== 18) {
/*
* Request to update Media record. Comes typically at the end
/* Request to update Media record. Comes typically at the end
* of a Storage daemon Job Session, when labeling/relabeling a
* Volume, or when an EOF mark is written.
*/
* Volume, or when an EOF mark is written. */
DbLocker _{jcr->db};
Dmsg3(400, "Update media %s oldStat=%s newStat=%s\n", sdmr.VolumeName,
mr.VolStatus, sdmr.VolStatus);
Expand Down Expand Up @@ -267,19 +259,15 @@ void CatalogRequest(JobControlRecord* jcr, BareosSocket* bs)
Dmsg2(400, "Update media: BefVolJobs=%u After=%u\n", mr.VolJobs,
sdmr.VolJobs);

/*
* Check if the volume has been written by the job,
* and update the LastWritten field if needed.
*/
/* Check if the volume has been written by the job,
* and update the LastWritten field if needed. */
if (mr.VolBlocks != sdmr.VolBlocks && VolLastWritten != 0) {
mr.LastWritten = VolLastWritten;
}

/*
* Update to point to the last device used to write the Volume.
/* Update to point to the last device used to write the Volume.
* However, do so only if we are writing the tape, i.e.
* the number of VolWrites has increased.
*/
* the number of VolWrites has increased. */
if (jcr->dir_impl->res.write_storage && sdmr.VolWrites > mr.VolWrites) {
Dmsg2(050, "Update StorageId old=%d new=%d\n", mr.StorageId,
jcr->dir_impl->res.write_storage->StorageId);
Expand Down Expand Up @@ -307,10 +295,8 @@ void CatalogRequest(JobControlRecord* jcr, BareosSocket* bs)
Dmsg2(400, "UpdateMediaRecord. Stat=%s Vol=%s\n", mr.VolStatus,
mr.VolumeName);

/*
* Update the database, then before sending the response to the SD,
* check if the Volume has expired.
*/
/* Update the database, then before sending the response to the SD,
* check if the Volume has expired. */
if (!jcr->db->UpdateMediaRecord(jcr, &mr)) {
Jmsg(jcr, M_FATAL, 0, _("Catalog error updating Media record. %s"),
jcr->db->strerror());
Expand Down Expand Up @@ -412,11 +398,9 @@ static void UpdateAttribute(JobControlRecord* jcr,
jcr->db->StartTransaction(jcr); /* start transaction if not already open */
ar = jcr->ar;

/*
* Start by scanning directly in the message buffer to get Stream
/* Start by scanning directly in the message buffer to get Stream
* there may be a cached attr so we cannot yet write into
* jcr->attr or jcr->ar
*/
* jcr->attr or jcr->ar */
p = msg;
SkipNonspaces(&p); /* UpdCat */
SkipSpaces(&p);
Expand All @@ -433,8 +417,7 @@ static void UpdateAttribute(JobControlRecord* jcr,
unser_uint32(reclen); /* Record length */
p += UnserLength(p); /* Raw record follows */

/**
* At this point p points to the raw record, which varies according
/* At this point p points to the raw record, which varies according
* to what kind of a record (Stream) was sent. Note, the integer
* fields at the beginning of these "raw" records are in ASCII with
* spaces between them so one can use scanf or manual scanning to
Expand All @@ -458,8 +441,7 @@ static void UpdateAttribute(JobControlRecord* jcr,
* Object_compression
* Plugin_name
* Object_name
* Binary Object data
*/
* Binary Object data */

Dmsg1(400, "UpdCat msg=%s\n", msg);
Dmsg5(400, "UpdCat VolSessId=%d VolSessT=%d FI=%d Strm=%d reclen=%d\n",
Expand Down Expand Up @@ -525,11 +507,9 @@ static void UpdateAttribute(JobControlRecord* jcr,
jcr->cached_attribute = true;


/*
* Fhinfo and Fhnode are not sent from the SD,
/* Fhinfo and Fhnode are not sent from the SD,
* they exist only in NDMP 2-Way backups so we
* set them to 0 here
*/
* set them to 0 here */
ar->Fhinfo = 0;
ar->Fhnode = 0;

Expand Down Expand Up @@ -617,6 +597,10 @@ static void UpdateAttribute(JobControlRecord* jcr,
len = CRYPTO_DIGEST_SHA512_SIZE;
type = CRYPTO_DIGEST_SHA512;
break;
case STREAM_XXH128_DIGEST:
len = CRYPTO_DIGEST_XXH128_SIZE;
type = CRYPTO_DIGEST_XXH128;
break;
default:
Jmsg(jcr, M_ERROR, 0,
_("Catalog error updating file digest. Unsupported digest "
Expand Down
4 changes: 4 additions & 0 deletions core/src/dird/dird_conf.cc
View file
Expand Up @@ -1920,6 +1920,10 @@ void FilesetResource::PrintConfigIncludeExcludeOptions(
p++;
break;
#endif
case '4':
send.KeyQuotedString("Signature", "XXH128");
p++;
break;
default:
send.KeyQuotedString("Signature", "SHA1");
break;
Expand Down
9 changes: 4 additions & 5 deletions core/src/dird/inc_conf.cc
View file
Expand Up @@ -3,7 +3,7 @@
Copyright (C) 2000-2009 Free Software Foundation Europe e.V.
Copyright (C) 2011-2012 Planets Communications B.V.
Copyright (C) 2013-2022 Bareos GmbH & Co. KG
Copyright (C) 2013-2023 Bareos GmbH & Co. KG
This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -150,6 +150,7 @@ static struct s_fs_opt FS_options[]
{"sha1", INC_KW_DIGEST, "S"},
{"sha256", INC_KW_DIGEST, "S2"},
{"sha512", INC_KW_DIGEST, "S3"},
{"xxh128", INC_KW_DIGEST, "S4"},
{"gzip", INC_KW_COMPRESSION, "Z6"},
{"gzip1", INC_KW_COMPRESSION, "Z1"},
{"gzip2", INC_KW_COMPRESSION, "Z2"},
Expand Down Expand Up @@ -992,11 +993,9 @@ void StoreInc(LEX* lc, ResourceItem* item, int index, int pass)
{
int token;

/*
* Decide if we are doing a new Include or an old include. The
/* Decide if we are doing a new Include or an old include. The
* new Include is followed immediately by open brace, whereas the
* old include has options following the Include.
*/
* old include has options following the Include. */
token = LexGetToken(lc, BCT_SKIP_EOL);
if (token == BCT_BOB) {
StoreNewinc(lc, item, index, pass);
Expand Down
10 changes: 6 additions & 4 deletions core/src/dird/testfind.cc
View file
Expand Up @@ -2,7 +2,7 @@
BAREOS® - Backup Archiving REcovery Open Sourced
Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
Copyright (C) 2016-2022 Bareos GmbH & Co. KG
Copyright (C) 2016-2023 Bareos GmbH & Co. KG
This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -157,9 +157,7 @@ int main(int argc, char* const* argv)

MessagesResource* msg;

foreach_res (msg, R_MSGS) {
InitMsg(NULL, msg);
}
foreach_res (msg, R_MSGS) { InitMsg(NULL, msg); }

jcr = NewDirectorJcr(TestfindFreeJcr);
jcr->dir_impl->res.fileset
Expand Down Expand Up @@ -598,6 +596,10 @@ static void SetOptions(findFOPTS* fo, const char* opts)
p++;
break;
#endif
case '4':
SetBit(FO_XXH128, fo->flags);
p++;
break;
default:
/* Automatically downgrade to SHA-1 if an unsupported
* SHA variant is specified */
Expand Down
29 changes: 11 additions & 18 deletions core/src/filed/accurate.cc
View file
Expand Up @@ -3,7 +3,7 @@
Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
Copyright (C) 2013-2014 Planets Communications B.V.
Copyright (C) 2013-2022 Bareos GmbH & Co. KG
Copyright (C) 2013-2023 Bareos GmbH & Co. KG
This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -178,12 +178,10 @@ bool AccurateCheckFile(JobControlRecord* jcr, FindFilesPacket* ff_pkt)
goto bail_out;
}

/**
* Restore original name so we can check the actual file when we check
/* Restore original name so we can check the actual file when we check
* the accurate options later on. This is mostly important for the
* CalculateAndCompareFileChksum() function as that needs to calulate
* the checksum of the real file and not try to open the stripped pathname.
*/
* the checksum of the real file and not try to open the stripped pathname. */
UnstripPath(ff_pkt);

ff_pkt->accurate_found = true;
Expand Down Expand Up @@ -211,10 +209,8 @@ bool AccurateCheckFile(JobControlRecord* jcr, FindFilesPacket* ff_pkt)
}
break;
case 'p': /** Permissions bits */
/**
* TODO: If something change only in perm, user, group
* Backup only the attribute stream
*/
/* TODO: If something change only in perm, user, group
* Backup only the attribute stream */
if (statc.st_mode != ff_pkt->statp.st_mode) {
Dmsg3(debuglevel - 1,
"%s st_mode differ. Cat: %04o File: %04o\n", fname,
Expand Down Expand Up @@ -289,7 +285,8 @@ bool AccurateCheckFile(JobControlRecord* jcr, FindFilesPacket* ff_pkt)
&& (BitIsSet(FO_MD5, ff_pkt->flags)
|| BitIsSet(FO_SHA1, ff_pkt->flags)
|| BitIsSet(FO_SHA256, ff_pkt->flags)
|| BitIsSet(FO_SHA512, ff_pkt->flags)))) {
|| BitIsSet(FO_SHA512, ff_pkt->flags)
|| BitIsSet(FO_XXH128, ff_pkt->flags)))) {
if (!*payload->chksum && !jcr->rerunning) {
Jmsg(jcr, M_WARNING, 0, _("Cannot verify checksum for %s\n"),
ff_pkt->fname);
Expand All @@ -309,10 +306,8 @@ bool AccurateCheckFile(JobControlRecord* jcr, FindFilesPacket* ff_pkt)
}
}

/**
* In Incr/Diff accurate mode, we mark all files as seen
* When in Full+Base mode, we mark only if the file match exactly
*/
/* In Incr/Diff accurate mode, we mark all files as seen
* When in Full+Base mode, we mark only if the file match exactly */
if (jcr->getJobLevel() == L_FULL) {
if (!status) {
// Compute space saved with basefile.
Expand Down Expand Up @@ -378,10 +373,8 @@ bool AccurateCmd(JobControlRecord* jcr)
chksum_length = strlen(chksum);
delta_seq = str_to_int32(chksum + chksum_length + 1);

/**
* Sanity check total length of the received msg must be at least
* total of the 3 lengths calculated + 3 (\0)
*/
/* Sanity check total length of the received msg must be at least
* total of the 3 lengths calculated + 3 (\0) */
if ((fname_length + lstat_length + chksum_length + 3)
> dir->message_length) {
continue;
Expand Down
3 changes: 3 additions & 0 deletions core/src/filed/backup.cc
View file
Expand Up @@ -301,6 +301,9 @@ static inline bool SetupEncryptionDigests(b_save_ctx& bsctx)
} else if (BitIsSet(FO_SHA512, bsctx.ff_pkt->flags)) {
bsctx.digest = crypto_digest_new(bsctx.jcr, CRYPTO_DIGEST_SHA512);
bsctx.digest_stream = STREAM_SHA512_DIGEST;
} else if (BitIsSet(FO_XXH128, bsctx.ff_pkt->flags)) {
bsctx.digest = crypto_digest_new(bsctx.jcr, CRYPTO_DIGEST_XXH128);
bsctx.digest_stream = STREAM_XXH128_DIGEST;
}

// Did digest initialization fail?
Expand Down
24 changes: 11 additions & 13 deletions core/src/filed/fileset.cc
View file
Expand Up @@ -3,7 +3,7 @@
Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
Copyright (C) 2011-2012 Planets Communications B.V.
Copyright (C) 2013-2022 Bareos GmbH & Co. KG
Copyright (C) 2013-2023 Bareos GmbH & Co. KG
This program is Free Software; you can redistribute it and/or
modify it under the terms of version three of the GNU Affero General Public
Expand Down Expand Up @@ -280,12 +280,10 @@ void AddFileset(JobControlRecord* jcr, const char* item)
return;
}

/**
* The switch tests the code for validity.
/* The switch tests the code for validity.
* The subcode is always good if it is a space, otherwise we must confirm.
* We set state to state_error first assuming the subcode is invalid,
* requiring state to be set in cases below that handle subcodes.
*/
* requiring state to be set in cases below that handle subcodes. */
if (subcode != ' ') {
state = state_error;
Dmsg0(100, "Set state=error or double code.\n");
Expand Down Expand Up @@ -364,10 +362,8 @@ bool TermFileset(JobControlRecord* jcr)

fileset = jcr->fd_impl->ff->fileset;
#ifdef HAVE_WIN32
/*
* Expand the fileset to include all drive letters when the fileset includes a
* File = / entry.
*/
/* Expand the fileset to include all drive letters when the fileset includes a
* File = / entry. */
if (!expand_win32_fileset(jcr->fd_impl->ff->fileset)) { return false; }

// Exclude entries in NotToBackup registry key
Expand Down Expand Up @@ -581,11 +577,13 @@ static int SetOptions(findFOPTS* fo, const char* opts)
p++;
break;
#endif
case '4':
SetBit(FO_XXH128, fo->flags);
p++;
break;
default:
/*
* If 2 or 3 is seen here, SHA2 is not configured, so eat the
* option, and drop back to SHA-1.
*/
/* If 2 or 3 is seen here, SHA2 is not configured, so eat the
* option, and drop back to SHA-1. */
if (p[1] == '2' || p[1] == '3') { p++; }
SetBit(FO_SHA1, fo->flags);
break;
Expand Down
1 change: 1 addition & 0 deletions core/src/filed/restore.cc
View file
Expand Up @@ -976,6 +976,7 @@ void DoRestore(JobControlRecord* jcr)
case STREAM_SHA1_DIGEST:
case STREAM_SHA256_DIGEST:
case STREAM_SHA512_DIGEST:
case STREAM_XXH128_DIGEST:
break;

case STREAM_PROGRAM_NAMES:
Expand Down

0 comments on commit 8801b21

Please sign in to comment.