From 9474ba0f15502ecc278fe9dfcee007db2c24c842 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 18 Sep 2017 07:54:59 +0100 Subject: [PATCH] Refactor BackupStoreContext to use BackupFileSystem --- lib/backupstore/BackupCommands.cpp | 128 +--- lib/backupstore/BackupFileSystem.cpp | 92 ++- lib/backupstore/BackupFileSystem.h | 28 +- lib/backupstore/BackupStoreContext.cpp | 805 +++++++++++------------ lib/backupstore/BackupStoreContext.h | 46 +- lib/backupstore/BackupStoreDirectory.h | 8 +- lib/backupstore/BackupStoreException.txt | 1 + lib/backupstore/StoreTestUtils.cpp | 4 + lib/common/ByteCountingStream.h | 81 +++ test/backupstore/testbackupstore.cpp | 34 +- 10 files changed, 627 insertions(+), 600 deletions(-) create mode 100644 lib/common/ByteCountingStream.h diff --git a/lib/backupstore/BackupCommands.cpp b/lib/backupstore/BackupCommands.cpp index 194b47304..7a0039e7f 100644 --- a/lib/backupstore/BackupCommands.cpp +++ b/lib/backupstore/BackupCommands.cpp @@ -71,7 +71,9 @@ std::auto_ptr BackupProtocolReplyable::HandleException(Bo } else if (e.GetType() == BackupStoreException::ExceptionType) { - if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify) + // Slightly broken or really broken, both thrown by VerifyStream: + if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify || + e.GetSubType() == BackupStoreException::BadBackupStoreFile) { return PROTOCOL_ERROR(Err_FileDoesNotVerify); } @@ -348,132 +350,22 @@ std::auto_ptr BackupProtocolGetObject::DoCommand(BackupPr std::auto_ptr BackupProtocolGetFile::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const { CHECK_PHASE(Phase_Commands) - - // Check the objects exist - if(!rContext.ObjectExists(mObjectID) - || !rContext.ObjectExists(mInDirectory)) - { - return PROTOCOL_ERROR(Err_DoesNotExist); - } - - // Get the directory it's in - const BackupStoreDirectory &rdir(rContext.GetDirectory(mInDirectory)); - - // Find the object within the directory - BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(mObjectID); - if(pfileEntry == 0) - { - return PROTOCOL_ERROR(Err_DoesNotExistInDirectory); - } - - // The result std::auto_ptr stream; - // Does this depend on anything? - if(pfileEntry->GetDependsNewer() != 0) + try { - // File exists, but is a patch from a new version. Generate the older version. - std::vector patchChain; - int64_t id = mObjectID; - BackupStoreDirectory::Entry *en = 0; - do - { - patchChain.push_back(id); - en = rdir.FindEntryByID(id); - if(en == 0) - { - BOX_ERROR("Object " << - BOX_FORMAT_OBJECTID(mObjectID) << - " in dir " << - BOX_FORMAT_OBJECTID(mInDirectory) << - " for account " << - BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << - " references object " << - BOX_FORMAT_OBJECTID(id) << - " which does not exist in dir"); - return PROTOCOL_ERROR(Err_PatchConsistencyError); - } - id = en->GetDependsNewer(); - } - while(en != 0 && id != 0); - - // OK! The last entry in the chain is the full file, the others are patches back from it. - // Open the last one, which is the current from file - std::auto_ptr from(rContext.OpenObject(patchChain[patchChain.size() - 1])); - - // Then, for each patch in the chain, do a combine - for(int p = ((int)patchChain.size()) - 2; p >= 0; --p) - { - // ID of patch - int64_t patchID = patchChain[p]; - - // Open it a couple of times - std::auto_ptr diff(rContext.OpenObject(patchID)); - std::auto_ptr diff2(rContext.OpenObject(patchID)); - - // Choose a temporary filename for the result of the combination - std::ostringstream fs; - fs << rContext.GetAccountRoot() << ".recombinetemp." << p; - std::string tempFn = - RaidFileController::DiscSetPathToFileSystemPath( - rContext.GetStoreDiscSet(), fs.str(), - p + 16); - - // Open the temporary file - std::auto_ptr combined( - new InvisibleTempFileStream( - tempFn, O_RDWR | O_CREAT | O_EXCL | - O_BINARY | O_TRUNC)); - - // Do the combining - BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined); - - // Move to the beginning of the combined file - combined->Seek(0, IOStream::SeekType_Absolute); - - // Then shuffle round for the next go - if (from.get()) from->Close(); - from = combined; - } - - // Now, from contains a nice file to send to the client. Reorder it - { - // Write nastily to allow this to work with gcc 2.x - std::auto_ptr t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */)); - stream = t; - } - - // Release from file to avoid double deletion - from.release(); + stream = rContext.GetFile(mObjectID, mInDirectory); } - else + catch(BackupStoreException &e) { - // Simple case: file already exists on disc ready to go - - // Open the object - std::auto_ptr object(rContext.OpenObject(mObjectID)); - BufferedStream buf(*object); - - // Verify it - if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) + if(EXCEPTION_IS_TYPE(e, BackupStoreException, ObjectDoesNotExist)) { - return PROTOCOL_ERROR(Err_FileDoesNotVerify); + return PROTOCOL_ERROR(Err_DoesNotExist); } - - // Reset stream -- seek to beginning - object->Seek(0, IOStream::SeekType_Absolute); - - // Reorder the stream/file into stream order + else { - // Write nastily to allow this to work with gcc 2.x - std::auto_ptr t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */)); - stream = t; + throw; } - - // Object will be deleted when the stream is deleted, - // so can release the object auto_ptr here to avoid - // premature deletion - object.release(); } // Stream the reordered stream to the peer diff --git a/lib/backupstore/BackupFileSystem.cpp b/lib/backupstore/BackupFileSystem.cpp index 10f352bdf..934c5b92b 100644 --- a/lib/backupstore/BackupFileSystem.cpp +++ b/lib/backupstore/BackupFileSystem.cpp @@ -24,6 +24,7 @@ #include "BackupStoreRefCountDatabase.h" #include "BufferedStream.h" #include "BufferedWriteStream.h" +#include "ByteCountingStream.h" #include "CollectInBufferStream.h" #include "Configuration.h" #include "BackupStoreObjectMagic.h" @@ -510,8 +511,11 @@ RaidBackupFileSystem::PutFileComplete(int64_t ObjectID, IOStream& rFileData, // We can only do this when the file (ObjectID) doesn't already exist. ASSERT(refcount == 0); + // But RaidFileWrite won't allow us to write to a file that doesn't have exactly one + // reference, so we pretend for now that there is one. If the file already exists, + // then Open(false) below will raise an exception for us. RaidPutFileCompleteTransaction* pTrans = new RaidPutFileCompleteTransaction( - mStoreDiscSet, filename, refcount); + mStoreDiscSet, filename, 1); std::auto_ptr apTrans(pTrans); RaidFileWrite& rStoreFile(pTrans->GetRaidFile()); @@ -547,9 +551,14 @@ class RaidPutFilePatchTransaction : public BackupFileSystem::Transaction public: RaidPutFilePatchTransaction(int StoreDiscSet, const std::string& newCompleteFilename, - const std::string& reversedPatchFilename) - : mNewCompleteFile(StoreDiscSet, newCompleteFilename), - mReversedPatchFile(StoreDiscSet, reversedPatchFilename), + const std::string& reversedPatchFilename, + BackupStoreRefCountDatabase::refcount_t refcount) + // It's not quite true that mNewCompleteFile has 1 reference: it doesn't exist + // yet, so it has 0 right now. However when the transaction is committed it will + // have 1, and RaidFileWrite gets upset if we try to modify a file with != 1 + // references, so we need to pretend now that we already have the reference. + : mNewCompleteFile(StoreDiscSet, newCompleteFilename, 1), + mReversedPatchFile(StoreDiscSet, reversedPatchFilename, refcount), mReversedDiffIsCompletelyDifferent(false), mBlocksUsedByNewFile(0), mChangeInBlocksUsedByOldFile(0) @@ -593,7 +602,7 @@ void RaidPutFilePatchTransaction::Commit() std::auto_ptr RaidBackupFileSystem::PutFilePatch(int64_t ObjectID, int64_t DiffFromFileID, - IOStream& rPatchData) + IOStream& rPatchData, BackupStoreRefCountDatabase::refcount_t refcount) { // Create the containing directory if it doesn't exist. std::string newVersionFilename = GetObjectFileName(ObjectID, true); @@ -603,7 +612,7 @@ RaidBackupFileSystem::PutFilePatch(int64_t ObjectID, int64_t DiffFromFileID, false); // no need to make sure the directory it's in exists RaidPutFilePatchTransaction* pTrans = new RaidPutFilePatchTransaction( - mStoreDiscSet, newVersionFilename, oldVersionFilename); + mStoreDiscSet, newVersionFilename, oldVersionFilename, refcount); std::auto_ptr apTrans(pTrans); RaidFileWrite& rNewCompleteFile(pTrans->GetNewCompleteFile()); @@ -1470,6 +1479,77 @@ void S3BackupFileSystem::DeleteObjectUnknown(int64_t ObjectID) } + +class S3PutFileCompleteTransaction : public BackupFileSystem::Transaction +{ +private: + S3Client& mrClient; + std::string mFileURI; + bool mCommitted; + int64_t mNumBlocks; + +public: + S3PutFileCompleteTransaction(S3BackupFileSystem& fs, S3Client& client, + const std::string& file_uri, IOStream& file_data); + ~S3PutFileCompleteTransaction(); + virtual void Commit(); + virtual int64_t GetNumBlocks() { return mNumBlocks; } + + // It doesn't matter what we return here, because this should never be called + // for a PutFileCompleteTransaction (the API is intended for + // PutFilePatchTransaction instead): + virtual bool IsNewFileIndependent() { return false; } +}; + + +S3PutFileCompleteTransaction::S3PutFileCompleteTransaction(S3BackupFileSystem& fs, + S3Client& client, const std::string& file_uri, IOStream& file_data) +: mrClient(client), + mFileURI(file_uri), + mCommitted(false), + mNumBlocks(0) +{ + ByteCountingStream counter(file_data); + HTTPResponse response = mrClient.PutObject(file_uri, counter); + if(response.GetResponseCode() != HTTPResponse::Code_OK) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, FileUploadFailed, + "Failed to upload file to Amazon S3: " << + response.ResponseCodeString()); + } + mNumBlocks = fs.GetSizeInBlocks(counter.GetNumBytesRead()); +} + + +void S3PutFileCompleteTransaction::Commit() +{ + mCommitted = true; +} + + +S3PutFileCompleteTransaction::~S3PutFileCompleteTransaction() +{ + if(!mCommitted) + { + HTTPResponse response = mrClient.DeleteObject(mFileURI); + mrClient.CheckResponse(response, "Failed to delete uploaded file from Amazon S3", + true); // ExpectNoContent + } +} + + +std::auto_ptr +S3BackupFileSystem::PutFileComplete(int64_t ObjectID, IOStream& rFileData, + BackupStoreRefCountDatabase::refcount_t refcount) +{ + ASSERT(refcount == 0 || refcount == 1); + BackupStoreFile::VerifyStream validator(rFileData); + S3PutFileCompleteTransaction* pTrans = new S3PutFileCompleteTransaction(*this, + mrClient, GetFileURI(ObjectID), validator); + return std::auto_ptr(pTrans); +} + + //! GetObject() can be for either a file or a directory, so we need to try both. // TODO FIXME use a cached directory listing to determine which it is std::auto_ptr S3BackupFileSystem::GetObject(int64_t ObjectID, bool required) diff --git a/lib/backupstore/BackupFileSystem.h b/lib/backupstore/BackupFileSystem.h index a340dd894..aea1081ae 100644 --- a/lib/backupstore/BackupFileSystem.h +++ b/lib/backupstore/BackupFileSystem.h @@ -108,7 +108,8 @@ class BackupFileSystem virtual std::auto_ptr PutFileComplete(int64_t ObjectID, IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount) = 0; virtual std::auto_ptr PutFilePatch(int64_t ObjectID, - int64_t DiffFromFileID, IOStream& rPatchData) = 0; + int64_t DiffFromFileID, IOStream& rPatchData, + BackupStoreRefCountDatabase::refcount_t refcount) = 0; // GetObject() will retrieve either a file or directory, whichever exists. // GetFile() and GetDirectory() are only guaranteed to work on objects of the // correct type, but may be faster (depending on the implementation). @@ -145,6 +146,18 @@ class BackupFileSystem virtual CheckObjectsResult CheckObjects(bool fix_errors) = 0; virtual void EnsureObjectIsPermanent(int64_t ObjectID, bool fix_errors) = 0; + // CloseRefCountDatabase() closes the active database, saving changes to permanent + // storage if necessary. It invalidates any references to the current database! + // It is needed to allow a BackupStoreContext to be Finished, changes made to the + // BackupFileSystem's BackupStoreInfo by BackupStoreCheck and HousekeepStoreAccount, + // and the Context to be reopened. + void CloseRefCountDatabase(BackupStoreRefCountDatabase* p_refcount_db) + { + ASSERT(p_refcount_db == mapPermanentRefCountDatabase.get()); + SaveRefCountDatabase(*p_refcount_db); + mapPermanentRefCountDatabase.reset(); + } + protected: virtual std::auto_ptr GetBackupStoreInfoInternal(bool ReadOnly) = 0; std::auto_ptr mapBackupStoreInfo; @@ -228,7 +241,8 @@ class RaidBackupFileSystem : public BackupFileSystem virtual std::auto_ptr PutFileComplete(int64_t ObjectID, IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount); virtual std::auto_ptr PutFilePatch(int64_t ObjectID, - int64_t DiffFromFileID, IOStream& rPatchData); + int64_t DiffFromFileID, IOStream& rPatchData, + BackupStoreRefCountDatabase::refcount_t refcount); virtual std::auto_ptr GetFile(int64_t ObjectID) { // For RaidBackupFileSystem, GetObject() is equivalent to GetFile(). @@ -326,14 +340,12 @@ class S3BackupFileSystem : public BackupFileSystem virtual void GetDirectory(int64_t ObjectID, BackupStoreDirectory& rDirOut); virtual void PutDirectory(BackupStoreDirectory& rDir); virtual std::auto_ptr PutFileComplete(int64_t ObjectID, - IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount) - { - return std::auto_ptr(); - } + IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount); virtual std::auto_ptr PutFilePatch(int64_t ObjectID, - int64_t DiffFromFileID, IOStream& rPatchData) + int64_t DiffFromFileID, IOStream& rPatchData, + BackupStoreRefCountDatabase::refcount_t refcount) { - return std::auto_ptr(); + return PutFileComplete(ObjectID, rPatchData, refcount); } virtual std::auto_ptr GetFile(int64_t ObjectID); virtual std::auto_ptr GetFilePatch(int64_t ObjectID, diff --git a/lib/backupstore/BackupStoreContext.cpp b/lib/backupstore/BackupStoreContext.cpp index bc88464b4..d622b0e3e 100644 --- a/lib/backupstore/BackupStoreContext.cpp +++ b/lib/backupstore/BackupStoreContext.cpp @@ -31,14 +31,8 @@ // Maximum number of directories to keep in the cache When the cache is bigger -// than this, everything gets deleted. In tests, we set the cache size to zero -// to ensure that it's always flushed, which is very inefficient but helps to -// catch programming errors (use of freed data). -#ifdef BOX_RELEASE_BUILD - #define MAX_CACHE_SIZE 32 -#else - #define MAX_CACHE_SIZE 0 -#endif +// than this, everything gets deleted. +#define MAX_CACHE_SIZE 32 // Allow the housekeeping process 4 seconds to release an account #define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT 4 @@ -47,16 +41,16 @@ #define STORE_INFO_SAVE_DELAY 96 #define CHECK_FILESYSTEM_INITIALISED() \ - if(mapStoreInfo.get() == 0) \ + if(!mpFileSystem) \ { \ - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) \ + THROW_EXCEPTION(BackupStoreException, FileSystemNotInitialised); \ } // -------------------------------------------------------------------------- // // Function // Name: BackupStoreContext::BackupStoreContext() -// Purpose: Constructor +// Purpose: Traditional constructor (for RAID filesystems only) // Created: 2003/08/20 // // -------------------------------------------------------------------------- @@ -67,13 +61,37 @@ BackupStoreContext::BackupStoreContext(int32_t ClientID, mpHousekeeping(pHousekeeping), mProtocolPhase(Phase_START), mClientHasAccount(false), - mStoreDiscSet(-1), mReadOnly(true), mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), mpFileSystem(NULL), mpTestHook(NULL) // If you change the initialisers, be sure to update -// BackupStoreContext::CleanUp as well! +// BackupStoreContext::ReceivedFinishCommand as well! +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::BackupStoreContext() +// Purpose: New constructor (for any type of BackupFileSystem) +// Created: 2015/11/02 +// +// -------------------------------------------------------------------------- +BackupStoreContext::BackupStoreContext(BackupFileSystem& rFileSystem, int32_t ClientID, + HousekeepingInterface* pHousekeeping, const std::string& rConnectionDetails) +: mConnectionDetails(rConnectionDetails), + mClientID(ClientID), + mpHousekeeping(pHousekeeping), + mProtocolPhase(Phase_START), + mClientHasAccount(false), + mReadOnly(true), + mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), + mpFileSystem(&rFileSystem), + mpTestHook(NULL) +// If you change the initialisers, be sure to update +// BackupStoreContext::ReceivedFinishCommand as well! { } @@ -88,7 +106,15 @@ BackupStoreContext::BackupStoreContext(int32_t ClientID, // -------------------------------------------------------------------------- BackupStoreContext::~BackupStoreContext() { - ClearDirectoryCache(); + try + { + ClearDirectoryCache(); + ReleaseWriteLock(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to clean up BackupStoreContext: " << e.what()); + } } @@ -97,7 +123,7 @@ void BackupStoreContext::ClearDirectoryCache() // Delete the objects in the cache for(auto i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) { - delete (i->second); + delete i->second; } mDirectoryCache.clear(); } @@ -113,6 +139,13 @@ void BackupStoreContext::ClearDirectoryCache() // -------------------------------------------------------------------------- void BackupStoreContext::CleanUp() { + if(!mpFileSystem) + { + return; + } + + CHECK_FILESYSTEM_INITIALISED(); + if(!mReadOnly) { // Make sure the store info is saved, if it has been loaded, isn't @@ -123,6 +156,17 @@ void BackupStoreContext::CleanUp() // Save the store info, not delayed SaveStoreInfo(false); } + // Ask the BackupFileSystem to clear its BackupStoreInfo, so that we don't use + // a stale one if the Context is later reused. + mpFileSystem->DiscardBackupStoreInfo(info); + + // Make sure the refcount database is saved too, and removed from the + // BackupFileSystem (in case it's modified before reopening). + if(mpRefCount) + { + mpFileSystem->CloseRefCountDatabase(mpRefCount); + mpRefCount = NULL; + } } ReleaseWriteLock(); @@ -131,14 +175,10 @@ void BackupStoreContext::CleanUp() // put the context back to its initial state. mProtocolPhase = BackupStoreContext::Phase_Version; - // Avoid the need to check version again, by not resetting - // mClientHasAccount, mAccountRootDir or mStoreDiscSet - + // Avoid the need to check version again, by not resetting mClientHasAccount. mReadOnly = true; mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; mpTestHook = NULL; - mapStoreInfo.reset(); - mapRefCount.reset(); ClearDirectoryCache(); } @@ -153,36 +193,72 @@ void BackupStoreContext::CleanUp() // -------------------------------------------------------------------------- bool BackupStoreContext::AttemptToGetWriteLock() { - // Make the filename of the write lock file - std::string writeLockFile; - StoreStructure::MakeWriteLockFilename(mAccountRootDir, mStoreDiscSet, writeLockFile); + CHECK_FILESYSTEM_INITIALISED(); // Request the lock - bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + bool gotLock = false; - if(!gotLock && mpHousekeeping) + try + { + mpFileSystem->TryGetLock(); + // If we got to here, then it worked! + gotLock = true; + } + catch(BackupStoreException &e) { - // The housekeeping process might have the thing open -- ask it to stop - char msg[256]; - int msgLen = snprintf(msg, sizeof(msg), "r%x\n", mClientID); - // Send message - mpHousekeeping->SendMessageToHousekeepingProcess(msg, msgLen); + if(!EXCEPTION_IS_TYPE(e, BackupStoreException, CouldNotLockStoreAccount)) + { + // We don't know what this error is. + throw; + } - // Then try again a few times - int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; - do + if(mpHousekeeping) { - ::sleep(1 /* second */); - --tries; - gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + // The housekeeping process might have the thing open -- ask it to stop + char msg[256]; + int msgLen = snprintf(msg, sizeof(msg), "r%x\n", mClientID); + + // Send message + mpHousekeeping->SendMessageToHousekeepingProcess(msg, msgLen); - } while(!gotLock && tries > 0); + // Then try again a few times + for(int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; + tries >= 0; tries--) + { + try + { + ::sleep(1 /* second */); + mpFileSystem->TryGetLock(); + // If we got to here, then it worked! + gotLock = true; + break; + } + catch(BackupStoreException &e) + { + if(EXCEPTION_IS_TYPE(e, BackupStoreException, + CouldNotLockStoreAccount)) + { + // keep trying + } + else + { + // We don't know what this error is. + throw; + } + } + } + } } if(gotLock) { // Got the lock, mark as not read only mReadOnly = false; + + // GetDirectoryInternal assumes that if we have the write lock, everything in the + // cache was loaded with that lock, and cannot be stale. That would be violated if + // we had anything in the cache already when the lock was obtained, so clear it now. + ClearDirectoryCache(); } return gotLock; @@ -199,8 +275,6 @@ void BackupStoreContext::SetClientHasAccount(const std::string &rStoreRoot, mapOwnFileSystem.reset( new RaidBackupFileSystem(mClientID, rStoreRoot, StoreDiscSet)); mpFileSystem = mapOwnFileSystem.get(); - mAccountRootDir = rStoreRoot; - mStoreDiscSet = StoreDiscSet; } // -------------------------------------------------------------------------- @@ -213,32 +287,22 @@ void BackupStoreContext::SetClientHasAccount(const std::string &rStoreRoot, // -------------------------------------------------------------------------- void BackupStoreContext::LoadStoreInfo() { - if(mapStoreInfo.get() != 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) - } - - // Load it up! - std::auto_ptr i(BackupStoreInfo::Load(mClientID, mAccountRootDir, mStoreDiscSet, mReadOnly)); - - // Check it - if(i->GetAccountID() != mClientID) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) - } - - // Keep the pointer to it - mapStoreInfo = i; + CHECK_FILESYSTEM_INITIALISED(); - BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet); + // Load it up! This checks the account ID on RaidBackupFileSystem backends, + // but not on S3BackupFileSystem which don't use account IDs. + GetBackupStoreInfo(); // Try to load the reference count database try { - mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + mpRefCount = &(mpFileSystem->GetPermanentRefCountDatabase(mReadOnly)); } catch(BoxException &e) { + // Do not create a new refcount DB here, it is not safe! Users may wonder + // why they have lost all their files, and/or unwittingly overwrite their + // backup data. THROW_EXCEPTION_MESSAGE(BackupStoreException, CorruptReferenceCountDatabase, "Account " << BOX_FORMAT_ACCOUNT(mClientID) << " reference count database is " @@ -258,7 +322,10 @@ void BackupStoreContext::LoadStoreInfo() // -------------------------------------------------------------------------- void BackupStoreContext::SaveStoreInfo(bool AllowDelay) { - CHECK_FILESYSTEM_INITIALISED(); + if(!mpFileSystem) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } if(mReadOnly) { @@ -276,84 +343,93 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) } // Want to save now - mapStoreInfo->Save(); + CHECK_FILESYSTEM_INITIALISED(); + mpFileSystem->PutBackupStoreInfo(GetBackupStoreInfoInternal()); // Reset counter for next delayed save. mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; } - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreContext::MakeObjectFilename(int64_t, std::string &, bool) -// Purpose: Create the filename of an object in the store, optionally creating the -// containing directory if it doesn't already exist. -// Created: 2003/09/02 -// -// -------------------------------------------------------------------------- -void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) -{ - // Delegate to utility function - StoreStructure::MakeObjectFilename(ObjectID, mAccountRootDir, mStoreDiscSet, rOutput, EnsureDirectoryExists); -} - - // -------------------------------------------------------------------------- // // Function // Name: BackupStoreContext::GetDirectoryInternal(int64_t, // bool) -// Purpose: Return a reference to a directory. Valid only until -// the next time a function which affects directories -// is called. Mainly this function, and creation of -// files. Private version of this, which returns -// non-const directories. Unless called with -// AllowFlushCache == false, the cache may be flushed, -// invalidating all directory references that you may -// be holding, so beware. +// Purpose: Return a reference to a directory, valid only until +// the next time a function which may flush the +// directory cache is called: mainly this function +// (with AllowFlushCache == true) or creation of files. +// This is a private function which returns non-const +// references to directories in the cache. It will +// invalidate all directory references that you may be +// holding, except for the one that it returns. // Created: 2003/09/02 // // -------------------------------------------------------------------------- BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID, bool AllowFlushCache) { + CHECK_FILESYSTEM_INITIALISED(); + +#ifndef BOX_RELEASE_BUILD + // In debug builds, if AllowFlushCache is true, we invalidate the entire cache. That's + // because it could be flushed at any time, invalidating any pointers held to any entry + // except the one returned by this function. Invalidating makes all attempted accesses + // throw exceptions, so it should catch any such programming error. We will need to + // uninvalidate whatever entry we return, before returning it, otherwise it cannot be used. + if(AllowFlushCache) + { + for(auto i = mDirectoryCache.begin(); i != mDirectoryCache.end(); i++) + { + i->second->Invalidate(true); + } + } +#endif + // Get the filename - std::string filename; - MakeObjectFilename(ObjectID, filename); int64_t oldRevID = 0, newRevID = 0; + bool gotRevID = false; // Already in cache? auto item = mDirectoryCache.find(ObjectID); if(item != mDirectoryCache.end()) { -#ifndef BOX_RELEASE_BUILD // it might be in the cache, but invalidated - // in which case, delete it instead of returning it. - if(!item->second->IsInvalidated()) -#else - if(true) +#ifndef BOX_RELEASE_BUILD + // Uninvalidate this one entry (we invalidated them all above): + item->second->Invalidate(false); #endif + oldRevID = item->second->GetRevisionID(); + + // Check the revision ID of the file -- does it need refreshing? + // We assume that if we have the write lock, everything in the cache was loaded + // with that lock held, and therefore cannot be stale. + if(!mReadOnly) { - oldRevID = item->second->GetRevisionID(); + // Looks good... return the cached object + BOX_TRACE("Returning locked object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache, modtime = " << oldRevID) + return *(item->second); + } - // Check the revision ID of the file -- does it need refreshing? - if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &newRevID)) - { - THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) - } + if(!mpFileSystem->ObjectExists(ObjectID, &newRevID)) + { + THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) + } - if(newRevID == oldRevID) - { - // Looks good... return the cached object - BOX_TRACE("Returning object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " from cache, modtime = " << newRevID) - return *(item->second); - } + gotRevID = true; + + if(newRevID == oldRevID) + { + // Looks good... return the cached object + BOX_TRACE("Returning object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache, modtime = " << newRevID) + return *(item->second); } - // Delete this cached object + // The cached object is stale, so remove it from the cache. delete item->second; mDirectoryCache.erase(item); } @@ -363,26 +439,21 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID, // First check to see if the cache is too big if(mDirectoryCache.size() > MAX_CACHE_SIZE && AllowFlushCache) { - // Very simple. Just delete everything! But in debug builds, - // leave the entries in the cache and invalidate them instead, - // so that any attempt to access them will cause an assertion - // failure that helps to track down the error. -#ifdef BOX_RELEASE_BUILD + // Trivial policy: just delete everything! ClearDirectoryCache(); -#else - for(std::map::iterator - i = mDirectoryCache.begin(); - i != mDirectoryCache.end(); i++) + } + + if(!gotRevID) + { + // We failed to find it in the cache, so it might not exist at all (if it was in + // the cache then it definitely does). Check for it now: + if(!mpFileSystem->ObjectExists(ObjectID, &newRevID)) { - i->second->Invalidate(); + THROW_EXCEPTION(BackupStoreException, ObjectDoesNotExist); } -#endif } - // Get a RaidFileRead to read it - std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, - filename, &newRevID)); - + // Get an IOStream to read it in ASSERT(newRevID != 0); if (oldRevID == 0) @@ -398,29 +469,14 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID, } // Read it from the stream, then set it's revision ID - BufferedStream buf(*objectFile); - std::auto_ptr dir(new BackupStoreDirectory(buf)); - dir->SetRevisionID(newRevID); - - // Make sure the size of the directory is available for writing the dir back - int64_t dirSize = objectFile->GetDiscUsageInBlocks(); - ASSERT(dirSize > 0); - dir->SetUserInfo1_SizeInBlocks(dirSize); + std::auto_ptr dir(new BackupStoreDirectory); + mpFileSystem->GetDirectory(ObjectID, *dir); // Store in cache - BackupStoreDirectory *pdir = dir.release(); - try - { - mDirectoryCache[ObjectID] = pdir; - } - catch(...) - { - delete pdir; - throw; - } + mDirectoryCache[ObjectID] = dir.get(); - // Return it - return *pdir; + // Since it's freshly loaded, it won't be invalidated, and we can just return it: + return *(dir.release()); } @@ -448,11 +504,8 @@ int64_t BackupStoreContext::AllocateObjectID() BackupStoreInfo& info(GetBackupStoreInfoInternal()); int64_t id = info.AllocateObjectID(); - // Generate filename - std::string filename; - MakeObjectFilename(id, filename); // Check it doesn't exist - if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + if(!mpFileSystem->ObjectExists(id)) { // Success! return id; @@ -511,195 +564,52 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t id = AllocateObjectID(); // Stream the file to disc - std::string fn; - MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); - int64_t newObjectBlocksUsed = 0; - RaidFileWrite *ppreviousVerStoreFile = 0; - bool reversedDiffIsCompletelyDifferent = false; - int64_t oldVersionNewBlocksUsed = 0; BackupStoreInfo::Adjustment adjustment = {}; - BackupStoreInfo& info(GetBackupStoreInfoInternal()); + std::auto_ptr apTransaction; - try + // Diff or full file? + if(DiffFromFileID == 0) { - RaidFileWrite storeFile(mStoreDiscSet, fn); - storeFile.Open(false /* no overwriting */); - - int64_t spaceSavedByConversionToPatch = 0; - - // Diff or full file? - if(DiffFromFileID == 0) - { - // A full file, just store to disc - try - { - rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT); - } - catch(CommonException &e) - { - if(EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, - ReadFileFromStreamTimedOut, e.GetMessage()); - } - else - { - throw; - } - } - } - else - { - // Check that the diffed from ID actually exists in the directory - if(dir.FindEntryByID(DiffFromFileID) == 0) - { - THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) - } - - // Diff file, needs to be recreated. - // Choose a temporary filename. - std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", - 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); - - try - { - // Open it twice -#ifdef WIN32 - InvisibleTempFileStream diff(tempFn.c_str(), - O_RDWR | O_CREAT | O_BINARY); - InvisibleTempFileStream diff2(tempFn.c_str(), - O_RDWR | O_BINARY); -#else - FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); - FileStream diff2(tempFn.c_str(), O_RDONLY); - - // Unlink it immediately, so it definitely goes away - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } -#endif - - // Stream the incoming diff to this temporary file - try - { - rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT); - } - catch(CommonException &e) - { - if(EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, - ReadFileFromStreamTimedOut, e.GetMessage()); - } - else - { - throw; - } - } - - // Verify the diff - diff.Seek(0, IOStream::SeekType_Absolute); - if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) - { - THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) - } - - // Seek to beginning of diff file - diff.Seek(0, IOStream::SeekType_Absolute); - - // Filename of the old version - std::string oldVersionFilename; - MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); - - // Reassemble that diff -- open previous file, and combine the patch and file - std::auto_ptr from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); - BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); - - // Then... reverse the patch back (open the from file again, and create a write file to overwrite it) - std::auto_ptr from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); - ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename); - ppreviousVerStoreFile->Open(true /* allow overwriting */); - from->Seek(0, IOStream::SeekType_Absolute); - diff.Seek(0, IOStream::SeekType_Absolute); - BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, - DiffFromFileID, &reversedDiffIsCompletelyDifferent); - - // Store disc space used - oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); - - // And make a space adjustment for the size calculation - spaceSavedByConversionToPatch = - from->GetDiscUsageInBlocks() - - oldVersionNewBlocksUsed; - - adjustment.mBlocksUsed -= spaceSavedByConversionToPatch; - // The code below will change the patch from a - // Current file to an Old file, so we need to - // account for it as a Current file here. - adjustment.mBlocksInCurrentFiles -= - spaceSavedByConversionToPatch; - - // Don't adjust anything else here. We'll do it - // when we update the directory just below, - // which also accounts for non-diff replacements. - - // Everything cleans up here... - } - catch(...) - { - // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway - ::unlink(tempFn.c_str()); - throw; - } - } - - // Get the blocks used - newObjectBlocksUsed = storeFile.GetDiscUsageInBlocks(); - adjustment.mBlocksUsed += newObjectBlocksUsed; - adjustment.mBlocksInCurrentFiles += newObjectBlocksUsed; - adjustment.mNumCurrentFiles++; - - // Exceeds the hard limit? - int64_t newTotalBlocksUsed = info.GetBlocksUsed() + adjustment.mBlocksUsed; - if(newTotalBlocksUsed > info.GetBlocksHardLimit()) - { - THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) - // The store file will be deleted automatically by the RaidFile object - } - - // Commit the file - storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + apTransaction = mpFileSystem->PutFileComplete(id, rFile, + 0); // refcount: BackupStoreFile requires us to pass 0 to assert that + // the file doesn't already exist, because it will refuse to overwrite an + // existing file. The refcount will increase to 1 when we commit the change + // to the directory, dir. } - catch(...) + else { - // Delete any previous version store file - if(ppreviousVerStoreFile != 0) + // Check that the diffed from ID actually exists in the directory + if(dir.FindEntryByID(DiffFromFileID) == 0) { - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; + THROW_EXCEPTION(BackupStoreException, + DiffFromIDNotFoundInDirectory) } - throw; + apTransaction = mpFileSystem->PutFilePatch(id, DiffFromFileID, + rFile, mpRefCount->GetRefCount(DiffFromFileID)); } - // Verify the file -- only necessary for non-diffed versions - // NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because - // in the non-diffed code path it's never allocated. - if(DiffFromFileID == 0) - { - std::auto_ptr checkFile(RaidFileRead::Open(mStoreDiscSet, fn)); - if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile)) - { - // Error! Delete the file - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); + // Get the blocks used + int64_t changeInBlocksUsed = apTransaction->GetNumBlocks() + + apTransaction->GetChangeInBlocksUsedByOldFile(); + adjustment.mBlocksUsed += changeInBlocksUsed; + adjustment.mBlocksInCurrentFiles += changeInBlocksUsed; + adjustment.mNumCurrentFiles++; - // Exception - THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) - } + // Exceeds the hard limit? + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + int64_t newTotalBlocksUsed = info.GetBlocksUsed() + changeInBlocksUsed; + if(newTotalBlocksUsed > info.GetBlocksHardLimit()) + { + // This will cancel the Transaction and delete the RaidFile(s). + THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) } + // Can only get this before we commit the RaidFiles. + int64_t numBlocksInNewFile = apTransaction->GetNumBlocks(); + int64_t changeInBlocksUsedByOldFile = + apTransaction->GetChangeInBlocksUsedByOldFile(); + // Modify the directory -- first make all files with the same name // marked as an old version try @@ -716,7 +626,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Adjust size of old entry int64_t oldSize = poldEntry->GetSizeInBlocks(); - poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); + poldEntry->SetSizeInBlocks(oldSize + changeInBlocksUsedByOldFile); } if(MarkFileWithSameNameAsOldVersions) @@ -748,50 +658,36 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Then the new entry BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, - ModificationTime, id, newObjectBlocksUsed, + ModificationTime, id, numBlocksInNewFile, BackupStoreDirectory::Entry::Flags_File, AttributesHash); // Adjust dependency info of file? - if(DiffFromFileID && poldEntry && !reversedDiffIsCompletelyDifferent) + if(DiffFromFileID && poldEntry && !apTransaction->IsNewFileIndependent()) { poldEntry->SetDependsNewer(id); pnewEntry->SetDependsOlder(DiffFromFileID); } - // Write the directory back to disc + // Save the directory back SaveDirectory(dir); - // Commit the old version's new patched version, now that the directory safely reflects - // the state of the files on disc. - if(ppreviousVerStoreFile != 0) - { - ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } + // It is now safe to commit the old version's new patched version, now + // that the directory safely reflects the state of the files on disc. } catch(...) { - // Back out on adding that file - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); - // Remove this entry from the cache RemoveDirectoryFromCache(InDirectory); - // Delete any previous version store file - if(ppreviousVerStoreFile != 0) - { - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } - - // Don't worry about the incremented number in the store info + // Leaving this function without committing the Transaction will cancel + // it, deleting the new file and not modifying the old one. Don't worry + // about the incremented numbers in the store info, they won't cause + // any real problem and bbstoreaccounts check can fix them. throw; } - // Check logic - ASSERT(ppreviousVerStoreFile == 0); + // Commit the new file, and replace the old file (if any) with a patch. + apTransaction->Commit(); // Modify the store info info.AdjustNumCurrentFiles(adjustment.mNumCurrentFiles); @@ -805,7 +701,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, info.ChangeBlocksInDirectories(adjustment.mBlocksInDirectories); // Increment reference count on the new directory to one - mapRefCount->AddReference(id); + mpRefCount->AddReference(id); // Save the store info -- can cope if this exceptions because information // will be rebuilt by housekeeping, and ID allocation can recover. @@ -896,6 +792,8 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ { // Save the directory back SaveDirectory(dir); + + // Maybe postponed save of store info SaveStoreInfo(false); } } @@ -1015,7 +913,11 @@ void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID) // // Function // Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &) -// Purpose: Save directory back to disc, update time in cache +// Purpose: Save directory back to disc, update time in cache. +// Since this updates the parent directory, it needs to +// fetch it, which invalidates rDir along with the rest +// of the cache. But since it's usually the last thing +// we do to rDir, that should be fine. // Created: 2003/09/04 // // -------------------------------------------------------------------------- @@ -1024,68 +926,43 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir) CHECK_FILESYSTEM_INITIALISED(); int64_t ObjectID = rDir.GetObjectID(); + auto i = mDirectoryCache.find(ObjectID); + ASSERT(i != mDirectoryCache.end()); + + int64_t new_dir_size; try { // Write to disc, adjust size in store info - std::string dirfn; - MakeObjectFilename(ObjectID, dirfn); int64_t old_dir_size = rDir.GetUserInfo1_SizeInBlocks(); - { - RaidFileWrite writeDir(mStoreDiscSet, dirfn); - writeDir.Open(true /* allow overwriting */); - - BufferedWriteStream buffer(writeDir); - rDir.WriteToStream(buffer); - buffer.Flush(); + mpFileSystem->PutDirectory(rDir); + new_dir_size = rDir.GetUserInfo1_SizeInBlocks(); - // get the disc usage (must do this before commiting it) - int64_t dirSize = writeDir.GetDiscUsageInBlocks(); - - // Commit directory - writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - - // Make sure the size of the directory is available for writing the dir back - ASSERT(dirSize > 0); - int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); + { + int64_t sizeAdjustment = new_dir_size - old_dir_size; BackupStoreInfo& info(GetBackupStoreInfoInternal()); info.ChangeBlocksUsed(sizeAdjustment); info.ChangeBlocksInDirectories(sizeAdjustment); - // Update size stored in directory - rDir.SetUserInfo1_SizeInBlocks(dirSize); } - // Refresh revision ID in cache - { - int64_t revid = 0; - if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid)) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - BOX_TRACE("Saved directory " << - BOX_FORMAT_OBJECTID(ObjectID) << - ", modtime = " << revid); - - rDir.SetRevisionID(revid); - } // Update the directory entry in the grandparent, to ensure // that it reflects the current size of the parent directory. - int64_t new_dir_size = rDir.GetUserInfo1_SizeInBlocks(); if(new_dir_size != old_dir_size && ObjectID != BACKUPSTORE_ROOT_DIRECTORY_ID) { int64_t ContainerID = rDir.GetContainerID(); BackupStoreDirectory& parent( GetDirectoryInternal(ContainerID)); - // rDir is now invalid + // i and rDir are now invalid BackupStoreDirectory::Entry* en = parent.FindEntryByID(ObjectID); if(!en) { - BOX_ERROR("Missing entry for directory " << + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CouldNotFindEntryInDirectory, + "Missing entry for directory " << BOX_FORMAT_OBJECTID(ObjectID) << " in directory " << BOX_FORMAT_OBJECTID(ContainerID) << @@ -1160,33 +1037,28 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, int64_t id = AllocateObjectID(); // Create an empty directory with the given attributes on disc - std::string fn; - MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); int64_t dirSize; { BackupStoreDirectory emptyDir(id, InDirectory); - // add the atttribues + // Add the attributes: emptyDir.SetAttributes(Attributes, AttributesModTime); - // Write... - RaidFileWrite dirFile(mStoreDiscSet, fn); - dirFile.Open(false /* no overwriting */); - emptyDir.WriteToStream(dirFile); - // Get disc usage, before it's commited - dirSize = dirFile.GetDiscUsageInBlocks(); + // Write, but not using SaveDirectory() because that tries to update the entry + // in the parent directory with the new size, and that entry hasn't been added yet! + mpFileSystem->PutDirectory(emptyDir); + dirSize = emptyDir.GetUserInfo1_SizeInBlocks(); + } + { // Exceeds the hard limit? int64_t newTotalBlocksUsed = info.GetBlocksUsed() + dirSize; if(newTotalBlocksUsed > info.GetBlocksHardLimit()) { + mpFileSystem->DeleteDirectory(id); THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) - // The file will be deleted automatically by the RaidFile object } - // Commit the file - dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - // Make sure the size of the directory is added to the usage counts in the info ASSERT(dirSize > 0); info.ChangeBlocksUsed(dirSize); @@ -1201,19 +1073,30 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, 0 /* attributes hash */); SaveDirectory(dir); - // Increment reference count on the new directory to one - mapRefCount->AddReference(id); + // Increment reference count on the new directory to one: + mpRefCount->AddReference(id); } catch(...) { - // Back out on adding that directory - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); + // Back out on adding that directory: + mpFileSystem->DeleteDirectory(id); - // Remove this entry from the cache - RemoveDirectoryFromCache(InDirectory); + info.ChangeBlocksUsed(-dirSize); + info.ChangeBlocksInDirectories(-dirSize); + + // Remove the newly created directory from the cache: + RemoveDirectoryFromCache(id); - // Don't worry about the incremented number in the store info + // Try to remove the new entry from this directory. If + // SaveDirectory() above failed because there was something + // wrong with this directory, then it's likely to fail again, + // so we do this last. Also, SaveDirectory() above invalidated + // dir, so we need to fetch it again. + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + dir.DeleteEntry(id); + SaveDirectory(dir); + + // Don't worry about the incremented number in the store info. throw; } @@ -1400,7 +1283,6 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete) changesMade = true; } - // Save the directory if(changesMade) { SaveDirectory(dir); @@ -1535,20 +1417,17 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) return false; } - // Test to see if it exists on the disc - std::string filename; - MakeObjectFilename(ObjectID, filename); - if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + if (!mpFileSystem->ObjectExists(ObjectID)) { - // RaidFile reports no file there return false; } // Do we need to be more specific? if(MustBe != ObjectExists_Anything) { - // Open the file - std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); + // Open the file. TODO FIXME: don't download the entire file from S3 + // to read the first four bytes. + std::auto_ptr objectFile = mpFileSystem->GetObject(ObjectID); // Read the first integer uint32_t magic; @@ -1594,10 +1473,98 @@ std::auto_ptr BackupStoreContext::OpenObject(int64_t ObjectID) { CHECK_FILESYSTEM_INITIALISED(); - // Attempt to open the file - std::string fn; - MakeObjectFilename(ObjectID, fn); - return std::auto_ptr(RaidFileRead::Open(mStoreDiscSet, fn).release()); + return mpFileSystem->GetObject(ObjectID); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetFile() +// Purpose: Retrieve a file from the store +// Created: 2015/08/10 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupStoreContext::GetFile(int64_t ObjectID, int64_t InDirectory) +{ + CHECK_FILESYSTEM_INITIALISED(); + + // Get the directory it's in + const BackupStoreDirectory &rdir(GetDirectoryInternal(InDirectory, + true)); // AllowFlushCache + + // Find the object within the directory + BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(ObjectID); + if(pfileEntry == 0) + { + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory); + } + + // The result + std::auto_ptr stream; + + // Does this depend on anything? + if(pfileEntry->GetDependsNewer() != 0) + { + // File exists, but is a patch from a new version. Generate the older version. + std::vector patchChain; + int64_t id = ObjectID; + BackupStoreDirectory::Entry *en = 0; + do + { + patchChain.push_back(id); + en = rdir.FindEntryByID(id); + if(en == 0) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + PatchChainInfoBadInDirectory, + "Object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " in dir " << + BOX_FORMAT_OBJECTID(InDirectory) << + " for account " << + BOX_FORMAT_ACCOUNT(mClientID) << + " references object " << + BOX_FORMAT_OBJECTID(id) << + " which does not exist in dir"); + } + id = en->GetDependsNewer(); + } + while(en != 0 && id != 0); + + stream = mpFileSystem->GetFilePatch(ObjectID, patchChain); + } + else + { + // Simple case: file already exists on disc ready to go + + // Open the object + std::auto_ptr object(OpenObject(ObjectID)); + BufferedStream buf(*object); + + // Verify it + if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) + { + THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify); + } + + // Reset stream -- seek to beginning + object->Seek(0, IOStream::SeekType_Absolute); + + // Reorder the stream/file into stream order + { + // Write nastily to allow this to work with gcc 2.x + std::auto_ptr t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */)); + stream = t; + } + + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid + // premature deletion + object.release(); + } + + return stream; } diff --git a/lib/backupstore/BackupStoreContext.h b/lib/backupstore/BackupStoreContext.h index d888eef7e..2d607a5fd 100644 --- a/lib/backupstore/BackupStoreContext.h +++ b/lib/backupstore/BackupStoreContext.h @@ -19,7 +19,6 @@ #include "BackupFileSystem.h" #include "BackupStoreInfo.h" #include "BackupStoreRefCountDatabase.h" -#include "NamedLock.h" #include "Message.h" #include "Utils.h" @@ -50,6 +49,9 @@ class BackupStoreContext BackupStoreContext(int32_t ClientID, HousekeepingInterface* mpHousekeeping, const std::string& rConnectionDetails); + BackupStoreContext(BackupFileSystem& rFileSystem, int32_t ClientID, + HousekeepingInterface* mpHousekeeping, + const std::string& rConnectionDetails); virtual ~BackupStoreContext(); private: @@ -90,16 +92,20 @@ class BackupStoreContext // Not really an API, but useful for BackupProtocolLocal2. void ReleaseWriteLock() { - if(mWriteLock.GotLock()) - { - mWriteLock.ReleaseLock(); - } + // Even a read-only filesystem may hold some locks, for example + // S3BackupFileSystem's cache lock, so we always notify the filesystem + // to release any locks that it holds, even if we are read-only. + mpFileSystem->ReleaseLock(); } + // TODO: stop using this version, which has the side-effect of creating a + // BackupStoreFileSystem: void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet); + void SetClientHasAccount() + { + mClientHasAccount = true; + } bool GetClientHasAccount() const {return mClientHasAccount;} - const std::string &GetAccountRoot() const {return mAccountRootDir;} - int GetStoreDiscSet() const {return mStoreDiscSet;} // Store info void LoadStoreInfo(); @@ -113,11 +119,14 @@ class BackupStoreContext const std::string GetAccountName() { - if(!mapStoreInfo.get()) + if(!mpFileSystem) { - return "Unknown"; + // This can happen if the account doesn't exist on the server, e.g. not + // created yet, because BackupStoreDaemon doesn't call + // SetClientHasAccount(), which creates the BackupStoreFileSystem. + return "no such account"; } - return mapStoreInfo->GetAccountName(); + return mpFileSystem->GetBackupStoreInfo(true).GetAccountName(); // ReadOnly } // Client marker @@ -182,6 +191,7 @@ class BackupStoreContext }; bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything); std::auto_ptr OpenObject(int64_t ObjectID); + std::auto_ptr GetFile(int64_t ObjectID, int64_t InDirectory); // Info int32_t GetClientID() const {return mClientID;} @@ -189,7 +199,6 @@ class BackupStoreContext virtual int GetBlockSize() { return mpFileSystem->GetBlockSize(); } private: - void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID, bool AllowFlushCache = true); void SaveDirectory(BackupStoreDirectory &rDir); @@ -203,16 +212,9 @@ class BackupStoreContext HousekeepingInterface *mpHousekeeping; int mProtocolPhase; bool mClientHasAccount; - std::string mAccountRootDir; // has final directory separator - int mStoreDiscSet; - bool mReadOnly; - NamedLock mWriteLock; int mSaveStoreInfoDelay; // how many times to delay saving the store info - // Store info - std::auto_ptr mapStoreInfo; - // mapOwnFileSystem is initialised when we created our own BackupFileSystem, // using the old constructor. It ensures that the BackupFileSystem is deleted // when this BackupStoreContext is destroyed. TODO: stop using that old @@ -227,15 +229,15 @@ class BackupStoreContext // Non-const version for internal use: BackupStoreInfo& GetBackupStoreInfoInternal() const { - if(!mapStoreInfo.get()) + if(!mpFileSystem) { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded); + THROW_EXCEPTION(BackupStoreException, FileSystemNotInitialised); } - return *mapStoreInfo; + return mpFileSystem->GetBackupStoreInfo(mReadOnly); } // Refcount database - std::auto_ptr mapRefCount; + BackupStoreRefCountDatabase* mpRefCount; // Directory cache std::map mDirectoryCache; diff --git a/lib/backupstore/BackupStoreDirectory.h b/lib/backupstore/BackupStoreDirectory.h index 788a3ad03..25cf2d754 100644 --- a/lib/backupstore/BackupStoreDirectory.h +++ b/lib/backupstore/BackupStoreDirectory.h @@ -36,13 +36,13 @@ class BackupStoreDirectory public: #ifndef BOX_RELEASE_BUILD - void Invalidate() + void Invalidate(bool invalid = true) { - mInvalidated = true; + mInvalidated = invalid; for (std::vector::iterator i = mEntries.begin(); i != mEntries.end(); i++) { - (*i)->Invalidate(); + (*i)->Invalidate(invalid); } } #endif @@ -86,7 +86,7 @@ class BackupStoreDirectory public: #ifndef BOX_RELEASE_BUILD - void Invalidate() { mInvalidated = true; } + void Invalidate(bool invalid) { mInvalidated = invalid; } #endif friend class BackupStoreDirectory; diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt index 1dee4470b..d020e7675 100644 --- a/lib/backupstore/BackupStoreException.txt +++ b/lib/backupstore/BackupStoreException.txt @@ -78,6 +78,7 @@ FileSystemNotInitialised 74 No BackupFileSystem has been configured for this acc ObjectDoesNotExist 75 The specified object ID does not exist in the store, or is not of this type AccountAlreadyExists 76 Tried to create an account that already exists AccountDoesNotExist 77 Tried to open an account that does not exist +FileUploadFailed 79 Failed to upload the file to the storage server CacheDirectoryLocked 80 The cache directory is already locked by another process BadConfiguration 82 The configuration file contains an error or invalid value TooManyFilesInDirectory 83 The S3 directory contains too many files for it to be a Boxbackup store, please check and/or empty it diff --git a/lib/backupstore/StoreTestUtils.cpp b/lib/backupstore/StoreTestUtils.cpp index f349158b2..8ddbe7c0c 100644 --- a/lib/backupstore/StoreTestUtils.cpp +++ b/lib/backupstore/StoreTestUtils.cpp @@ -77,6 +77,10 @@ void set_refcount(int64_t ObjectID, uint32_t RefCount) } else { + // Don't keep going back up the list, as if we found a + // zero-referenced file higher up, we'd end up deleting + // the refcounts of referenced files further down the + // list (higher IDs). break; } } diff --git a/lib/common/ByteCountingStream.h b/lib/common/ByteCountingStream.h new file mode 100644 index 000000000..b62c2e362 --- /dev/null +++ b/lib/common/ByteCountingStream.h @@ -0,0 +1,81 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ByteCountingStream.h +// Purpose: A stream wrapper that counts the number of bytes +// transferred through it. +// Created: 2016/02/05 +// +// -------------------------------------------------------------------------- + +#ifndef BYTECOUNTINGSTREAM__H +#define BYTECOUNTINGSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: ByteCountingStream +// Purpose: A stream wrapper that counts the number of bytes +// transferred through it. +// Created: 2016/02/05 +// +// -------------------------------------------------------------------------- +class ByteCountingStream : public IOStream +{ +public: + ByteCountingStream(IOStream &underlying) + : mrUnderlying(underlying), + mNumBytesRead(0), + mNumBytesWritten(0) + { } + + ByteCountingStream(const ByteCountingStream &rToCopy) + : mrUnderlying(rToCopy.mrUnderlying), + mNumBytesRead(0), + mNumBytesWritten(0) + { } + +private: + // no copying from IOStream allowed + ByteCountingStream(const IOStream& rToCopy); + +public: + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + int bytes_read = mrUnderlying.Read(pBuffer, NBytes, Timeout); + mNumBytesRead += bytes_read; + return bytes_read; + } + virtual pos_type BytesLeftToRead() + { + return mrUnderlying.BytesLeftToRead(); + } + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + mrUnderlying.Write(pBuffer, NBytes, Timeout); + mNumBytesWritten += NBytes; + } + using IOStream::Write; + + virtual bool StreamDataLeft() + { + return mrUnderlying.StreamDataLeft(); + } + virtual bool StreamClosed() + { + return mrUnderlying.StreamClosed(); + } + int64_t GetNumBytesRead() { return mNumBytesRead; } + int64_t GetNumBytesWritten() { return mNumBytesWritten; } + +private: + IOStream &mrUnderlying; + int64_t mNumBytesRead, mNumBytesWritten; +}; + +#endif // BYTECOUNTINGSTREAM__H + diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp index ed1578828..191b59e70 100644 --- a/test/backupstore/testbackupstore.cpp +++ b/test/backupstore/testbackupstore.cpp @@ -1940,9 +1940,9 @@ bool test_server_commands() } } - // Create some nice recursive directories TEST_THAT(check_reference_counts()); + // Create some nice recursive directories write_test_file(1); int64_t dirtodelete; @@ -2125,7 +2125,11 @@ bool test_directory_parent_entry_tracks_directory_size() protocol.Reopen(); protocolReadOnly.Reopen(); - // Now modify the root directory to remove its entry for this one + // Now modify the root directory to remove its entry for this one, and + // then add a directory inside this one. This should try to push the + // object size back up, which will try to modify the subdir's entry in + // its parent, which no longer exists, which should just return a + // protocol error instead of aborting/segfaulting. BackupStoreDirectory root(*get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID), IOStream::TimeOutInfinite); BackupStoreDirectory::Entry *en = root.FindEntryByID(subdirid); @@ -2133,34 +2137,18 @@ bool test_directory_parent_entry_tracks_directory_size() BackupStoreDirectory::Entry enCopy(*en); root.DeleteEntry(subdirid); TEST_THAT(write_dir(root)); - - // Add a directory, this should try to push the object size back up, - // which will try to modify the subdir's entry in its parent, which - // no longer exists, which should just log an error instead of - // aborting/segfaulting. - create_directory(protocol, subdirid); + TEST_CHECK_THROWS( + create_directory(protocol, subdirid), + ConnectionException, + Protocol_UnexpectedReply); + TEST_PROTOCOL_ERROR_OR(protocol, Err_DoesNotExistInDirectory,); // Repair the error ourselves, as bbstoreaccounts can't. protocol.QueryFinished(); enCopy.SetSizeInBlocks(get_raid_file(subdirid)->GetDiscUsageInBlocks()); root.AddEntry(enCopy); TEST_THAT(write_dir(root)); - - // We also have to remove the entry for lovely_directory created by - // create_directory(), because otherwise we can't create it again. - // (Perhaps it should not have been committed because we failed to - // update the parent, but currently it is.) - BackupStoreDirectory subdir(*get_raid_file(subdirid), - IOStream::TimeOutInfinite); - { - BackupStoreDirectory::Iterator i(subdir); - en = i.FindMatchingClearName( - BackupStoreFilenameClear("lovely_directory")); - } - TEST_THAT_OR(en, return false); protocol.Reopen(); - protocol.QueryDeleteDirectory(en->GetObjectID()); - set_refcount(en->GetObjectID(), 0); // This should have fixed the error, so we should be able to add the // entry now. This should push the object size back up.