Skip to content

Commit

Permalink
Refactor BackupStoreContext to use BackupFileSystem
Browse files Browse the repository at this point in the history
  • Loading branch information
@qris
qris committed Oct 7, 2017
1 parent 1556500 commit 9474ba0
Show file tree
Hide file tree
Showing 10 changed files with 627 additions and 600 deletions.
128 changes: 10 additions & 118 deletions lib/backupstore/BackupCommands.cpp
View file
Expand Up @@ -71,7 +71,9 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolReplyable::HandleException(Bo
}
else if (e.GetType() == BackupStoreException::ExceptionType)
{
if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify)
// Slightly broken or really broken, both thrown by VerifyStream:
if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify ||
e.GetSubType() == BackupStoreException::BadBackupStoreFile)
{
return PROTOCOL_ERROR(Err_FileDoesNotVerify);
}
Expand Down Expand Up @@ -348,132 +350,22 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObject::DoCommand(BackupPr
std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const
{
CHECK_PHASE(Phase_Commands)

// Check the objects exist
if(!rContext.ObjectExists(mObjectID)
|| !rContext.ObjectExists(mInDirectory))
{
return PROTOCOL_ERROR(Err_DoesNotExist);
}

// Get the directory it's in
const BackupStoreDirectory &rdir(rContext.GetDirectory(mInDirectory));

// Find the object within the directory
BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(mObjectID);
if(pfileEntry == 0)
{
return PROTOCOL_ERROR(Err_DoesNotExistInDirectory);
}

// The result
std::auto_ptr<IOStream> stream;

// Does this depend on anything?
if(pfileEntry->GetDependsNewer() != 0)
try
{
// File exists, but is a patch from a new version. Generate the older version.
std::vector<int64_t> patchChain;
int64_t id = mObjectID;
BackupStoreDirectory::Entry *en = 0;
do
{
patchChain.push_back(id);
en = rdir.FindEntryByID(id);
if(en == 0)
{
BOX_ERROR("Object " <<
BOX_FORMAT_OBJECTID(mObjectID) <<
" in dir " <<
BOX_FORMAT_OBJECTID(mInDirectory) <<
" for account " <<
BOX_FORMAT_ACCOUNT(rContext.GetClientID()) <<
" references object " <<
BOX_FORMAT_OBJECTID(id) <<
" which does not exist in dir");
return PROTOCOL_ERROR(Err_PatchConsistencyError);
}
id = en->GetDependsNewer();
}
while(en != 0 && id != 0);

// OK! The last entry in the chain is the full file, the others are patches back from it.
// Open the last one, which is the current from file
std::auto_ptr<IOStream> from(rContext.OpenObject(patchChain[patchChain.size() - 1]));

// Then, for each patch in the chain, do a combine
for(int p = ((int)patchChain.size()) - 2; p >= 0; --p)
{
// ID of patch
int64_t patchID = patchChain[p];

// Open it a couple of times
std::auto_ptr<IOStream> diff(rContext.OpenObject(patchID));
std::auto_ptr<IOStream> diff2(rContext.OpenObject(patchID));

// Choose a temporary filename for the result of the combination
std::ostringstream fs;
fs << rContext.GetAccountRoot() << ".recombinetemp." << p;
std::string tempFn =
RaidFileController::DiscSetPathToFileSystemPath(
rContext.GetStoreDiscSet(), fs.str(),
p + 16);

// Open the temporary file
std::auto_ptr<IOStream> combined(
new InvisibleTempFileStream(
tempFn, O_RDWR | O_CREAT | O_EXCL |
O_BINARY | O_TRUNC));

// Do the combining
BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined);

// Move to the beginning of the combined file
combined->Seek(0, IOStream::SeekType_Absolute);

// Then shuffle round for the next go
if (from.get()) from->Close();
from = combined;
}

// Now, from contains a nice file to send to the client. Reorder it
{
// Write nastily to allow this to work with gcc 2.x
std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */));
stream = t;
}

// Release from file to avoid double deletion
from.release();
stream = rContext.GetFile(mObjectID, mInDirectory);
}
else
catch(BackupStoreException &e)
{
// Simple case: file already exists on disc ready to go

// Open the object
std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID));
BufferedStream buf(*object);

// Verify it
if(!BackupStoreFile::VerifyEncodedFileFormat(buf))
if(EXCEPTION_IS_TYPE(e, BackupStoreException, ObjectDoesNotExist))
{
return PROTOCOL_ERROR(Err_FileDoesNotVerify);
return PROTOCOL_ERROR(Err_DoesNotExist);
}

// Reset stream -- seek to beginning
object->Seek(0, IOStream::SeekType_Absolute);

// Reorder the stream/file into stream order
else
{
// Write nastily to allow this to work with gcc 2.x
std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */));
stream = t;
throw;
}

// Object will be deleted when the stream is deleted,
// so can release the object auto_ptr here to avoid
// premature deletion
object.release();
}

// Stream the reordered stream to the peer
Expand Down
92 changes: 86 additions & 6 deletions lib/backupstore/BackupFileSystem.cpp
View file
Expand Up @@ -24,6 +24,7 @@
#include "BackupStoreRefCountDatabase.h"
#include "BufferedStream.h"
#include "BufferedWriteStream.h"
#include "ByteCountingStream.h"
#include "CollectInBufferStream.h"
#include "Configuration.h"
#include "BackupStoreObjectMagic.h"
Expand Down Expand Up @@ -510,8 +511,11 @@ RaidBackupFileSystem::PutFileComplete(int64_t ObjectID, IOStream& rFileData,
// We can only do this when the file (ObjectID) doesn't already exist.
ASSERT(refcount == 0);

// But RaidFileWrite won't allow us to write to a file that doesn't have exactly one
// reference, so we pretend for now that there is one. If the file already exists,
// then Open(false) below will raise an exception for us.
RaidPutFileCompleteTransaction* pTrans = new RaidPutFileCompleteTransaction(
mStoreDiscSet, filename, refcount);
mStoreDiscSet, filename, 1);
std::auto_ptr<BackupFileSystem::Transaction> apTrans(pTrans);

RaidFileWrite& rStoreFile(pTrans->GetRaidFile());
Expand Down Expand Up @@ -547,9 +551,14 @@ class RaidPutFilePatchTransaction : public BackupFileSystem::Transaction
public:
RaidPutFilePatchTransaction(int StoreDiscSet,
const std::string& newCompleteFilename,
const std::string& reversedPatchFilename)
: mNewCompleteFile(StoreDiscSet, newCompleteFilename),
mReversedPatchFile(StoreDiscSet, reversedPatchFilename),
const std::string& reversedPatchFilename,
BackupStoreRefCountDatabase::refcount_t refcount)
// It's not quite true that mNewCompleteFile has 1 reference: it doesn't exist
// yet, so it has 0 right now. However when the transaction is committed it will
// have 1, and RaidFileWrite gets upset if we try to modify a file with != 1
// references, so we need to pretend now that we already have the reference.
: mNewCompleteFile(StoreDiscSet, newCompleteFilename, 1),
mReversedPatchFile(StoreDiscSet, reversedPatchFilename, refcount),
mReversedDiffIsCompletelyDifferent(false),
mBlocksUsedByNewFile(0),
mChangeInBlocksUsedByOldFile(0)
Expand Down Expand Up @@ -593,7 +602,7 @@ void RaidPutFilePatchTransaction::Commit()

std::auto_ptr<BackupFileSystem::Transaction>
RaidBackupFileSystem::PutFilePatch(int64_t ObjectID, int64_t DiffFromFileID,
IOStream& rPatchData)
IOStream& rPatchData, BackupStoreRefCountDatabase::refcount_t refcount)
{
// Create the containing directory if it doesn't exist.
std::string newVersionFilename = GetObjectFileName(ObjectID, true);
Expand All @@ -603,7 +612,7 @@ RaidBackupFileSystem::PutFilePatch(int64_t ObjectID, int64_t DiffFromFileID,
false); // no need to make sure the directory it's in exists

RaidPutFilePatchTransaction* pTrans = new RaidPutFilePatchTransaction(
mStoreDiscSet, newVersionFilename, oldVersionFilename);
mStoreDiscSet, newVersionFilename, oldVersionFilename, refcount);
std::auto_ptr<BackupFileSystem::Transaction> apTrans(pTrans);

RaidFileWrite& rNewCompleteFile(pTrans->GetNewCompleteFile());
Expand Down Expand Up @@ -1470,6 +1479,77 @@ void S3BackupFileSystem::DeleteObjectUnknown(int64_t ObjectID)
}



class S3PutFileCompleteTransaction : public BackupFileSystem::Transaction
{
private:
S3Client& mrClient;
std::string mFileURI;
bool mCommitted;
int64_t mNumBlocks;

public:
S3PutFileCompleteTransaction(S3BackupFileSystem& fs, S3Client& client,
const std::string& file_uri, IOStream& file_data);
~S3PutFileCompleteTransaction();
virtual void Commit();
virtual int64_t GetNumBlocks() { return mNumBlocks; }

// It doesn't matter what we return here, because this should never be called
// for a PutFileCompleteTransaction (the API is intended for
// PutFilePatchTransaction instead):
virtual bool IsNewFileIndependent() { return false; }
};


S3PutFileCompleteTransaction::S3PutFileCompleteTransaction(S3BackupFileSystem& fs,
S3Client& client, const std::string& file_uri, IOStream& file_data)
: mrClient(client),
mFileURI(file_uri),
mCommitted(false),
mNumBlocks(0)
{
ByteCountingStream counter(file_data);
HTTPResponse response = mrClient.PutObject(file_uri, counter);
if(response.GetResponseCode() != HTTPResponse::Code_OK)
{
THROW_EXCEPTION_MESSAGE(BackupStoreException, FileUploadFailed,
"Failed to upload file to Amazon S3: " <<
response.ResponseCodeString());
}
mNumBlocks = fs.GetSizeInBlocks(counter.GetNumBytesRead());
}


void S3PutFileCompleteTransaction::Commit()
{
mCommitted = true;
}


S3PutFileCompleteTransaction::~S3PutFileCompleteTransaction()
{
if(!mCommitted)
{
HTTPResponse response = mrClient.DeleteObject(mFileURI);
mrClient.CheckResponse(response, "Failed to delete uploaded file from Amazon S3",
true); // ExpectNoContent
}
}


std::auto_ptr<BackupFileSystem::Transaction>
S3BackupFileSystem::PutFileComplete(int64_t ObjectID, IOStream& rFileData,
BackupStoreRefCountDatabase::refcount_t refcount)
{
ASSERT(refcount == 0 || refcount == 1);
BackupStoreFile::VerifyStream validator(rFileData);
S3PutFileCompleteTransaction* pTrans = new S3PutFileCompleteTransaction(*this,
mrClient, GetFileURI(ObjectID), validator);
return std::auto_ptr<BackupFileSystem::Transaction>(pTrans);
}


//! GetObject() can be for either a file or a directory, so we need to try both.
// TODO FIXME use a cached directory listing to determine which it is
std::auto_ptr<IOStream> S3BackupFileSystem::GetObject(int64_t ObjectID, bool required)
Expand Down
28 changes: 20 additions & 8 deletions lib/backupstore/BackupFileSystem.h
View file
Expand Up @@ -108,7 +108,8 @@ class BackupFileSystem
virtual std::auto_ptr<Transaction> PutFileComplete(int64_t ObjectID,
IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount) = 0;
virtual std::auto_ptr<Transaction> PutFilePatch(int64_t ObjectID,
int64_t DiffFromFileID, IOStream& rPatchData) = 0;
int64_t DiffFromFileID, IOStream& rPatchData,
BackupStoreRefCountDatabase::refcount_t refcount) = 0;
// GetObject() will retrieve either a file or directory, whichever exists.
// GetFile() and GetDirectory() are only guaranteed to work on objects of the
// correct type, but may be faster (depending on the implementation).
Expand Down Expand Up @@ -145,6 +146,18 @@ class BackupFileSystem
virtual CheckObjectsResult CheckObjects(bool fix_errors) = 0;
virtual void EnsureObjectIsPermanent(int64_t ObjectID, bool fix_errors) = 0;

// CloseRefCountDatabase() closes the active database, saving changes to permanent
// storage if necessary. It invalidates any references to the current database!
// It is needed to allow a BackupStoreContext to be Finished, changes made to the
// BackupFileSystem's BackupStoreInfo by BackupStoreCheck and HousekeepStoreAccount,
// and the Context to be reopened.
void CloseRefCountDatabase(BackupStoreRefCountDatabase* p_refcount_db)
{
ASSERT(p_refcount_db == mapPermanentRefCountDatabase.get());
SaveRefCountDatabase(*p_refcount_db);
mapPermanentRefCountDatabase.reset();
}

protected:
virtual std::auto_ptr<BackupStoreInfo> GetBackupStoreInfoInternal(bool ReadOnly) = 0;
std::auto_ptr<BackupStoreInfo> mapBackupStoreInfo;
Expand Down Expand Up @@ -228,7 +241,8 @@ class RaidBackupFileSystem : public BackupFileSystem
virtual std::auto_ptr<Transaction> PutFileComplete(int64_t ObjectID,
IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount);
virtual std::auto_ptr<Transaction> PutFilePatch(int64_t ObjectID,
int64_t DiffFromFileID, IOStream& rPatchData);
int64_t DiffFromFileID, IOStream& rPatchData,
BackupStoreRefCountDatabase::refcount_t refcount);
virtual std::auto_ptr<IOStream> GetFile(int64_t ObjectID)
{
// For RaidBackupFileSystem, GetObject() is equivalent to GetFile().
Expand Down Expand Up @@ -326,14 +340,12 @@ class S3BackupFileSystem : public BackupFileSystem
virtual void GetDirectory(int64_t ObjectID, BackupStoreDirectory& rDirOut);
virtual void PutDirectory(BackupStoreDirectory& rDir);
virtual std::auto_ptr<Transaction> PutFileComplete(int64_t ObjectID,
IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount)
{
return std::auto_ptr<Transaction>();
}
IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount);
virtual std::auto_ptr<Transaction> PutFilePatch(int64_t ObjectID,
int64_t DiffFromFileID, IOStream& rPatchData)
int64_t DiffFromFileID, IOStream& rPatchData,
BackupStoreRefCountDatabase::refcount_t refcount)
{
return std::auto_ptr<Transaction>();
return PutFileComplete(ObjectID, rPatchData, refcount);
}
virtual std::auto_ptr<IOStream> GetFile(int64_t ObjectID);
virtual std::auto_ptr<IOStream> GetFilePatch(int64_t ObjectID,
Expand Down

0 comments on commit 9474ba0

Please sign in to comment.