diff --git a/lib/backupstore/BackupFileSystem.cpp b/lib/backupstore/BackupFileSystem.cpp index 02e00f40a..52cac76f2 100644 --- a/lib/backupstore/BackupFileSystem.cpp +++ b/lib/backupstore/BackupFileSystem.cpp @@ -1273,6 +1273,89 @@ void S3BackupFileSystem::GetCacheLock() BackupStoreRefCountDatabase& S3BackupFileSystem::GetPermanentRefCountDatabase( bool ReadOnly) { + if(mapPermanentRefCountDatabase.get()) + { + return *mapPermanentRefCountDatabase; + } + + // It's dangerous to have two read-write databases open at the same time (it would + // be too easy to update the refcounts in the wrong one by mistake), and temporary + // databases are always read-write, so if a temporary database is already open + // then we should only allow a read-only permanent database to be opened. + ASSERT(!mapPotentialRefCountDatabase.get() || ReadOnly); + + GetCacheLock(); + + // If we have a cached copy, check that it's up to date. + std::string local_path = GetRefCountDatabaseCachePath(); + std::string digest; + + if(FileExists(local_path)) + { + FileStream fs(local_path); + MD5DigestStream digester; + fs.CopyStreamTo(digester); + digester.Close(); + digest = digester.DigestAsString(); + } + + // Try to fetch it from the remote server. If we pass a digest, and if it matches, + // then the server won't send us the same file data again. + std::string uri = GetMetadataURI(S3_REFCOUNT_FILE_NAME); + HTTPResponse response = mrClient.GetObject(uri, digest); + if(response.GetResponseCode() == HTTPResponse::Code_OK) + { + if(digest.size() > 0) + { + BOX_WARNING("We had a cached copy of the refcount DB, but it " + "didn't match the one in S3, so the server sent us a new " + "copy and the cache will be updated"); + } + + FileStream fs(local_path, O_CREAT | O_RDWR); + response.CopyStreamTo(fs); + + // Check that we got the full and correct file. TODO: calculate the MD5 + // digest while writing the file, instead of afterwards. + fs.Seek(0, IOStream::SeekType_Absolute); + MD5DigestStream digester; + fs.CopyStreamTo(digester); + digester.Close(); + digest = digester.DigestAsString(); + + if(response.GetHeaders().GetHeaderValue("etag") != + "\"" + digest + "\"") + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + FileDownloadedIncorrectly, "Downloaded invalid file from " + "S3: expected MD5 digest to be " << + response.GetHeaders().GetHeaderValue("etag") << " but " + "it was actually " << digest); + } + } + else if(response.GetResponseCode() == HTTPResponse::Code_NotFound) + { + // Do not create a new refcount DB here, it is not safe! Users may wonder + // why they have lost all their files, and/or unwittingly overwrite their + // backup data. + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CorruptReferenceCountDatabase, "Reference count database is " + "missing, cannot safely open account. Please run bbstoreaccounts " + "check on the account to recreate it. 404 Not Found: " << uri); + } + else if(response.GetResponseCode() == HTTPResponse::Code_NotModified) + { + // The copy on the server is the same as the one in our cache, so we don't + // need to download it again, nothing to do! + } + else + { + mrClient.CheckResponse(response, "Failed to download reference " + "count database"); + } + + mapPermanentRefCountDatabase = BackupStoreRefCountDatabase::Load(local_path, + S3_FAKE_ACCOUNT_ID, ReadOnly); return *mapPermanentRefCountDatabase; } @@ -1605,6 +1688,25 @@ std::auto_ptr S3BackupFileSystem::GetFile(int64_t ObjectID) } +S3BackupFileSystem::~S3BackupFileSystem() +{ + // Close any open refcount DBs before partially destroying the + // BackupFileSystem that they need to close down. Need to do this in + // the subclass to avoid calling SaveRefCountDatabase() when the + // subclass has already been partially destroyed. + // http://stackoverflow.com/questions/10707286/how-to-resolve-pure-virtual-method-called + mapPotentialRefCountDatabase.reset(); + mapPermanentRefCountDatabase.reset(); + + // This needs to be in the source file, not inline, as long as we don't include + // the whole of SimpleDBClient.h in BackupFileSystem.h. + if(mHaveLock) + { + ReleaseLock(); + } +} + + // TODO FIXME check if file is in cache and return size from there int64_t S3BackupFileSystem::GetFileSizeInBlocks(int64_t ObjectID) { diff --git a/lib/backupstore/BackupFileSystem.h b/lib/backupstore/BackupFileSystem.h index aea1081ae..61c25efce 100644 --- a/lib/backupstore/BackupFileSystem.h +++ b/lib/backupstore/BackupFileSystem.h @@ -317,17 +317,7 @@ class S3BackupFileSystem : public BackupFileSystem public: S3BackupFileSystem(const Configuration& config, const std::string& BasePath, const std::string& CacheDirectory, S3Client& rClient); - virtual ~S3BackupFileSystem() - { - // Call ReleaseLock() to close any open refcount DBs before - // partially destroying the BackupFileSystem that they need to - // close down. Need to do this in the subclass to avoid calling - // SaveRefCountDatabase() (from - // ~BackupStoreRefCountDatabaseWrapper) when the subclass has - // already been partially destroyed. - // http://stackoverflow.com/questions/10707286/how-to-resolve-pure-virtual-method-called - ReleaseLock(); - } + virtual ~S3BackupFileSystem(); virtual void TryGetLock() { } virtual bool HaveLock() { return false; } diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt index d020e7675..811bf0f04 100644 --- a/lib/backupstore/BackupStoreException.txt +++ b/lib/backupstore/BackupStoreException.txt @@ -78,6 +78,7 @@ FileSystemNotInitialised 74 No BackupFileSystem has been configured for this acc ObjectDoesNotExist 75 The specified object ID does not exist in the store, or is not of this type AccountAlreadyExists 76 Tried to create an account that already exists AccountDoesNotExist 77 Tried to open an account that does not exist +FileDownloadedIncorrectly 78 The downloaded file had different data than expected, so was downloaded badly FileUploadFailed 79 Failed to upload the file to the storage server CacheDirectoryLocked 80 The cache directory is already locked by another process BadConfiguration 82 The configuration file contains an error or invalid value