Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Move S3BackupFileSystem class into its own file.
Create BackupFileSystem, an abstract interface for reading and writing files and directories, abstracting over RaidFile, S3, FTP etc. as a parent class for S3BackupFileSystem.
- Loading branch information
Showing
6 changed files
with
367 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,197 @@ | ||
| // -------------------------------------------------------------------------- | ||
| // | ||
| // File | ||
| // Name: BackupFileSystem.h | ||
| // Purpose: Generic interface for reading and writing files and | ||
| // directories, abstracting over RaidFile, S3, FTP etc. | ||
| // Created: 2015/08/03 | ||
| // | ||
| // -------------------------------------------------------------------------- | ||
|
|
||
| #include "Box.h" | ||
|
|
||
| #include <sys/types.h> | ||
|
|
||
| #include "autogen_BackupStoreException.h" | ||
| #include "BackupConstants.h" | ||
| #include "BackupStoreDirectory.h" | ||
| #include "BackupFileSystem.h" | ||
| #include "BackupStoreAccountDatabase.h" | ||
| #include "BackupStoreFile.h" | ||
| #include "BackupStoreInfo.h" | ||
| #include "BackupStoreRefCountDatabase.h" | ||
| #include "BufferedStream.h" | ||
| #include "BufferedWriteStream.h" | ||
| #include "CollectInBufferStream.h" | ||
| #include "Configuration.h" | ||
| #include "BackupStoreObjectMagic.h" | ||
| #include "HTTPResponse.h" | ||
| #include "IOStream.h" | ||
| #include "InvisibleTempFileStream.h" | ||
| #include "S3Client.h" | ||
|
|
||
| #include "MemLeakFindOn.h" | ||
|
|
||
| int S3BackupFileSystem::GetBlockSize() | ||
| { | ||
| return S3_NOTIONAL_BLOCK_SIZE; | ||
| } | ||
|
|
||
| std::string S3BackupFileSystem::GetObjectURL(const std::string& ObjectPath) const | ||
| { | ||
| const Configuration s3config = mrConfig.GetSubConfiguration("S3Store"); | ||
| return std::string("http://") + s3config.GetKeyValue("HostName") + ":" + | ||
| s3config.GetKeyValue("Port") + GetObjectURI(ObjectPath); | ||
| } | ||
|
|
||
| int64_t S3BackupFileSystem::GetRevisionID(const std::string& uri, | ||
| HTTPResponse& response) const | ||
| { | ||
| std::string etag; | ||
|
|
||
| if(!response.GetHeader("etag", &etag)) | ||
| { | ||
| THROW_EXCEPTION_MESSAGE(BackupStoreException, MissingEtagHeader, | ||
| "Failed to get the MD5 checksum of the file or directory " | ||
| "at this URL: " << GetObjectURL(uri)); | ||
| } | ||
|
|
||
| if(etag[0] != '"') | ||
| { | ||
| THROW_EXCEPTION_MESSAGE(BackupStoreException, InvalidEtagHeader, | ||
| "Failed to get the MD5 checksum of the file or directory " | ||
| "at this URL: " << GetObjectURL(uri)); | ||
| } | ||
|
|
||
| char* pEnd = NULL; | ||
| int64_t revID = strtoull(etag.substr(1, 17).c_str(), &pEnd, 16); | ||
| if(*pEnd != '\0') | ||
| { | ||
| THROW_SYS_ERROR("Failed to get the MD5 checksum of the file or " | ||
| "directory at this URL: " << uri << ": invalid character '" << | ||
| *pEnd << "' in '" << etag << "'", BackupStoreException, | ||
| InvalidEtagHeader); | ||
| } | ||
|
|
||
| return revID; | ||
| } | ||
|
|
||
| std::auto_ptr<BackupStoreInfo> S3BackupFileSystem::GetBackupStoreInfo(int32_t AccountID, | ||
| bool ReadOnly) | ||
| { | ||
| std::string info_url = GetObjectURL(S3_INFO_FILE_NAME); | ||
| HTTPResponse response = GetObject(S3_INFO_FILE_NAME); | ||
| mrClient.CheckResponse(response, std::string("No BackupStoreInfo file exists " | ||
| "at this URL: ") + info_url); | ||
|
|
||
| std::auto_ptr<BackupStoreInfo> info = | ||
| BackupStoreInfo::Load(response, info_url, ReadOnly); | ||
|
|
||
| // Check it | ||
| if(info->GetAccountID() != AccountID) | ||
| { | ||
| THROW_FILE_ERROR("Found wrong account ID in store info", | ||
| info_url, BackupStoreException, BadStoreInfoOnLoad); | ||
| } | ||
|
|
||
| return info; | ||
| } | ||
|
|
||
| void S3BackupFileSystem::PutBackupStoreInfo(BackupStoreInfo& rInfo) | ||
| { | ||
| CollectInBufferStream out; | ||
| rInfo.Save(out); | ||
| out.SetForReading(); | ||
|
|
||
| HTTPResponse response = PutObject(S3_INFO_FILE_NAME, out); | ||
|
|
||
| std::string info_url = GetObjectURL(S3_INFO_FILE_NAME); | ||
| mrClient.CheckResponse(response, std::string("Failed to upload the new " | ||
| "BackupStoreInfo file to this URL: ") + info_url); | ||
| } | ||
|
|
||
| //! Returns whether an object (a file or directory) exists with this object ID, and its | ||
| //! revision ID, which for a RaidFile is based on its timestamp and file size. | ||
| bool S3BackupFileSystem::ObjectExists(int64_t ObjectID, int64_t *pRevisionID) | ||
| { | ||
| std::string uri = GetDirectoryURI(ObjectID); | ||
| HTTPResponse response = mrClient.HeadObject(uri); | ||
|
|
||
| if(response.GetResponseCode() == HTTPResponse::Code_NotFound) | ||
| { | ||
| // A file might exist, check that too. | ||
| uri = GetFileURI(ObjectID); | ||
| response = mrClient.HeadObject(uri); | ||
| } | ||
|
|
||
| if(response.GetResponseCode() == HTTPResponse::Code_NotFound) | ||
| { | ||
| return false; | ||
| } | ||
|
|
||
| if(response.GetResponseCode() != HTTPResponse::Code_OK) | ||
| { | ||
| // Throw an appropriate exception. | ||
| mrClient.CheckResponse(response, std::string("Failed to check whether " | ||
| "a file or directory exists at this URL: ") + | ||
| GetObjectURL(uri)); | ||
| } | ||
|
|
||
| if(pRevisionID) | ||
| { | ||
| *pRevisionID = GetRevisionID(uri, response); | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| // This URI should NOT include the BasePath, because the S3Client will add that | ||
| // automatically when we make a request for this URI. | ||
| std::string S3BackupFileSystem::GetDirectoryURI(int64_t ObjectID) | ||
| { | ||
| std::ostringstream out; | ||
| out << "dirs/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir"; | ||
| return out.str(); | ||
| } | ||
|
|
||
| // This URI should NOT include the BasePath, because the S3Client will add that | ||
| // automatically when we make a request for this URI. | ||
| std::string S3BackupFileSystem::GetFileURI(int64_t ObjectID) | ||
| { | ||
| std::ostringstream out; | ||
| out << "files/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir"; | ||
| return out.str(); | ||
| } | ||
|
|
||
| //! Reads a directory with the specified ID into the supplied BackupStoreDirectory | ||
| //! object, also initialising its revision ID and SizeInBlocks. | ||
| void S3BackupFileSystem::GetDirectory(int64_t ObjectID, BackupStoreDirectory& rDirOut) | ||
| { | ||
| std::string uri = GetDirectoryURI(ObjectID); | ||
| HTTPResponse response = GetObject(uri); | ||
| mrClient.CheckResponse(response, | ||
| std::string("Failed to download directory: ") + uri); | ||
| rDirOut.ReadFromStream(response, mrClient.GetNetworkTimeout()); | ||
|
|
||
| rDirOut.SetRevisionID(GetRevisionID(uri, response)); | ||
| ASSERT(false); // set the size in blocks | ||
| rDirOut.SetUserInfo1_SizeInBlocks(GetSizeInBlocks(response.GetContentLength())); | ||
| } | ||
|
|
||
| //! Writes the supplied BackupStoreDirectory object to the store, and updates its revision | ||
| //! ID and SizeInBlocks. | ||
| void S3BackupFileSystem::PutDirectory(BackupStoreDirectory& rDir) | ||
| { | ||
| CollectInBufferStream out; | ||
| rDir.WriteToStream(out); | ||
| out.SetForReading(); | ||
|
|
||
| std::string uri = GetDirectoryURI(rDir.GetObjectID()); | ||
| HTTPResponse response = PutObject(uri, out); | ||
| mrClient.CheckResponse(response, | ||
| std::string("Failed to upload directory: ") + uri); | ||
|
|
||
| rDir.SetRevisionID(GetRevisionID(uri, response)); | ||
| rDir.SetUserInfo1_SizeInBlocks(GetSizeInBlocks(out.GetSize())); | ||
| } | ||
|
|
Oops, something went wrong.