test/bbackupd: add specialised versions of some tests for S3

Not all tests are specialised because many are quite slow, and would behave in
the same way, not expected to exercise any additional S3-related code paths
that are not covered by the tests that have been implemented.

.gitignore

@@ -34,6 +34,7 @@ test/backupstorefix/testfiles/testbackupstorefix.pl
 test/bbackupd/testfiles/bbackupd.conf
 test/bbackupd/testfiles/bbackupd.s3.conf
 test/bbackupd/testfiles/bbackupd-snapshot.conf
+test/bbackupd/testfiles/bbackupd-snapshot.s3.conf
 test/bbackupd/testfiles/bbackupd-symlink.conf
 test/bbackupd/testfiles/bbackupd-temploc.conf
 test/bbackupd/testfiles/extcheck1.pl

configure.ac

@@ -91,6 +91,7 @@ AC_CONFIG_FILES([infrastructure/BoxPlatform.pm
                  test/bbackupd/testfiles/bbackupd.conf
                  test/bbackupd/testfiles/bbackupd.s3.conf
                  test/bbackupd/testfiles/bbackupd-snapshot.conf
+                 test/bbackupd/testfiles/bbackupd-snapshot.s3.conf
                  test/bbackupd/testfiles/bbackupd-symlink.conf
                  test/bbackupd/testfiles/bbackupd-temploc.conf
   		 ])

infrastructure/cmake/CMakeLists.txt

@@ -43,6 +43,7 @@ set(files_to_configure
 	test/bbackupd/testfiles/bbackupd.conf
 	test/bbackupd/testfiles/bbackupd.s3.conf
 	test/bbackupd/testfiles/bbackupd-snapshot.conf
+	test/bbackupd/testfiles/bbackupd-snapshot.s3.conf
 	test/bbackupd/testfiles/bbackupd-symlink.conf
 	test/bbackupd/testfiles/bbackupd-temploc.conf
 	test/bbackupd/testfiles/extcheck1.pl

lib/bbackupd/BackupDaemon.cpp

@@ -2514,12 +2514,12 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con
 			}
 		}
 
-		const Configuration& rConfig(
-			rLocationsConf.GetSubConfiguration(*pLocName));
+		const Configuration& rConfig(rLocationsConf.GetSubConfiguration(*pLocName));
 		std::auto_ptr<Location> apLoc;
 
 		BackupStoreFilenameClear dirname(*pLocName); // generate the filename
-		int64_t existing_dir_id = 0;
+		bool local_dir_exists = true;
+		int64_t existing_remote_dir_id = 0;
 
 		try
 		{
@@ -2548,11 +2548,11 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con
 			BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname);
 			if(en != NULL)
 			{
-				existing_dir_id = en->GetObjectID();
+				existing_remote_dir_id = en->GetObjectID();
 
 				// Delete the entry from the directory, so we get a list of
 				// unused root directories at the end of this.
-				dir.DeleteEntry(existing_dir_id);
+				dir.DeleteEntry(existing_remote_dir_id);
 			}
 
 			// Do a fsstat on the pathname to find out which mount it's on
@@ -2569,10 +2569,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con
 				if(::statfs(pLoc->mPath.c_str(), &s) != 0)
 # endif // HAVE_STRUCT_STATVFS_F_MNTONNAME
 				{
-					THROW_SYS_ERROR("Failed to stat path "
-						"'" << pLoc->mPath << "' "
-						"for location "
-						"'" << pLoc->mName << "'",
+					THROW_SYS_FILE_ERROR(pLoc->mPath, "Failed to stat local path",
 						CommonException, OSFileError);
 				}
 
@@ -2634,23 +2631,30 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con
 		}
 		catch (std::exception &e)
 		{
-			BOX_ERROR("Failed to configure location '"
-				<< pLoc->mName << "' path '"
-				<< pLoc->mPath << "': " << e.what() <<
-				": please check for previous errors");
-			mReadErrorsOnFilesystemObjects = true;
+			BOX_ERROR("Failed to configure location '" << pLoc->mName << "': " <<
+				e.what());
+			local_dir_exists = false;
 		}
 		catch(...)
 		{
-			BOX_ERROR("Failed to configure location '"
-				<< pLoc->mName << "' path '"
-				<< pLoc->mPath << "': please check for "
-				"previous errors");
-			mReadErrorsOnFilesystemObjects = true;
+			BOX_ERROR("Failed to configure location '" << pLoc->mName << "': please "
+				"check for previous errors");
+			local_dir_exists = false;
 		}
 
+		// Remove it from the temporary list to avoid deletion, even if it doesn't actually
+		// exist locally at this time:
+		tmpLocations.remove(pLoc);
+
 		// Does this exist on the server?
-		if(existing_dir_id == 0)
+		if(!local_dir_exists)
+		{
+			mReadErrorsOnFilesystemObjects = true;
+			// Don't try to create it remotely, just skip it.
+			continue;
+		}
+
+		if(existing_remote_dir_id == 0)
 		{
 			// Doesn't exist, so it has to be created on the server. Let's go!
 			// First, get the directory's attributes and modification time
@@ -2665,47 +2669,32 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con
 			}
 			catch (BoxException &e)
 			{
-				BOX_ERROR("Failed to get attributes "
-					"for path '" << pLoc->mPath
-					<< "', skipping location '" <<
-					pLoc->mName << "'");
-				throw;
+				BOX_ERROR("Failed to get attributes for path '" << pLoc->mPath <<
+					"', skipping location '" << pLoc->mName << "'");
+				continue;
 			}
 
-			// Execute create directory command
-			try
-			{
-				std::auto_ptr<IOStream> attrStream(
-					new MemBlockStream(attr));
-				std::auto_ptr<BackupProtocolSuccess>
-					dirCreate(connection.QueryCreateDirectory(
+			// Create the remote directory for this location. If this fails, then we
+			// abort the whole backup, otherwise the error isn't reported sufficiently
+			// severely for test_bbackupd_responds_to_connection_failure_in_process_s3.
+			std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr));
+			std::auto_ptr<BackupProtocolSuccess> dirCreate(
+				connection.QueryCreateDirectory(
 					BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory
 					attrModTime, dirname, attrStream));
 
-				// Object ID for later creation
-				existing_dir_id = dirCreate->GetObjectID();
-			}
-			catch (BoxException &e)
-			{
-				BOX_ERROR("Failed to create remote "
-					"directory '/" << pLoc->mName <<
-					"', skipping location '" <<
-					pLoc->mName << "'");
-				throw;
-			}
+			// Object ID for later creation
+			existing_remote_dir_id = dirCreate->GetObjectID();
 		}
 
 		// Create and store the directory object for the root of this location
-		ASSERT(existing_dir_id != 0);
+		ASSERT(existing_remote_dir_id != 0);
 		if(pLoc->mapDirectoryRecord.get() == NULL)
 		{
 			pLoc->mapDirectoryRecord.reset(
-				new BackupClientDirectoryRecord(existing_dir_id, *pLocName));
+				new BackupClientDirectoryRecord(existing_remote_dir_id, *pLocName));
 		}
 
-		// Remove it from the temporary list to avoid deletion
-		tmpLocations.remove(pLoc);
-
 		// Push it back on the vector of locations
 		mLocations.push_back(pLoc);
 
@@ -2721,8 +2710,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con
 		i  = tmpLocations.begin();
 		i != tmpLocations.end(); i++)
 	{
-		BOX_INFO("Removing obsolete location from memory: " <<
-			(*i)->mName);
+		BOX_INFO("Removing obsolete location from memory: " << (*i)->mName);
 		delete *i;
 	}
 
@@ -2752,8 +2740,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con
 				SecondsToBoxTime(mDeleteRedundantLocationsAfter);
 		}
 
-		int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter
-			- now);
+		int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter - now);
 
 		BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations "
 			"in root directory found, will delete from store "
@@ -2769,8 +2756,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con
 			BackupStoreFilenameClear clear(en->GetName());
 			const std::string &name(clear.GetClearFilename());
 			mUnusedRootDirEntries.push_back(
-				std::pair<int64_t,std::string>
-				(en->GetObjectID(), name));
+				std::pair<int64_t,std::string>(en->GetObjectID(), name));
 			// Log this
 			BOX_INFO("Unused location in root: " << name);
 		}

lib/common/Test.cpp

@@ -509,21 +509,16 @@ void terminate_bbackupd(int pid)
 
 
 // Wait a given number of seconds for something to complete
-void wait_for_operation(int seconds, const char* message)
+void wait_for_operation(float seconds, const char* message)
 {
 	BOX_INFO("Waiting " << seconds << " seconds for " << message);
-
-	for(int l = 0; l < seconds; ++l)
-	{
-		sleep(1);
-	}
-
+	ShortSleep(MilliSecondsToBoxTime(seconds * 1000), true); // logDuration
 	BOX_TRACE("Finished waiting for " << message);
 }
 
 void safe_sleep(int seconds)
 {
-	ShortSleep(SecondsToBoxTime(seconds), true);
+	ShortSleep(SecondsToBoxTime(seconds), true); // logDuration
 }
 
 std::auto_ptr<Configuration> load_config_file(const std::string& config_file,

lib/common/Test.h

@@ -268,7 +268,7 @@ void sync_and_wait();
 void terminate_bbackupd(int pid);
 
 // Wait a given number of seconds for something to complete
-void wait_for_operation(int seconds, const char* message);
+void wait_for_operation(float seconds, const char* message);
 void safe_sleep(int seconds);
 std::auto_ptr<Configuration> load_config_file(const std::string& config_file,
 	const ConfigurationVerify& verify);

lib/httpserver/HTTPException.txt

@@ -27,3 +27,5 @@ SimpleDBItemNotFound			23	The requested item does not exist on the server
 FileNotFound				24	The requested file or directory does not exist on the server
 RequestTimedOut				25	The client took too long to send the request
 ResponseTimedOut			26	The client took too long to read the response
+TerminateWorkerNow			27	Forces the HTTP server worker process to terminate immediately, without returning a 500 response
+TerminateServerNow			28	Forces the HTTP server worker and master processes to terminate immediately

lib/httpserver/HTTPServer.cpp

@@ -162,6 +162,16 @@ void HTTPServer::Connection(std::auto_ptr<SocketStream> apConn)
 		}
 		catch(BoxException &e)
 		{
+			if(EXCEPTION_IS_TYPE(e, HTTPException, TerminateWorkerNow) ||
+				EXCEPTION_IS_TYPE(e, HTTPException, TerminateServerNow))
+			{
+				if(EXCEPTION_IS_TYPE(e, HTTPException, TerminateServerNow))
+				{
+					SetTerminateWanted();
+				}
+				throw;
+			}
+
 			char exceptionCode[256];
 			::sprintf(exceptionCode, "%s (%d/%d)", e.what(),
 				e.GetType(), e.GetSubType());

lib/httpserver/S3Client.h

@@ -65,6 +65,8 @@ class S3Client
 	  mNetworkTimeout(600000)
 	{ }
 
+	virtual ~S3Client() { }
+
 	class BucketEntry {
 	public:
 		BucketEntry(const std::string& name, const std::string& etag,
@@ -81,20 +83,20 @@ class S3Client
 		int64_t mSize;
 	};
 
-	int ListBucket(std::vector<S3Client::BucketEntry>* p_contents_out,
+	virtual int ListBucket(std::vector<S3Client::BucketEntry>* p_contents_out,
 		std::vector<std::string>* p_common_prefixes_out,
 		const std::string& prefix = "", const std::string& delimiter = "/",
 		bool* p_truncated_out = NULL, int max_keys = -1,
 		const std::string& marker = "");
-	HTTPResponse GetObject(const std::string& rObjectURI,
+	virtual HTTPResponse GetObject(const std::string& rObjectURI,
 		const std::string& MD5Checksum = "");
-	HTTPResponse HeadObject(const std::string& rObjectURI);
-	HTTPResponse PutObject(const std::string& rObjectURI,
+	virtual HTTPResponse HeadObject(const std::string& rObjectURI);
+	virtual HTTPResponse PutObject(const std::string& rObjectURI,
 		IOStream& rStreamToSend, const char* pContentType = NULL);
-	HTTPResponse DeleteObject(const std::string& rObjectURI);
-	bool DeleteObjectChecked(const std::string& rObjectURI, const std::string& message,
+	virtual HTTPResponse DeleteObject(const std::string& rObjectURI);
+	virtual bool DeleteObjectChecked(const std::string& rObjectURI, const std::string& message,
 		bool missing_ok = false);
-	void CheckResponse(const HTTPResponse& response, const std::string& message,
+	virtual void CheckResponse(const HTTPResponse& response, const std::string& message,
 		bool ExpectNoContent = false) const;
 	int GetNetworkTimeout() const { return mNetworkTimeout; }
 	const std::string& GetAccessKey() const { return mAccessKey; }

lib/httpserver/S3Simulator.cpp

@@ -404,6 +404,12 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse)
 	}
 	catch (BoxException &ce)
 	{
+		if(EXCEPTION_IS_TYPE(ce, HTTPException, TerminateWorkerNow) ||
+			EXCEPTION_IS_TYPE(ce, HTTPException, TerminateServerNow))
+		{
+			throw;
+		}
+
 		SendInternalErrorResponse(ce.what(), rResponse);
 
 		// Override the default status code 500 for a few specific exceptions.