test/bbackupd: add specialised versions of some tests for S3

Not all tests are specialised because many are quite slow, and would behave in
the same way, not expected to exercise any additional S3-related code paths
that are not covered by the tests that have been implemented.

.gitignore

@@ -34,6 +34,7 @@ test/backupstorefix/testfiles/testbackupstorefix.pl
 test/bbackupd/testfiles/bbackupd.conf
 test/bbackupd/testfiles/bbackupd.s3.conf
 test/bbackupd/testfiles/bbackupd-snapshot.conf
+test/bbackupd/testfiles/bbackupd-snapshot.s3.conf
 test/bbackupd/testfiles/bbackupd-symlink.conf
 test/bbackupd/testfiles/bbackupd-temploc.conf
 test/bbackupd/testfiles/extcheck1.pl

configure.ac

@@ -91,6 +91,7 @@ AC_CONFIG_FILES([infrastructure/BoxPlatform.pm
                  test/bbackupd/testfiles/bbackupd.conf
                  test/bbackupd/testfiles/bbackupd.s3.conf
                  test/bbackupd/testfiles/bbackupd-snapshot.conf
+                 test/bbackupd/testfiles/bbackupd-snapshot.s3.conf
                  test/bbackupd/testfiles/bbackupd-symlink.conf
                  test/bbackupd/testfiles/bbackupd-temploc.conf
   		 ])

infrastructure/cmake/CMakeLists.txt

@@ -43,6 +43,7 @@ set(files_to_configure
 	test/bbackupd/testfiles/bbackupd.conf
 	test/bbackupd/testfiles/bbackupd.s3.conf
 	test/bbackupd/testfiles/bbackupd-snapshot.conf
+	test/bbackupd/testfiles/bbackupd-snapshot.s3.conf
 	test/bbackupd/testfiles/bbackupd-symlink.conf
 	test/bbackupd/testfiles/bbackupd-temploc.conf
 	test/bbackupd/testfiles/extcheck1.pl

lib/bbackupd/BackupDaemon.cpp

@@ -2751,8 +2751,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con
 			BackupStoreFilenameClear clear(en->GetName());
 			const std::string &name(clear.GetClearFilename());
 			mUnusedRootDirEntries.push_back(
-				std::pair<int64_t,std::string>
-				(en->GetObjectID(), name));
+				std::pair<int64_t,std::string>(en->GetObjectID(), name));
 			// Log this
 			BOX_INFO("Unused location in root: " << name);
 		}

lib/common/Test.cpp

@@ -509,21 +509,16 @@ void terminate_bbackupd(int pid)
 
 
 // Wait a given number of seconds for something to complete
-void wait_for_operation(int seconds, const char* message)
+void wait_for_operation(float seconds, const char* message)
 {
 	BOX_INFO("Waiting " << seconds << " seconds for " << message);
-
-	for(int l = 0; l < seconds; ++l)
-	{
-		sleep(1);
-	}
-
+	ShortSleep(MilliSecondsToBoxTime(seconds * 1000), true); // logDuration
 	BOX_TRACE("Finished waiting for " << message);
 }
 
 void safe_sleep(int seconds)
 {
-	ShortSleep(SecondsToBoxTime(seconds), true);
+	ShortSleep(SecondsToBoxTime(seconds), true); // logDuration
 }
 
 std::auto_ptr<Configuration> load_config_file(const std::string& config_file,

lib/common/Test.h

@@ -268,7 +268,7 @@ void sync_and_wait();
 void terminate_bbackupd(int pid);
 
 // Wait a given number of seconds for something to complete
-void wait_for_operation(int seconds, const char* message);
+void wait_for_operation(float seconds, const char* message);
 void safe_sleep(int seconds);
 std::auto_ptr<Configuration> load_config_file(const std::string& config_file,
 	const ConfigurationVerify& verify);

lib/httpserver/HTTPException.txt

@@ -27,3 +27,5 @@ SimpleDBItemNotFound			23	The requested item does not exist on the server
 FileNotFound				24	The requested file or directory does not exist on the server
 RequestTimedOut				25	The client took too long to send the request
 ResponseTimedOut			26	The client took too long to read the response
+TerminateWorkerNow			27	Forces the HTTP server worker process to terminate immediately, without returning a 500 response
+TerminateServerNow			28	Forces the HTTP server worker and master processes to terminate immediately

lib/httpserver/HTTPServer.cpp

@@ -162,6 +162,16 @@ void HTTPServer::Connection(std::auto_ptr<SocketStream> apConn)
 		}
 		catch(BoxException &e)
 		{
+			if(EXCEPTION_IS_TYPE(e, HTTPException, TerminateWorkerNow) ||
+				EXCEPTION_IS_TYPE(e, HTTPException, TerminateServerNow))
+			{
+				if(EXCEPTION_IS_TYPE(e, HTTPException, TerminateServerNow))
+				{
+					SetTerminateWanted();
+				}
+				throw;
+			}
+
 			char exceptionCode[256];
 			::sprintf(exceptionCode, "%s (%d/%d)", e.what(),
 				e.GetType(), e.GetSubType());

lib/httpserver/S3Client.h

@@ -65,6 +65,8 @@ class S3Client
 	  mNetworkTimeout(600000)
 	{ }
 
+	virtual ~S3Client() { }
+
 	class BucketEntry {
 	public:
 		BucketEntry(const std::string& name, const std::string& etag,
@@ -81,20 +83,20 @@ class S3Client
 		int64_t mSize;
 	};
 
-	int ListBucket(std::vector<S3Client::BucketEntry>* p_contents_out,
+	virtual int ListBucket(std::vector<S3Client::BucketEntry>* p_contents_out,
 		std::vector<std::string>* p_common_prefixes_out,
 		const std::string& prefix = "", const std::string& delimiter = "/",
 		bool* p_truncated_out = NULL, int max_keys = -1,
 		const std::string& marker = "");
-	HTTPResponse GetObject(const std::string& rObjectURI,
+	virtual HTTPResponse GetObject(const std::string& rObjectURI,
 		const std::string& MD5Checksum = "");
-	HTTPResponse HeadObject(const std::string& rObjectURI);
-	HTTPResponse PutObject(const std::string& rObjectURI,
+	virtual HTTPResponse HeadObject(const std::string& rObjectURI);
+	virtual HTTPResponse PutObject(const std::string& rObjectURI,
 		IOStream& rStreamToSend, const char* pContentType = NULL);
-	HTTPResponse DeleteObject(const std::string& rObjectURI);
-	bool DeleteObjectChecked(const std::string& rObjectURI, const std::string& message,
+	virtual HTTPResponse DeleteObject(const std::string& rObjectURI);
+	virtual bool DeleteObjectChecked(const std::string& rObjectURI, const std::string& message,
 		bool missing_ok = false);
-	void CheckResponse(const HTTPResponse& response, const std::string& message,
+	virtual void CheckResponse(const HTTPResponse& response, const std::string& message,
 		bool ExpectNoContent = false) const;
 	int GetNetworkTimeout() const { return mNetworkTimeout; }
 	const std::string& GetAccessKey() const { return mAccessKey; }

lib/httpserver/S3Simulator.cpp

@@ -404,6 +404,12 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse)
 	}
 	catch (BoxException &ce)
 	{
+		if(EXCEPTION_IS_TYPE(ce, HTTPException, TerminateWorkerNow) ||
+			EXCEPTION_IS_TYPE(ce, HTTPException, TerminateServerNow))
+		{
+			throw;
+		}
+
 		SendInternalErrorResponse(ce.what(), rResponse);
 
 		// Override the default status code 500 for a few specific exceptions.

lib/server/ServerControl.cpp

@@ -212,15 +212,15 @@ bool KillServerInternal(int pid)
 
 #endif // WIN32
 
-bool KillServer(int pid, bool WaitForProcess)
+bool KillServer(int pid, bool wait_for_process)
 {
 	if(!KillServerInternal(pid))
 	{
 		return false;
 	}
 
 #ifdef HAVE_WAITPID
-	if(WaitForProcess)
+	if(wait_for_process)
 	{
 		int status, result;
 

lib/server/ServerControl.h

@@ -6,8 +6,9 @@
 #include "Test.h"
 
 bool HUPServer(int pid);
-bool KillServer(int pid, bool WaitForProcess = false);
+bool KillServer(int pid, bool wait_for_process = false);
 bool KillServer(const std::string& pid_file, bool WaitForProcess = false);
+bool KillServerInternal(int pid);
 int StartDaemon(int current_pid, const std::string& cmd_line, const char* pid_file, int port = 0,
 	const std::string& socket_path = "");
 bool StopDaemon(int current_pid, const std::string& pid_file,