Add real ETag and If-None-Match support to Amazon S3 simulator and cl…

…ient.

Add an MD5-calculating stream and use it to calculate MD5 checksum of files
being received during upload.

lib/httpserver/HTTPHeaders.h

@@ -48,6 +48,11 @@ class HTTPHeaders
 	void AddHeader(const std::string& name, const std::string& value);
 	void WriteTo(IOStream& rOutput, int Timeout) const;
 	typedef std::pair<std::string, std::string> Header;
+	bool HasHeader(const std::string& name) const
+	{
+		std::string dummy;
+		return GetHeader(name, &dummy);
+	}
 	bool GetHeader(const std::string& name, std::string* pValueOut) const;
 	std::string GetHeaderValue(const std::string& name, bool required = true) const
 	{

lib/httpserver/HTTPResponse.cpp

@@ -63,6 +63,10 @@ HTTPResponse::HTTPResponse(const HTTPResponse& rOther)
   mHeaders(rOther.mHeaders)
 {
 	Write(rOther.GetBuffer(), rOther.GetSize());
+	if(rOther.IsSetForReading())
+	{
+		SetForReading();
+	}
 }
 
 
@@ -74,6 +78,10 @@ HTTPResponse &HTTPResponse::operator=(const HTTPResponse &rOther)
 	mResponseIsDynamicContent = rOther.mResponseIsDynamicContent;
 	mHeaders = rOther.mHeaders;
 	mpStreamToSendTo = rOther.mpStreamToSendTo;
+	if(rOther.IsSetForReading())
+	{
+		SetForReading();
+	}
 	return *this;
 }
 

lib/httpserver/S3Client.cpp

@@ -31,16 +31,26 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    S3Client::GetObject(const std::string& rObjectURI)
+//		Name:    S3Client::GetObject(const std::string& rObjectURI,
+//			 const std::string& MD5Checksum)
 //		Purpose: Retrieve the object with the specified URI (key)
-//			 from your S3 bucket.
+//			 from your S3 bucket. If you supply an MD5 checksum,
+//			 then it is assumed that you already have the file
+//			 data with that checksum, and if the file version on
+//			 the server is the same, then you will get a 304
+//			 Not Modified response instead of a 200 OK, and no
+//			 file data.
 //		Created: 09/01/2009
 //
 // --------------------------------------------------------------------------
 
-HTTPResponse S3Client::GetObject(const std::string& rObjectURI)
+HTTPResponse S3Client::GetObject(const std::string& rObjectURI,
+	const std::string& MD5Checksum)
 {
-	return FinishAndSendRequest(HTTPRequest::Method_GET, rObjectURI);
+	return FinishAndSendRequest(HTTPRequest::Method_GET, rObjectURI,
+		NULL, // pStreamToSend
+		NULL, // pStreamContentType
+		MD5Checksum);
 }
 
 // --------------------------------------------------------------------------
@@ -100,7 +110,7 @@ HTTPResponse S3Client::PutObject(const std::string& rObjectURI,
 
 HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method,
 	const std::string& rRequestURI, IOStream* pStreamToSend,
-	const char* pStreamContentType)
+	const char* pStreamContentType, const std::string& MD5Checksum)
 {
 	HTTPRequest request(Method, rRequestURI);
 	request.SetHostName(mHostName);
@@ -131,6 +141,12 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method,
 		request.AddHeader("Content-Type", pStreamContentType);
 	}
 
+	if (!MD5Checksum.empty())
+	{
+		request.AddHeader("If-None-Match",
+			std::string("\"") + MD5Checksum + "\"");
+	}
+
 	std::string s3suffix = ".s3.amazonaws.com";
 	std::string bucket;
 	if (mHostName.size() > s3suffix.size())
@@ -176,20 +192,27 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method,
 	}
 
 	request.AddHeader("Authorization", auth_code);
-
+	HTTPResponse response;
+
 	if (mpSimulator)
 	{
 		if (pStreamToSend)
 		{
-			pStreamToSend->CopyStreamTo(request);
+			request.SetDataStream(pStreamToSend);
 		}
 
 		request.SetForReading();
-		CollectInBufferStream response_buffer;
-		HTTPResponse response(&response_buffer);
-
 		mpSimulator->Handle(request, response);
-		return response;
+
+		// TODO FIXME: HTTPServer::Connection does some post-processing on every
+		// response to determine whether Connection: keep-alive is possible.
+		// We should do that here too, but currently our HTTP implementation
+		// doesn't support chunked encoding, so it's disabled there, so we don't
+		// do it here either.
+
+		// We are definitely finished writing to the HTTPResponse, so leave it
+		// ready for reading back.
+		response.SetForReading();
 	}
 	else
 	{
@@ -201,7 +224,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method,
 				mapClientSocket->Open(Socket::TypeINET,
 					mHostName, mPort);
 			}
-			return SendRequest(request, pStreamToSend,
+			response = SendRequest(request, pStreamToSend,
 				pStreamContentType);
 		}
 		catch (ConnectionException &ce)
@@ -212,7 +235,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method,
 				// try to reconnect, just once
 				mapClientSocket->Open(Socket::TypeINET,
 					mHostName, mPort);
-				return SendRequest(request, pStreamToSend,
+				response = SendRequest(request, pStreamToSend,
 					pStreamContentType);
 			}
 			else
@@ -221,7 +244,20 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method,
 				throw;
 			}
 		}
+
+		// No need to call response.SetForReading() because HTTPResponse::Receive()
+		// already did that.
 	}
+
+	// It's not valid to have a keep-alive response if the length isn't known.
+	// S3Simulator should really check this, but depending on how it's called above,
+	// it might be possible to bypass that check, so this is a double-check.
+	ASSERT(response.GetContentLength() >= 0 || !response.IsKeepAlive());
+
+	BOX_TRACE("S3Client: " << mHostName << " < " << response.GetResponseCode() <<
+		": " << response.GetContentLength() << " bytes")
+
+	return response;
 }
 
 // --------------------------------------------------------------------------

lib/httpserver/S3Client.h

@@ -13,6 +13,7 @@
 #include <string>
 #include <map>
 
+#include "Configuration.h"
 #include "HTTPRequest.h"
 #include "SocketStream.h"
 
@@ -39,7 +40,7 @@ class S3Client
 	  mSecretKey(rSecretKey),
 	  mNetworkTimeout(30000)
 	{ }
-	
+
 	S3Client(std::string HostName, int Port, const std::string& rAccessKey,
 		const std::string& rSecretKey)
 	: mpSimulator(NULL),
@@ -49,8 +50,18 @@ class S3Client
 	  mSecretKey(rSecretKey),
 	  mNetworkTimeout(30000)
 	{ }
-
-	HTTPResponse GetObject(const std::string& rObjectURI);
+
+	S3Client(const Configuration& s3config)
+	: mpSimulator(NULL),
+	  mHostName(s3config.GetKeyValue("HostName")),
+	  mPort(s3config.GetKeyValueInt("Port")),
+	  mAccessKey(s3config.GetKeyValue("AccessKey")),
+	  mSecretKey(s3config.GetKeyValue("SecretKey")),
+	  mNetworkTimeout(30000)
+	{ }
+
+	HTTPResponse GetObject(const std::string& rObjectURI,
+		const std::string& MD5Checksum = "");
 	HTTPResponse HeadObject(const std::string& rObjectURI);
 	HTTPResponse PutObject(const std::string& rObjectURI,
 		IOStream& rStreamToSend, const char* pContentType = NULL);
@@ -68,7 +79,8 @@ class S3Client
 	HTTPResponse FinishAndSendRequest(HTTPRequest::Method Method,
 		const std::string& rRequestURI,
 		IOStream* pStreamToSend = NULL,
-		const char* pStreamContentType = NULL);
+		const char* pStreamContentType = NULL,
+		const std::string& MD5Checksum = "");
 	HTTPResponse SendRequest(HTTPRequest& rRequest,
 		IOStream* pStreamToSend = NULL,
 		const char* pStreamContentType = NULL);

lib/httpserver/S3Simulator.cpp

@@ -22,6 +22,7 @@
 #include "autogen_HTTPException.h"
 #include "IOStream.h"
 #include "Logging.h"
+#include "MD5Digest.h"
 #include "S3Simulator.h"
 #include "decode.h"
 #include "encode.h"
@@ -269,17 +270,55 @@ void S3Simulator::HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse)
 	path += rRequest.GetRequestURI();
 	std::auto_ptr<FileStream> apFile;
 	apFile.reset(new FileStream(path));
+	bool IncludeContent = true;
+	int64_t file_length;
+
+	std::string digest;
+	{
+		MD5DigestStream digester;
+		apFile->CopyStreamTo(digester);
+		file_length = apFile->GetPosition();
+		apFile->Seek(0, IOStream::SeekType_Absolute);
+		digester.Close();
+		digest = "\"" + digester.DigestAsString() + "\"";
+	}
 
 	rResponse.SetResponseCode(HTTPResponse::Code_OK);
 
-	if(true)
+	if(!IncludeContent)
+	{
+		// For HEAD requests, we must set the Content-Length. See RFC 2616 section
+		// 4.4, and the Amazon Simple Storage Service API Reference, section "HEAD
+		// Object" examples, which set it. Also, our S3BackupFileSystem needs it!
+		//
+		// There are no examples for 304 Not Modified responses to requests with
+		// If-None-Match (ETag match) so clients should not depend on this, so the
+		// S3Simulator should return 0 instead of the object size and no ETag, to
+		// ensure that any code which tries to use the Content-Length or ETag of
+		// such a response will fail.
+		//
+		// We do that by checking IncludeContent here, before clearing it in case
+		// of a digest match below, to leave them unset in that case.
+		rResponse.GetHeaders().SetContentLength(file_length);
+		rResponse.AddHeader("ETag", digest);
+	}
+
+	std::string if_none_match = rRequest.GetHeaders().GetHeaderValue("if-none-match",
+		false); // required
+	if(digest == if_none_match)
+	{
+		rResponse.SetResponseCode(HTTPResponse::Code_NotModified);
+		IncludeContent = false;
+	}
+
+	if(IncludeContent)
 	{
 		apFile->CopyStreamTo(rResponse);
 		// We allow the HTTPResponse to set the response size itself in this case,
 		// but we must add the ETag header. TODO: proper support for streaming
 		// responses will require us to set the content-length here, because we
 		// know it but the HTTPResponse does not.
-		rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\"");
+		rResponse.AddHeader("ETag", digest);
 	}
 
 	// http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingRESTOperations.html
@@ -310,7 +349,7 @@ void S3Simulator::HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse)
 
 	try
 	{
-		apFile.reset(new FileStream(path, O_CREAT | O_WRONLY));
+		apFile.reset(new FileStream(path, O_CREAT | O_RDWR));
 	}
 	catch (CommonException &ce)
 	{
@@ -330,14 +369,23 @@ void S3Simulator::HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse)
 		rResponse.SendContinue();
 	}
 
-	rRequest.ReadContent(*apFile, IOStream::TimeOutInfinite);
+	rRequest.ReadContent(*apFile, GetTimeout());
+	apFile->Seek(0, IOStream::SeekType_Absolute);
+
+	std::string digest;
+	{
+		MD5DigestStream digester;
+		apFile->CopyStreamTo(digester);
+		digester.Close();
+		digest = "\"" + digester.DigestAsString() + "\"";
+	}
 
 	// http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectPUT.html
 	rResponse.AddHeader("x-amz-id-2", "LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7");
 	rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D");
 	rResponse.AddHeader("Date", "Wed, 01 Mar  2006 12:00:00 GMT");
 	rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT");
-	rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\"");
+	rResponse.AddHeader("ETag", digest);
 	rResponse.SetContentType("");
 	rResponse.AddHeader("Server", "AmazonS3");
 	rResponse.SetResponseCode(HTTPResponse::Code_OK);