Permalink
Browse files

Incorporated change from Yehuda Sadeh which allows runtime

customization of the S3 server to talk to, with additional small
improvements, enhanced by me to support multiple S3 hosts in the same
runtime.
  • Loading branch information...
1 parent 73366b8 commit 5465b5523a323f239cfc0f724afc1f900e24fcc4 @bji bji committed Sep 28, 2011
Showing with 144 additions and 62 deletions.
  1. +3 −0 GNUmakefile
  2. +3 −0 GNUmakefile.mingw
  3. +3 −0 GNUmakefile.osx
  4. +38 −11 inc/libs3.h
  5. +2 −1 inc/request.h
  6. +9 −1 inc/util.h
  7. +7 −8 src/acl.c
  8. +14 −8 src/bucket.c
  9. +6 −8 src/general.c
  10. +10 −5 src/object.c
  11. +19 −4 src/request.c
  12. +19 −5 src/s3.c
  13. +3 −2 src/service.c
  14. +8 −9 src/service_access_logging.c
View
@@ -37,6 +37,9 @@
# --------------------------------------------------------------------------
# Set libs3 version number, unless it is already set.
+# This is trunk0.trunk0 on the libs3 git master branch; release branches
+# are created with this set to specific version numbers when releases are
+# made.
LIBS3_VER_MAJOR ?= trunk0
LIBS3_VER_MINOR ?= trunk0
View
@@ -37,6 +37,9 @@
# --------------------------------------------------------------------------
# Set libs3 version number, unless it is already set.
+# This is trunk0.trunk0 on the libs3 git master branch; release branches
+# are created with this set to specific version numbers when releases are
+# made.
LIBS3_VER_MAJOR ?= trunk0
LIBS3_VER_MINOR ?= trunk0
View
@@ -37,6 +37,9 @@
# --------------------------------------------------------------------------
# Set libs3 version number, unless it is already set.
+# This is trunk0.trunk0 on the libs3 git master branch; release branches
+# are created with this set to specific version numbers when releases are
+# made.
LIBS3_VER_MAJOR ?= trunk0
LIBS3_VER_MINOR ?= trunk0
View
@@ -116,11 +116,14 @@ extern "C" {
************************************************************************** **/
/**
- * This is the hostname that all S3 requests will go through; virtual-host
- * style requests will prepend the bucket name to this host name, and
- * path-style requests will use this hostname directly
+ * S3_MAX_HOSTNAME_SIZE is the maximum size we allow for a host name
**/
-#define S3_HOSTNAME "s3.amazonaws.com"
+#define S3_MAX_HOSTNAME_SIZE 255
+
+/**
+ * This is the default hostname that is being used for the S3 requests
+ **/
+#define S3_DEFAULT_HOSTNAME "s3.amazonaws.com"
/**
@@ -194,7 +197,7 @@ extern "C" {
* query string
**/
#define S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE \
- (sizeof("https://" S3_HOSTNAME "/") + (S3_MAX_KEY_SIZE * 3) + \
+ (sizeof("https:///") + S3_MAX_HOSTNAME_SIZE + (S3_MAX_KEY_SIZE * 3) + \
sizeof("?AWSAccessKeyId=") + 32 + sizeof("&Expires=") + 32 + \
sizeof("&Signature=") + 28 + 1)
@@ -636,6 +639,12 @@ typedef struct S3AclGrant
typedef struct S3BucketContext
{
/**
+ * The name of the host to connect to when making S3 requests. If set to
+ * NULL, the default S3 hostname passed in to S3_initialize will be used.
+ **/
+ const char *hostName;
+
+ /**
* The name of the bucket to use in the bucket context
**/
const char *bucketName;
@@ -1145,13 +1154,20 @@ typedef struct S3GetObjectHandler
* all necessary initialization; however, be warned that things may
* break if your application re-initializes the dependent libraries
* later.
+ * @param defaultS3Hostname is a string the specifies the default S3 server
+ * hostname to use when making S3 requests; this value is used
+ * whenever the hostName of an S3BucketContext is NULL. If NULL is
+ * passed here then the default of S3_DEFAULT_HOSTNAME will be used.
* @return One of:
* S3StatusOK on success
+ * S3StatusUriTooLong if the defaultS3HostName is longer than
+ * S3_MAX_HOSTNAME_SIZE
* S3StatusInternalError if dependent libraries could not be
* initialized
* S3StatusOutOfMemory on failure due to out of memory
**/
-S3Status S3_initialize(const char *userAgentInfo, int flags);
+S3Status S3_initialize(const char *userAgentInfo, int flags,
+ const char *defaultS3HostName);
/**
@@ -1428,6 +1444,8 @@ S3Status S3_generate_authenticated_query_string
* buckets
* @param secretAccessKey gives the Amazon Secret Access Key for which to list
* owned buckets
+ * @param hostName is the S3 host name to use; if NULL is passed in, the
+ * default S3 host as provided to S3_initialize() will be used.
* @param requestContext if non-NULL, gives the S3RequestContext to add this
* request to, and does not perform the request immediately. If NULL,
* performs the request immediately and synchronously.
@@ -1437,7 +1455,7 @@ S3Status S3_generate_authenticated_query_string
* all callbacks for this request
**/
void S3_list_service(S3Protocol protocol, const char *accessKeyId,
- const char *secretAccessKey,
+ const char *secretAccessKey, const char *hostName,
S3RequestContext *requestContext,
const S3ListServiceHandler *handler,
void *callbackData);
@@ -1457,6 +1475,8 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId,
* buckets
* @param secretAccessKey gives the Amazon Secret Access Key for which to list
* owned buckets
+ * @param hostName is the S3 host name to use; if NULL is passed in, the
+ * default S3 host as provided to S3_initialize() will be used.
* @param bucketName is the bucket name to test
* @param locationConstraintReturnSize gives the number of bytes in the
* locationConstraintReturn parameter
@@ -1477,7 +1497,8 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId,
**/
void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle,
const char *accessKeyId, const char *secretAccessKey,
- const char *bucketName, int locationConstraintReturnSize,
+ const char *hostName, const char *bucketName,
+ int locationConstraintReturnSize,
char *locationConstraintReturn,
S3RequestContext *requestContext,
const S3ResponseHandler *handler, void *callbackData);
@@ -1491,6 +1512,8 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle,
* buckets
* @param secretAccessKey gives the Amazon Secret Access Key for which to list
* owned buckets
+ * @param hostName is the S3 host name to use; if NULL is passed in, the
+ * default S3 host as provided to S3_initialize() will be used.
* @param bucketName is the name of the bucket to be created
* @param cannedAcl gives the "REST canned ACL" to use for the created bucket
* @param locationConstraint if non-NULL, gives the geographic location for
@@ -1504,8 +1527,9 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle,
* all callbacks for this request
**/
void S3_create_bucket(S3Protocol protocol, const char *accessKeyId,
- const char *secretAccessKey, const char *bucketName,
- S3CannedAcl cannedAcl, const char *locationConstraint,
+ const char *secretAccessKey, const char *hostName,
+ const char *bucketName, S3CannedAcl cannedAcl,
+ const char *locationConstraint,
S3RequestContext *requestContext,
const S3ResponseHandler *handler, void *callbackData);
@@ -1520,6 +1544,8 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId,
* buckets
* @param secretAccessKey gives the Amazon Secret Access Key for which to list
* owned buckets
+ * @param hostName is the S3 host name to use; if NULL is passed in, the
+ * default S3 host as provided to S3_initialize() will be used.
* @param bucketName is the name of the bucket to be deleted
* @param requestContext if non-NULL, gives the S3RequestContext to add this
* request to, and does not perform the request immediately. If NULL,
@@ -1531,7 +1557,8 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId,
**/
void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle,
const char *accessKeyId, const char *secretAccessKey,
- const char *bucketName, S3RequestContext *requestContext,
+ const char *hostName, const char *bucketName,
+ S3RequestContext *requestContext,
const S3ResponseHandler *handler, void *callbackData);
View
@@ -165,7 +165,8 @@ typedef struct Request
// ----------------------------------------------------------------------------
// Initialize the API
-S3Status request_api_initialize(const char *userAgentInfo, int flags);
+S3Status request_api_initialize(const char *userAgentInfo, int flags,
+ const char *hostName);
// Deinitialize the API
void request_api_deinitialize();
View
@@ -32,6 +32,14 @@
#include <stdint.h>
#include "libs3.h"
+// acl groups
+#define ACS_URL "http://acs.amazonaws.com/groups/"
+
+#define ACS_GROUP_ALL_USERS ACS_URL "global/AllUsers"
+#define ACS_GROUP_AWS_USERS ACS_URL "global/AuthenticatedUsers"
+#define ACS_GROUP_LOG_DELIVERY ACS_URL "s3/LogDelivery"
+
+
// Derived from S3 documentation
@@ -49,7 +57,7 @@
// https://s3.amazonaws.com/${BUCKET}/${KEY}?acl
// 255 is the maximum bucket length
#define MAX_URI_SIZE \
- ((sizeof("https://" S3_HOSTNAME "/") - 1) + 255 + 1 + \
+ ((sizeof("https:///") - 1) + S3_MAX_HOSTNAME_SIZE + 255 + 1 + \
MAX_URLENCODED_KEY_SIZE + (sizeof("?torrent" - 1)) + 1)
// Maximum size of a canonicalized resource
View
@@ -122,7 +122,8 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key,
RequestParams params =
{
HttpRequestTypeGET, // httpRequestType
- { bucketContext->bucketName, // bucketName
+ { bucketContext->hostName, // hostName
+ bucketContext->bucketName, // bucketName
bucketContext->protocol, // protocol
bucketContext->uriStyle, // uriStyle
bucketContext->accessKeyId, // accessKeyId
@@ -195,16 +196,13 @@ static S3Status generateAclXmlDocument(const char *ownerId,
const char *grantee;
switch (grant->granteeType) {
case S3GranteeTypeAllAwsUsers:
- grantee = "http://acs.amazonaws.com/groups/global/"
- "AuthenticatedUsers";
+ grantee = ACS_GROUP_AWS_USERS;
break;
case S3GranteeTypeAllUsers:
- grantee = "http://acs.amazonaws.com/groups/global/"
- "AllUsers";
+ grantee = ACS_GROUP_ALL_USERS;
break;
default:
- grantee = "http://acs.amazonaws.com/groups/s3/"
- "LogDelivery";
+ grantee = ACS_GROUP_LOG_DELIVERY;
break;
}
append("Group\"><URI>%s</URI>", grantee);
@@ -322,7 +320,8 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key,
RequestParams params =
{
HttpRequestTypePUT, // httpRequestType
- { bucketContext->bucketName, // bucketName
+ { bucketContext->hostName, // hostName
+ bucketContext->bucketName, // bucketName
bucketContext->protocol, // protocol
bucketContext->uriStyle, // uriStyle
bucketContext->accessKeyId, // accessKeyId
View
@@ -107,7 +107,8 @@ static void testBucketCompleteCallback(S3Status requestStatus,
void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle,
const char *accessKeyId, const char *secretAccessKey,
- const char *bucketName, int locationConstraintReturnSize,
+ const char *hostName, const char *bucketName,
+ int locationConstraintReturnSize,
char *locationConstraintReturn,
S3RequestContext *requestContext,
const S3ResponseHandler *handler, void *callbackData)
@@ -134,7 +135,8 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle,
RequestParams params =
{
HttpRequestTypeGET, // httpRequestType
- { bucketName, // bucketName
+ { hostName, // hostName
+ bucketName, // bucketName
protocol, // protocol
uriStyle, // uriStyle
accessKeyId, // accessKeyId
@@ -223,8 +225,9 @@ static void createBucketCompleteCallback(S3Status requestStatus,
void S3_create_bucket(S3Protocol protocol, const char *accessKeyId,
- const char *secretAccessKey, const char *bucketName,
- S3CannedAcl cannedAcl, const char *locationConstraint,
+ const char *secretAccessKey, const char *hostName,
+ const char *bucketName, S3CannedAcl cannedAcl,
+ const char *locationConstraint,
S3RequestContext *requestContext,
const S3ResponseHandler *handler, void *callbackData)
{
@@ -270,7 +273,8 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId,
RequestParams params =
{
HttpRequestTypePUT, // httpRequestType
- { bucketName, // bucketName
+ { hostName, // hostName
+ bucketName, // bucketName
protocol, // protocol
S3UriStylePath, // uriStyle
accessKeyId, // accessKeyId
@@ -332,7 +336,7 @@ static void deleteBucketCompleteCallback(S3Status requestStatus,
void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle,
const char *accessKeyId, const char *secretAccessKey,
- const char *bucketName,
+ const char *hostName, const char *bucketName,
S3RequestContext *requestContext,
const S3ResponseHandler *handler, void *callbackData)
{
@@ -352,7 +356,8 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle,
RequestParams params =
{
HttpRequestTypeDELETE, // httpRequestType
- { bucketName, // bucketName
+ { hostName, // hostName
+ bucketName, // bucketName
protocol, // protocol
uriStyle, // uriStyle
accessKeyId, // accessKeyId
@@ -710,7 +715,8 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix,
RequestParams params =
{
HttpRequestTypeGET, // httpRequestType
- { bucketContext->bucketName, // bucketName
+ { bucketContext->hostName, // hostName
+ bucketContext->bucketName, // bucketName
bucketContext->protocol, // protocol
bucketContext->uriStyle, // uriStyle
bucketContext->accessKeyId, // accessKeyId
View
@@ -32,13 +32,14 @@
static int initializeCountG = 0;
-S3Status S3_initialize(const char *userAgentInfo, int flags)
+S3Status S3_initialize(const char *userAgentInfo, int flags,
+ const char *defaultS3HostName)
{
if (initializeCountG++) {
return S3StatusOK;
}
- return request_api_initialize(userAgentInfo, flags);
+ return request_api_initialize(userAgentInfo, flags, defaultS3HostName);
}
@@ -372,18 +373,15 @@ static S3Status convertAclXmlCallback(const char *elementPath,
}
else if (caData->groupUri[0]) {
if (!strcmp(caData->groupUri,
- "http://acs.amazonaws.com/groups/global/"
- "AuthenticatedUsers")) {
+ ACS_GROUP_AWS_USERS)) {
grant->granteeType = S3GranteeTypeAllAwsUsers;
}
else if (!strcmp(caData->groupUri,
- "http://acs.amazonaws.com/groups/global/"
- "AllUsers")) {
+ ACS_GROUP_ALL_USERS)) {
grant->granteeType = S3GranteeTypeAllUsers;
}
else if (!strcmp(caData->groupUri,
- "http://acs.amazonaws.com/groups/s3/"
- "LogDelivery")) {
+ ACS_GROUP_LOG_DELIVERY)) {
grant->granteeType = S3GranteeTypeLogDelivery;
}
else {
Oops, something went wrong.

0 comments on commit 5465b55

Please sign in to comment.