Skip to content
Permalink
Browse files

Add pathRemove() and remove() to S3 storage.

These functions will be needed for the expire command.
  • Loading branch information...
dwsteele committed May 24, 2019
1 parent 6c385df commit 39645fc1a99540063399a7a3b86177a330bd743c
Showing with 245 additions and 7 deletions.
  1. +1 −0 src/common/io/http/client.c
  2. +2 −0 src/common/io/http/client.h
  3. +108 −4 src/storage/s3/storage.c
  4. +134 −3 test/src/module/storage/s3Test.c
@@ -19,6 +19,7 @@ Http constants
#define HTTP_VERSION "HTTP/1.1"
STRING_STATIC(HTTP_VERSION_STR, HTTP_VERSION);

STRING_EXTERN(HTTP_VERB_DELETE_STR, HTTP_VERB_DELETE);
STRING_EXTERN(HTTP_VERB_GET_STR, HTTP_VERB_GET);
STRING_EXTERN(HTTP_VERB_POST_STR, HTTP_VERB_POST);
STRING_EXTERN(HTTP_VERB_PUT_STR, HTTP_VERB_PUT);
@@ -28,6 +28,8 @@ typedef struct HttpClient HttpClient;
/***********************************************************************************************************************************
HTTP Constants
***********************************************************************************************************************************/
#define HTTP_VERB_DELETE "DELETE"
STRING_DECLARE(HTTP_VERB_DELETE_STR);
#define HTTP_VERB_GET "GET"
STRING_DECLARE(HTTP_VERB_GET_STR);
#define HTTP_VERB_POST "POST"
@@ -37,6 +37,7 @@ STRING_STATIC(S3_HEADER_TOKEN_STR, "x-amz-secur
S3 query tokens
***********************************************************************************************************************************/
STRING_STATIC(S3_QUERY_CONTINUATION_TOKEN_STR, "continuation-token");
STRING_STATIC(S3_QUERY_DELETE_STR, "delete");
STRING_STATIC(S3_QUERY_DELIMITER_STR, "delimiter");
STRING_STATIC(S3_QUERY_LIST_TYPE_STR, "list-type");
STRING_STATIC(S3_QUERY_PREFIX_STR, "prefix");
@@ -46,11 +47,17 @@ STRING_STATIC(S3_QUERY_VALUE_LIST_TYPE_2_STR, "2");
/***********************************************************************************************************************************
XML tags
***********************************************************************************************************************************/
STRING_STATIC(S3_XML_TAG_CODE_STR, "Code");
STRING_STATIC(S3_XML_TAG_COMMON_PREFIXES_STR, "CommonPrefixes");
STRING_STATIC(S3_XML_TAG_CONTENTS_STR, "Contents");
STRING_STATIC(S3_XML_TAG_DELETE_STR, "Delete");
STRING_STATIC(S3_XML_TAG_ERROR_STR, "Error");
STRING_STATIC(S3_XML_TAG_KEY_STR, "Key");
STRING_STATIC(S3_XML_TAG_MESSAGE_STR, "Message");
STRING_STATIC(S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, "NextContinuationToken");
STRING_STATIC(S3_XML_TAG_OBJECT_STR, "Object");
STRING_STATIC(S3_XML_TAG_PREFIX_STR, "Prefix");
STRING_STATIC(S3_XML_TAG_QUIET_STR, "Quiet");

/***********************************************************************************************************************************
AWS authentication v4 constants
@@ -435,7 +442,7 @@ storageS3List(THIS_VOID, const String *path, bool errorOnMissing, const String *
// Prepare regexp if an expression was passed
RegExp *regExp = expression == NULL ? NULL : regExpNew(expression);

// Build the base prefix by stripping of the initial /
// Build the base prefix by stripping off the initial /
const String *basePrefix;

if (strSize(path) == 1)
@@ -631,8 +638,104 @@ storageS3PathRemove(THIS_VOID, const String *path, bool errorOnMissing, bool rec

ASSERT(this != NULL);
ASSERT(path != NULL);
ASSERT(!errorOnMissing);

THROW(AssertError, "NOT YET IMPLEMENTED");
// S3 doesn't have paths that need to be deleted so nothing to do unless recursing
if (recurse)
{
MEM_CONTEXT_TEMP_BEGIN()
{
const String *continuationToken = NULL;

// Build the base prefix by stripping off the initial /
const String *basePrefix;

if (strSize(path) == 1)
basePrefix = EMPTY_STR;
else
basePrefix = strNewFmt("%s/", strPtr(strSub(path, 1)));

// Loop as long as a continuation token returned
do
{
// Use an inner mem context here because we could potentially be retrieving millions of files so it is a good idea
// to free memory at regular intervals
MEM_CONTEXT_TEMP_BEGIN()
{
HttpQuery *query = httpQueryNew();

// Add continuation token from the prior loop if any
if (continuationToken != NULL)
httpQueryAdd(query, S3_QUERY_CONTINUATION_TOKEN_STR, continuationToken);

// Use list type 2
httpQueryAdd(query, S3_QUERY_LIST_TYPE_STR, S3_QUERY_VALUE_LIST_TYPE_2_STR);

// Don't specified empty prefix because it is the default
if (!strEmpty(basePrefix))
httpQueryAdd(query, S3_QUERY_PREFIX_STR, basePrefix);

XmlNode *xmlRoot = xmlDocumentRoot(
xmlDocumentNewBuf(
storageS3Request(this, HTTP_VERB_GET_STR, FSLASH_STR, query, NULL, true, false).response));

// Get file list to delete
XmlNodeList *fileList = xmlNodeChildList(xmlRoot, S3_XML_TAG_CONTENTS_STR);
XmlDocument *delete = NULL;

for (unsigned int fileIdx = 0; fileIdx < xmlNodeLstSize(fileList); fileIdx++)
{
// If there is something to delete then create the request
if (delete == NULL)
{
delete = xmlDocumentNew(S3_XML_TAG_DELETE_STR);
xmlNodeContentSet(xmlNodeAdd(xmlDocumentRoot(delete), S3_XML_TAG_QUIET_STR), TRUE_STR);
}

// Add to delete list
xmlNodeContentSet(
xmlNodeAdd(xmlNodeAdd(xmlDocumentRoot(delete), S3_XML_TAG_OBJECT_STR), S3_XML_TAG_KEY_STR),
xmlNodeContent(xmlNodeChild(xmlNodeLstGet(fileList, fileIdx), S3_XML_TAG_KEY_STR, true)));
}

// If there is something to delete then send the request
if (delete != NULL)
{
// Delete file list
Buffer *xml = storageS3Request(
this, HTTP_VERB_POST_STR, FSLASH_STR, httpQueryAdd(httpQueryNew(), S3_QUERY_DELETE_STR, EMPTY_STR),
xmlDocumentBuf(delete), true, false).response;

// Nothing is returned when there are no errors
if (xml != NULL)
{
XmlNodeList *errorList = xmlNodeChildList(
xmlDocumentRoot(xmlDocumentNewBuf(xml)), S3_XML_TAG_ERROR_STR);

if (xmlNodeLstSize(errorList) > 0)
{
XmlNode *error = xmlNodeLstGet(errorList, 0);

THROW_FMT(
FileRemoveError, "unable to remove '%s': [%s] %s",
strPtr(xmlNodeContent(xmlNodeChild(error, S3_XML_TAG_KEY_STR, true))),
strPtr(xmlNodeContent(xmlNodeChild(error, S3_XML_TAG_CODE_STR, true))),
strPtr(xmlNodeContent(xmlNodeChild(error, S3_XML_TAG_MESSAGE_STR, true))));
}
}
}

// Get the continuation token and store it in the outer temp context
memContextSwitch(MEM_CONTEXT_OLD());
continuationToken = xmlNodeContent(xmlNodeChild(xmlRoot, S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, false));
memContextSwitch(MEM_CONTEXT_TEMP());
}
MEM_CONTEXT_TEMP_END();
}
while (continuationToken != NULL);
}
MEM_CONTEXT_TEMP_END();
}

FUNCTION_LOG_RETURN_VOID();
}
@@ -675,8 +778,9 @@ storageS3Remove(THIS_VOID, const String *file, bool errorOnMissing)

ASSERT(this != NULL);
ASSERT(file != NULL);
ASSERT(!errorOnMissing);

THROW(AssertError, "NOT YET IMPLEMENTED");
storageS3Request(this, HTTP_VERB_DELETE_STR, file, NULL, NULL, false, false);

FUNCTION_LOG_RETURN_VOID();
}
@@ -716,7 +820,7 @@ storageS3New(
FUNCTION_TEST_PARAM(STRING, accessKey);
FUNCTION_TEST_PARAM(STRING, secretAccessKey);
FUNCTION_TEST_PARAM(STRING, securityToken);
FUNCTION_TEST_PARAM(SIZE, partSize);
FUNCTION_LOG_PARAM(SIZE, partSize);
FUNCTION_LOG_PARAM(STRING, host);
FUNCTION_LOG_PARAM(UINT, port);
FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
@@ -306,8 +306,127 @@ testS3Server(void)
" </CommonPrefixes>"
"</ListBucketResult>"));

harnessTlsServerClose();
// storageDriverPathRemove()
// -------------------------------------------------------------------------------------------------------------------------
// delete files from root
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
" <Key>test1.txt</Key>"
" </Contents>"
" <Contents>"
" <Key>path1/xxx.zzz</Key>"
" </Contents>"
"</ListBucketResult>"));

harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_POST, "/?delete=",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
"<Object><Key>test1.txt</Key></Object>"
"<Object><Key>path1/xxx.zzz</Key></Object>"
"</Delete>\n"));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL, "<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"></DeleteResult>"));

// nothing to do in empty subpath
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=path%2F", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
"</ListBucketResult>"));

// delete with continuation
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=path%2Fto%2F", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <NextContinuationToken>continue</NextContinuationToken>"
" <Contents>"
" <Key>path/to/test1.txt</Key>"
" </Contents>"
" <Contents>"
" <Key>path/to/test2.txt</Key>"
" </Contents>"
"</ListBucketResult>"));

harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_POST, "/?delete=",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
"<Object><Key>path/to/test1.txt</Key></Object>"
"<Object><Key>path/to/test2.txt</Key></Object>"
"</Delete>\n"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));

harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?continuation-token=continue&list-type=2&prefix=path%2Fto%2F", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
" <Key>path/to/test3.txt</Key>"
" </Contents>"
"</ListBucketResult>"));

harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_POST, "/?delete=",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
"<Object><Key>path/to/test3.txt</Key></Object>"
"</Delete>\n"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));

// delete error
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=path%2F", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
" <Key>path/sample.txt</Key>"
" </Contents>"
" <Contents>"
" <Key>path/sample2.txt</Key>"
" </Contents>"
"</ListBucketResult>"));

harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_POST, "/?delete=",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
"<Object><Key>path/sample.txt</Key></Object>"
"<Object><Key>path/sample2.txt</Key></Object>"
"</Delete>\n"));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
"<Error><Key>sample2.txt</Key><Code>AccessDenied</Code><Message>Access Denied</Message></Error>"
"</DeleteResult>"));

// storageDriverRemove()
// -------------------------------------------------------------------------------------------------------------------------
// remove file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_DELETE, "/path/to/test.txt", NULL));
harnessTlsServerReply(testS3ServerResponse(204, "No Content", NULL, NULL));

harnessTlsServerClose();
exit(0);
}
}
@@ -644,11 +763,23 @@ testRun(void)
strPtr(strLstJoin(storageListP(s3, strNew("/path/to"), .expression = strNew("^test(1|3)")), ",")),
"test1.path,test1.txt,test3.txt", "list files with expression");

// storageDriverPathRemove()
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(storagePathRemoveNP(s3, strNew("/")), "do nothing when no recurse");
TEST_RESULT_VOID(storagePathRemoveP(s3, strNew("/"), .recurse = true), "remove root path");
TEST_RESULT_VOID(storagePathRemoveP(s3, strNew("/path"), .recurse = true), "nothing to do in empty subpath");
TEST_RESULT_VOID(storagePathRemoveP(s3, strNew("/path/to"), .recurse = true), "delete with continuation");
TEST_ERROR(
storagePathRemoveP(s3, strNew("/path"), .recurse = true), FileRemoveError,
"unable to remove 'sample2.txt': [AccessDenied] Access Denied");

// storageDriverRemove()
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(storageRemoveNP(s3, strNew("/path/to/test.txt")), "remove file");

// Coverage for unimplemented functions
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(storageInfoNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
TEST_ERROR(storagePathRemoveNP(s3, strNew("path")), AssertError, "NOT YET IMPLEMENTED");
TEST_ERROR(storageRemoveNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
}

FUNCTION_HARNESS_RESULT_VOID();

0 comments on commit 39645fc

Please sign in to comment.
You can’t perform that action at this time.