Skip to content
Permalink
Browse files

Add info() and infoList() to S3 driver.

These should be the last functions required to complete the implementation of the S3 driver.
  • Loading branch information...
dwsteele committed Jun 10, 2019
1 parent 9d1b037 commit 7f2f535460e3499c1811b5beca9130f84b78c982
Showing with 173 additions and 2 deletions.
  1. +101 −2 src/storage/s3/storage.c
  2. +72 −0 test/src/module/storage/s3Test.c
@@ -58,6 +58,7 @@ STRING_STATIC(S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, "NextContinu
STRING_STATIC(S3_XML_TAG_OBJECT_STR, "Object");
STRING_STATIC(S3_XML_TAG_PREFIX_STR, "Prefix");
STRING_STATIC(S3_XML_TAG_QUIET_STR, "Quiet");
STRING_STATIC(S3_XML_TAG_SIZE_STR, "Size");

/***********************************************************************************************************************************
AWS authentication v4 constants
@@ -497,6 +498,103 @@ storageS3Exists(THIS_VOID, const String *file)
FUNCTION_LOG_RETURN(BOOL, result);
}

/***********************************************************************************************************************************
File info
***********************************************************************************************************************************/
static StorageInfo
storageS3Info(THIS_VOID, const String *file, bool followLink)
{
THIS(StorageS3);

FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_S3, this);
FUNCTION_LOG_PARAM(STRING, file);
FUNCTION_LOG_END();

ASSERT(this != NULL);
ASSERT(file != NULL);
(void)followLink;

StorageInfo result = {0};

// Attempt to get file info
StorageS3RequestResult httpResult = storageS3Request(this, HTTP_VERB_HEAD_STR, file, NULL, NULL, false, true);

// On success load info into a structure
if (httpClientResponseCodeOk(this->httpClient))
{
result.exists = true;
result.size = cvtZToUInt64(strPtr(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_CONTENT_LENGTH_STR)));
}

FUNCTION_LOG_RETURN(STORAGE_INFO, result);
}

/***********************************************************************************************************************************
Info for all files/paths in a path
***********************************************************************************************************************************/
typedef struct StorageS3InfoListData
{
StorageInfoListCallback callback; // User-supplied callback function
void *callbackData; // User-supplied callback data
} StorageS3InfoListData;

static void
storageS3InfoListCallback(StorageS3 *this, void *callbackData, const String *name, StorageType type, const XmlNode *xml)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_S3, this);
FUNCTION_TEST_PARAM_P(VOID, callbackData);
FUNCTION_TEST_PARAM(STRING, name);
FUNCTION_TEST_PARAM(ENUM, type);
FUNCTION_TEST_PARAM(XML_NODE, xml);
FUNCTION_TEST_END();

(void)this;
ASSERT(callbackData != NULL);
ASSERT(name != NULL);
ASSERT(xml != NULL);

StorageS3InfoListData *data = (StorageS3InfoListData *)callbackData;

StorageInfo info =
{
.type = type,
.name = name,
.size = type == storageTypeFile ? cvtZToUInt64(strPtr(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_SIZE_STR, true)))) : 0,
};

data->callback(data->callbackData, &info);

FUNCTION_TEST_RETURN_VOID();
}

static bool
storageS3InfoList(THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData)
{
THIS(StorageS3);

FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_S3, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_END();

ASSERT(this != NULL);
ASSERT(path != NULL);
ASSERT(callback != NULL);

MEM_CONTEXT_TEMP_BEGIN()
{
StorageS3InfoListData data = {.callback = callback, .callbackData = callbackData};
storageS3ListInternal(this, path, false, false, storageS3InfoListCallback, &data);
}
MEM_CONTEXT_TEMP_END();

FUNCTION_LOG_RETURN(BOOL, true);
}

/***********************************************************************************************************************************
Get a list of files from a directory
***********************************************************************************************************************************/
@@ -828,8 +926,9 @@ storageS3New(

this = storageNewP(
STORAGE_S3_TYPE_STR, path, 0, 0, write, pathExpressionFunction, driver,
.exists = storageS3Exists, .list = storageS3List, .newRead = storageS3NewRead, .newWrite = storageS3NewWrite,
.pathRemove = storageS3PathRemove, .remove = storageS3Remove);
.exists = storageS3Exists, .info = storageS3Info, .infoList = storageS3InfoList, .list = storageS3List,
.newRead = storageS3NewRead, .newWrite = storageS3NewWrite, .pathRemove = storageS3PathRemove,
.remove = storageS3Remove);
}
MEM_CONTEXT_NEW_END();

@@ -179,6 +179,34 @@ testS3Server(void)
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:999", NULL));

// Info()
// -------------------------------------------------------------------------------------------------------------------------
// File missing
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL));
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL));

// File exists
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:9999", NULL));

// InfoList()
// -------------------------------------------------------------------------------------------------------------------------
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
" <Key>path/to/test_file</Key>"
" <Size>787</Size>"
" </Contents>"
" <CommonPrefixes>"
" <Prefix>path/to/test_path/</Prefix>"
" </CommonPrefixes>"
"</ListBucketResult>"));

// storageDriverList()
// -------------------------------------------------------------------------------------------------------------------------
// Throw error
@@ -404,6 +432,24 @@ testS3Server(void)
}
}

/***********************************************************************************************************************************
Callback and data for storageInfoList() tests
***********************************************************************************************************************************/
unsigned int testStorageInfoListSize = 0;
StorageInfo testStorageInfoList[256];

void
testStorageInfoListCallback(void *callbackData, const StorageInfo *info)
{
MEM_CONTEXT_BEGIN((MemContext *)callbackData)
{
testStorageInfoList[testStorageInfoListSize] = *info;
testStorageInfoList[testStorageInfoListSize].name = strDup(info->name);
testStorageInfoListSize++;
}
MEM_CONTEXT_END();
}

/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
@@ -712,6 +758,32 @@ testRun(void)
TEST_RESULT_BOOL(storageExistsNP(s3, strNew("BOGUS")), false, "file does not exist");
TEST_RESULT_BOOL(storageExistsNP(s3, strNew("subdir/file1.txt")), true, "file exists");

// Info()
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_BOOL(storageInfoP(s3, strNew("BOGUS"), .ignoreMissing = true).exists, false, "file does not exist");

StorageInfo info;
TEST_ASSIGN(info, storageInfoNP(s3, strNew("subdir/file1.txt")), "file exists");
TEST_RESULT_BOOL(info.exists, true, " check exists");
TEST_RESULT_UINT(info.size, 9999, " check exists");

// InfoList()
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(
storageInfoListP(s3, strNew("/"), testStorageInfoListCallback, NULL, .errorOnMissing = true),
AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");

TEST_RESULT_VOID(
storageInfoListNP(s3, strNew("/path/to"), testStorageInfoListCallback, (void *)memContextCurrent()), "info list files");

TEST_RESULT_UINT(testStorageInfoListSize, 2, " file and path returned");
TEST_RESULT_STR(strPtr(testStorageInfoList[0].name), "test_path", " check name");
TEST_RESULT_UINT(testStorageInfoList[0].size, 0, " check size");
TEST_RESULT_UINT(testStorageInfoList[0].type, storageTypePath, " check type");
TEST_RESULT_STR(strPtr(testStorageInfoList[1].name), "test_file", " check name");
TEST_RESULT_UINT(testStorageInfoList[1].size, 787, " check size");
TEST_RESULT_UINT(testStorageInfoList[1].type, storageTypeFile, " check type");

// storageDriverList()
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(

0 comments on commit 7f2f535

Please sign in to comment.
You can’t perform that action at this time.