Permalink
Browse files

Merge pull request #1082 from rouault/s3_ignore_glacier

/vsis3/: ignores files with GLACIER storage class in directory listing, unless CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE=NO
  • Loading branch information...
rouault committed Nov 8, 2018
2 parents c1985e7 + 11d60d4 commit f7a6a86da9761e9d17274181d324c4cb964a93fa
@@ -735,6 +735,12 @@ def method(request):
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<Contents>
<Key>a_dir/i_am_a_glacier_file</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
<StorageClass>GLACIER</StorageClass>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
@@ -764,6 +770,7 @@ def method(request):
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL('/vsis3/s3_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
return 'fail'
@@ -825,6 +832,36 @@ def method(request):
print(dir_contents)
return 'fail'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/s3_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F', 200, {},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<Contents>
<Key>a_dir/resource4.bin</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<Contents>
<Key>a_dir/i_am_a_glacier_file</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
<StorageClass>GLACIER</StorageClass>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
</ListBucketResult>
""")
with gdaltest.config_option('CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE', 'NO'):
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsis3/s3_fake_bucket2/a_dir')
if dir_contents != ['resource4.bin', 'i_am_a_glacier_file', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# Test CPL_VSIL_CURL_NON_CACHED
for config_option_value in ['/vsis3/s3_non_cached/test.txt',
@@ -439,6 +439,10 @@ storage. You'll have to abort yourself with other means such "ghost" uploads
files smaller than the chunk size, a simple PUT request is used instead of
the multipart upload API.
Since GDAL 2.4, when listing a directory, files with GLACIER storage class are
ignored unless the CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE configuration option
is set to NO.
@since GDAL 2.1
\subsection gdal_virtual_file_systems_vsis3_streaming /vsis3_streaming/ (AWS S3 files: streaming)
@@ -2677,7 +2677,10 @@ const char* VSICurlFilesystemHandler::GetActualURL(const char* pszFilename)
"file' default='16384' min='1024' max='10485760'/>" \
" <Option name='CPL_VSIL_CURL_CACHE_SIZE' type='integer' " \
"description='Size in bytes of the global /vsicurl/ cache' " \
"default='16384000'/>"
"default='16384000'/>" \
" <Option name='CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE' type='boolean' " \
"description='Whether to skip files with Glacier storage class in " \
"directory listing.' default='YES'/>"
const char* VSICurlFilesystemHandler::GetOptionsStatic()
{
@@ -3649,10 +3652,13 @@ char** VSICurlFilesystemHandler::GetFileList(const char *pszDirname,
CPLString osBaseURL(pszDirname);
osBaseURL += "/";
bool bIsTruncated = true;
const bool bIgnoreGlacier = CPLTestBool(
CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE", "YES"));
AnalyseS3FileList( osBaseURL,
sWriteFuncData.pBuffer,
osFileList,
nMaxFiles,
bIgnoreGlacier,
bIsTruncated,
osNextMarker );
// If the list is truncated, then don't report it.
@@ -178,6 +178,7 @@ class VSICurlFilesystemHandler : public VSIFilesystemHandler
const char* pszXML,
CPLStringList& osFileList,
int nMaxFiles,
bool bIgnoreGlacierStorageClass,
bool& bIsTruncated,
CPLString& osNextMarker );
@@ -68,8 +68,9 @@ void VSICurlFilesystemHandler::AnalyseS3FileList(
const char* pszXML,
CPLStringList& osFileList,
int nMaxFiles,
bool bIgnoreGlacierStorageClass,
bool& bIsTruncated,
CPLString& osNextMarker )
CPLString& osNextMarker)
{
#if DEBUG_VERBOSE
CPLDebug("S3", "%s", pszXML);
@@ -135,10 +136,16 @@ void VSICurlFilesystemHandler::AnalyseS3FileList(
CPLYMDHMSToUnixTime(&brokendowntime));
}
aoProps.push_back(
std::pair<CPLString, FileProp>
(pszKey + osPrefix.size(), prop));
aoNameCount[pszKey + osPrefix.size()] ++;
const char* pszStorageClass = CPLGetXMLValue(psIter,
"StorageClass", "");
if( !bIgnoreGlacierStorageClass ||
!EQUAL(pszStorageClass, "GLACIER") )
{
aoProps.push_back(
std::pair<CPLString, FileProp>
(pszKey + osPrefix.size(), prop));
aoNameCount[pszKey + osPrefix.size()] ++;
}
}
}
else if( strcmp(psIter->pszValue, "CommonPrefixes") == 0 )
@@ -1768,10 +1775,13 @@ char** IVSIS3LikeFSHandler::GetFileList( const char *pszDirname,
{
*pbGotFileList = true;
bool bIsTruncated;
const bool bIgnoreGlacier = CPLTestBool(
CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE", "YES"));
AnalyseS3FileList( osBaseURL,
sWriteFuncData.pBuffer,
osFileList,
nMaxFiles,
bIgnoreGlacier,
bIsTruncated,
osNextMarker );

0 comments on commit f7a6a86

Please sign in to comment.