39 changes: 12 additions & 27 deletions src/providers/gdal/qgsgdalprovider.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,19 +111,12 @@ QgsGdalProvider::QgsGdalProvider( QString const & uri )
mGdalDataset = NULL;

// Try to open using VSIFileHandler (see qgsogrprovider.cpp)
// TODO suppress error messages and report in debug, like in OGR provider
// TODO use the file name of the file inside the zip, needs unzip.h
if ( uri.endsWith( ".zip", Qt::CaseInsensitive ) )
{
if ( !uri.startsWith( "/vsizip/" ) )
setDataSourceUri( "/vsizip/" + uri );
QgsDebugMsg( QString( "Trying /vsizip syntax, uri= %1" ).arg( dataSourceUri() ) );
}
else if ( uri.endsWith( ".gz", Qt::CaseInsensitive ) )
QString vsiPrefix = QgsZipItem::vsiPrefix( uri );
if ( vsiPrefix != "" )
{
if ( !uri.startsWith( "/vsigzip/" ) )
setDataSourceUri( "/vsigzip/" + uri );
QgsDebugMsg( QString( "Trying /vsigzip syntax, uri= %1" ).arg( dataSourceUri() ) );
if ( !uri.startsWith( vsiPrefix ) )
setDataSourceUri( vsiPrefix + uri );
QgsDebugMsg( QString( "Trying %1 syntax, uri= %2" ).arg( vsiPrefix ).arg( dataSourceUri() ) );
}

// The uri is either a file name or encoded parameters for WCS
Expand Down Expand Up @@ -1920,8 +1913,9 @@ void buildSupportedRasterFileFilterAndExtensions( QString & theFileFiltersString
{
QString glob = "*.zip";
glob += " *.gz";
glob += " *.tar *.tar.gz *.tgz";
theFileFiltersString += ";;[GDAL] " + QObject::tr( "GDAL/OGR VSIFileHandler" ) + " (" + glob.toLower() + " " + glob.toUpper() + ")";
theExtensions << "zip" << "gz";
theExtensions << "zip" << "gz" << "tar" << "tar.gz" << "tgz";
}
#endif

Expand All @@ -1940,21 +1934,12 @@ QGISEXTERN bool isValidRasterFileName( QString const & theFileNameQString, QStri

// Try to open using VSIFileHandler (see qgsogrprovider.cpp)
// TODO suppress error messages and report in debug, like in OGR provider
if ( fileName.endsWith( ".zip", Qt::CaseInsensitive ) )
QString vsiPrefix = QgsZipItem::vsiPrefix( fileName );
if ( vsiPrefix != "" )
{
if ( !fileName.startsWith( "/vsizip/" ) )
{
fileName = "/vsizip/" + fileName;
}
QgsDebugMsg( QString( "Trying /vsizip syntax, fileName= %1" ).arg( fileName ) );
}
if ( fileName.endsWith( ".gz", Qt::CaseInsensitive ) )
{
if ( !fileName.startsWith( "/vsigzip/" ) )
{
fileName = "/vsigzip/" + fileName;
}
QgsDebugMsg( QString( "Trying /vsigzip syntax, fileName= %1" ).arg( fileName ) );
if ( !fileName.startsWith( vsiPrefix ) )
fileName = vsiPrefix + fileName;
QgsDebugMsg( QString( "Trying %1 syntax, fileName= %2" ).arg( vsiPrefix ).arg( fileName ) );
}

//open the file using gdal making sure we have handled locale properly
Expand Down
42 changes: 18 additions & 24 deletions src/providers/ogr/qgsogrdataitems.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,10 +240,10 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
// zip settings + info
QSettings settings;
QString scanZipSetting = settings.value( "/qgis/scanZipInBrowser", "basic" ).toString();
bool is_vsizip = ( thePath.startsWith( "/vsizip/" ) ||
thePath.endsWith( ".zip", Qt::CaseInsensitive ) );
bool is_vsigzip = ( thePath.startsWith( "/vsigzip/" ) ||
thePath.endsWith( ".gz", Qt::CaseInsensitive ) );
QString vsiPrefix = QgsZipItem::vsiPrefix( thePath );
bool is_vsizip = ( vsiPrefix == "/vsizip/" );
bool is_vsigzip = ( vsiPrefix == "/vsigzip/" );
bool is_vsitar = ( vsiPrefix == "/vsitar/" );

// get suffix, removing .gz if present
QString tmpPath = thePath; //path used for testing, not for layer creation
Expand All @@ -255,10 +255,11 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
info.setFile( thePath );
QString name = info.fileName();

QgsDebugMsg( "thePath= " + thePath + " tmpPath= " + tmpPath + " name= " + name + " suffix= " + suffix );
QgsDebugMsg( "thePath= " + thePath + " tmpPath= " + tmpPath + " name= " + name
+ " suffix= " + suffix + " vsiPrefix= " + vsiPrefix );

// allow only normal files or VSIFILE items to continue
if ( !info.isFile() && !is_vsizip && !is_vsigzip )
if ( !info.isFile() && vsiPrefix == "" )
return 0;

QStringList myExtensions = fileExtensions();
Expand All @@ -272,10 +273,6 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
!myExtensions.contains( "shp.xml" ) )
return 0;

// skip .tar.gz files
if ( thePath.endsWith( ".tar.gz", Qt::CaseInsensitive ) )
return 0;

// We have to filter by extensions, otherwise e.g. all Shapefile files are displayed
// because OGR drive can open also .dbf, .shx.
if ( myExtensions.indexOf( suffix ) < 0 )
Expand All @@ -302,30 +299,27 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
return 0;
}

// add /vsizip/ or /vsigzip/ to path if file extension is .zip or .gz
if ( is_vsigzip )
{
if ( !thePath.startsWith( "/vsigzip/" ) )
thePath = "/vsigzip/" + thePath;
}
else if ( is_vsizip )
// fix vsifile path and name
if ( vsiPrefix != "" )
{
if ( !thePath.startsWith( "/vsizip/" ) )
thePath = "/vsizip/" + thePath;
// add vsiPrefix to path if needed
if ( !thePath.startsWith( vsiPrefix ) )
thePath = vsiPrefix + thePath;
// if this is a /vsigzip/path_to_zip.zip/file_inside_zip remove the full path from the name
if ( thePath != "/vsizip/" + parentItem->path() )
if (( is_vsizip || is_vsitar ) && ( thePath != vsiPrefix + parentItem->path() ) )
{
name = thePath;
name = name.replace( "/vsizip/" + parentItem->path() + "/", "" );
name = name.replace( vsiPrefix + parentItem->path() + "/", "" );
}
}

// return a /vsizip/ item without testing if:
// zipfile and scan zip == "Basic scan"
// not zipfile and scan items == "Check extension"
if (( is_vsizip && scanZipSetting == "basic" ) ||
( !is_vsizip && ( settings.value( "/qgis/scanItemsInBrowser",
"extension" ).toString() == "extension" ) ) )
if ((( is_vsizip || is_vsitar ) && scanZipSetting == "basic" ) ||
( !is_vsizip && !is_vsitar &&
( settings.value( "/qgis/scanItemsInBrowser",
"extension" ).toString() == "extension" ) ) )
{
// if this is a VRT file make sure it is vector VRT to avoid duplicates
if ( suffix == "vrt" )
Expand Down
26 changes: 8 additions & 18 deletions src/providers/ogr/qgsogrprovider.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,26 +264,18 @@ QgsOgrProvider::QgsOgrProvider( QString const & uri )

// Try to open using VSIFileHandler
// see http://trac.osgeo.org/gdal/wiki/UserDocs/ReadInZip
if ( mFilePath.endsWith( ".zip", Qt::CaseInsensitive ) )
QString vsiPrefix = QgsZipItem::vsiPrefix( uri );
if ( vsiPrefix != "" )
{
// GDAL>=1.8.0 has write support for zip, but read and write operations
// cannot be interleaved, so for now just use read-only.
openReadOnly = true;
if ( !mFilePath.startsWith( "/vsizip/" ) )
if ( !mFilePath.startsWith( vsiPrefix ) )
{
mFilePath = "/vsizip/" + mFilePath;
mFilePath = vsiPrefix + mFilePath;
setDataSourceUri( mFilePath );
}
QgsDebugMsg( QString( "Trying /vsizip syntax, mFilePath= %1" ).arg( mFilePath ) );
}
else if ( mFilePath.endsWith( ".gz", Qt::CaseInsensitive ) )
{
if ( !mFilePath.startsWith( "/vsigzip/" ) )
{
mFilePath = "/vsigzip/" + mFilePath;
setDataSourceUri( mFilePath );
}
QgsDebugMsg( QString( "Trying /vsigzip syntax, mFilePath= %1" ).arg( mFilePath ) );
QgsDebugMsg( QString( "Trying %1 syntax, mFilePath= %2" ).arg( vsiPrefix ).arg( mFilePath ) );
}

QgsDebugMsg( "mFilePath: " + mFilePath );
Expand Down Expand Up @@ -1792,16 +1784,14 @@ QString createFilters( QString type )
// VSIFileHandler (.zip and .gz files)
// see http://trac.osgeo.org/gdal/wiki/UserDocs/ReadInZip
// Requires GDAL>=1.6.0 with libz support, let's assume we have it.
// For .zip this works only if there is one file (or dataset) in the root of the zip.
// Only tested with tiff, shape (zip) and spatialite (zip and gz).
// This does not work for some file types, see VSIFileHandler doc.
// Ideally we should also add support for /vsitar/ (requires cpl_vsil_tar.cpp).
#if defined(GDAL_VERSION_NUM) && GDAL_VERSION_NUM >= 1600
QSettings settings;
if ( settings.value( "/qgis/scanZipInBrowser", "basic" ).toString() != "no" )
{
myFileFilters += createFileFilter_( QObject::tr( "GDAL/OGR VSIFileHandler" ), "*.zip *.gz" );
myExtensions << "zip" << "gz";
myFileFilters += createFileFilter_( QObject::tr( "GDAL/OGR VSIFileHandler" ), "*.zip *.gz *.tar *.tar.gz *.tgz" );
myExtensions << "zip" << "gz" << "tar" << "tar.gz" << "tgz";

}
#endif

Expand Down
157 changes: 125 additions & 32 deletions tests/src/core/testziplayer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ class TestZipLayer: public QObject
bool testZipItem( QString myFileName, QString myChildName = "", QString myDriverName = "" );
// get layer transparency to test for .qml loading
int getLayerTransparency( QString myFileName, QString myProviderKey, QString myScanZipSetting = "basic" );
bool testZipItemTransparency( QString myFileName, QString myProviderKey, int myTarget );

private slots:

Expand All @@ -65,20 +66,28 @@ class TestZipLayer: public QObject
// tests
// test for .zip and .gz files using all options
void testPassthruVectorZip();
void testPassthruVectorTar();
void testPassthruVectorGzip();
void testPassthruRasterZip();
void testPassthruRasterTar();
void testPassthruRasterGzip();
// test both "Basic Scan" and "Full scan" for .zip files
void testZipItemRaster();
void testTarItemRaster();
void testZipItemVector();
void testTarItemVector();
void testZipItemAll();
void testTarItemAll();
// test that styles are loaded from .qml files outside zip files
void testZipItemVectorTransparency();
void testGZipItemVectorTransparency();
void testTarItemVectorTransparency();
void testGzipItemVectorTransparency();
void testZipItemRasterTransparency();
void testGZipItemRasterTransparency();
void testTarItemRasterTransparency();
void testGzipItemRasterTransparency();
//make sure items inside subfolders can be read
void testZipItemSubfolder();
void testTarItemSubfolder();
//make sure .vrt items are loaded by proper provider (gdal/ogr)
void testZipItemVRT();
};
Expand Down Expand Up @@ -225,6 +234,21 @@ int TestZipLayer::getLayerTransparency( QString myFileName, QString myProviderKe
return myTransparency;
}

bool TestZipLayer::testZipItemTransparency( QString myFileName, QString myProviderKey, int myTarget )
{
int myTransparency;
foreach( QString s, mScanZipSettings )
{
myTransparency = getLayerTransparency( myFileName, myProviderKey, s );
if ( myTransparency != myTarget )
{
QWARN( QString( "Transparency of %1 is %2, should be %3" ).arg( myFileName ).arg( myTransparency ).arg( myTarget ).toLocal8Bit().data() );
return false;
}
}
return true;
}


// slots
void TestZipLayer::initTestCase()
Expand Down Expand Up @@ -257,7 +281,6 @@ void TestZipLayer::cleanupTestCase()
settings.setValue( "/qgis/scanZipInBrowser", mScanZipSetting );
}


void TestZipLayer::testPassthruVectorZip()
{
QSettings settings;
Expand All @@ -275,6 +298,21 @@ void TestZipLayer::testPassthruVectorZip()
}
}

void TestZipLayer::testPassthruVectorTar()
{
QSettings settings;
QString myFileName = mDataDir + "points2.tar";
#if GDAL_VERSION_NUM < 1800
QSKIP( "This test requires GDAL >= 1.8", SkipSingle );
#endif
foreach( QString s, mScanZipSettings )
{
settings.setValue( "/qgis/scanZipInBrowser", s );
QVERIFY( s == settings.value( "/qgis/scanZipInBrowser" ).toString() );
QVERIFY( testZipItemPassthru( myFileName, "ogr" ) );
}
}

void TestZipLayer::testPassthruVectorGzip()
{
QSettings settings;
Expand All @@ -300,6 +338,20 @@ void TestZipLayer::testPassthruRasterZip()
}
}

void TestZipLayer::testPassthruRasterTar()
{
#if GDAL_VERSION_NUM < 1800
QSKIP( "This test requires GDAL >= 1.8", SkipSingle );
#endif
QSettings settings;
foreach( QString s, mScanZipSettings )
{
settings.setValue( "/qgis/scanZipInBrowser", s );
QVERIFY( s == settings.value( "/qgis/scanZipInBrowser" ).toString() );
QVERIFY( testZipItemPassthru( mDataDir + "landsat_b1.tar", "gdal" ) );
}
}

void TestZipLayer::testPassthruRasterGzip()
{
QSettings settings;
Expand All @@ -322,6 +374,20 @@ void TestZipLayer::testZipItemRaster()
}
}

void TestZipLayer::testTarItemRaster()
{
#if GDAL_VERSION_NUM < 1800
QSKIP( "This test requires GDAL >= 1.8", SkipSingle );
#endif
QSettings settings;
foreach( QString s, mScanZipSettings )
{
settings.setValue( "/qgis/scanZipInBrowser", s );
QVERIFY( s == settings.value( "/qgis/scanZipInBrowser" ).toString() );
QVERIFY( testZipItem( mDataDir + "testtar.tgz", "landsat_b1.tif" ) );
}
}

void TestZipLayer::testZipItemVector()
{
QSettings settings;
Expand All @@ -333,6 +399,20 @@ void TestZipLayer::testZipItemVector()
}
}

void TestZipLayer::testTarItemVector()
{
#if GDAL_VERSION_NUM < 1800
QSKIP( "This test requires GDAL >= 1.8", SkipSingle );
#endif
QSettings settings;
foreach( QString s, mScanZipSettings )
{
settings.setValue( "/qgis/scanZipInBrowser", s );
QVERIFY( s == settings.value( "/qgis/scanZipInBrowser" ).toString() );
QVERIFY( testZipItem( mDataDir + "testtar.tgz", "points.shp" ) );
}
}

void TestZipLayer::testZipItemAll()
{
// test file contains invalid items (tmp1.tif, tmp1.txt and tmp1.xml)
Expand All @@ -345,65 +425,78 @@ void TestZipLayer::testZipItemAll()
QVERIFY( testZipItem( mDataDir + "testzip.zip", "" ) );
}

void TestZipLayer::testTarItemAll()
{
#if GDAL_VERSION_NUM < 1800
QSKIP( "This test requires GDAL >= 1.8", SkipSingle );
#endif
QSettings settings;
settings.setValue( "/qgis/scanZipInBrowser", "full" );
QVERIFY( "full" == settings.value( "/qgis/scanZipInBrowser" ).toString() );
QVERIFY( testZipItem( mDataDir + "testtar.tgz", "" ) );
}

void TestZipLayer::testZipItemVectorTransparency()
{
#if GDAL_VERSION_NUM < 1800
QSKIP( "This test requires GDAL >= 1.8", SkipSingle );
#endif
int myTarget = 250;
int myTransparency;
foreach( QString s, mScanZipSettings )
{
myTransparency = getLayerTransparency( mDataDir + "points2.zip", "ogr", "basic" );
QVERIFY2(( myTransparency == myTarget ), QString( "Transparency is %1, should be %2" ).arg( myTransparency ).arg( myTarget ).toLocal8Bit().data() );
}
QVERIFY( testZipItemTransparency( mDataDir + "points2.zip", "ogr", 250 ) );
}

void TestZipLayer::testTarItemVectorTransparency()
{
#if GDAL_VERSION_NUM < 1800
QSKIP( "This test requires GDAL >= 1.8", SkipSingle );
#endif
QVERIFY( testZipItemTransparency( mDataDir + "points2.tar", "ogr", 250 ) );
}

void TestZipLayer::testGZipItemVectorTransparency()
void TestZipLayer::testGzipItemVectorTransparency()
{
#if GDAL_VERSION_NUM < 1700
QSKIP( "This test requires GDAL >= 1.7", SkipSingle );
#endif
int myTarget = 250;
int myTransparency;
foreach( QString s, mScanZipSettings )
{
myTransparency = getLayerTransparency( mDataDir + "points3.geojson.gz", "ogr", s );
QVERIFY2(( myTransparency == myTarget ), QString( "Transparency is %1, should be %2" ).arg( myTransparency ).arg( myTarget ).toLocal8Bit().data() );
}
QVERIFY( testZipItemTransparency( mDataDir + "points3.geojson.gz", "ogr", 250 ) );
}

void TestZipLayer::testZipItemRasterTransparency()
{
int myTarget = 250;
int myTransparency;
foreach( QString s, mScanZipSettings )
{
myTransparency = getLayerTransparency( mDataDir + "landsat_b1.zip", "gdal", s );
QVERIFY2(( myTransparency == myTarget ), QString( "Transparency is %1, should be %2" ).arg( myTransparency ).arg( myTarget ).toLocal8Bit().data() );
}
QVERIFY( testZipItemTransparency( mDataDir + "landsat_b1.zip", "gdal", 250 ) );
}

void TestZipLayer::testGZipItemRasterTransparency()
void TestZipLayer::testTarItemRasterTransparency()
{
int myTarget = 250;
int myTransparency;
#if GDAL_VERSION_NUM < 1800
QSKIP( "This test requires GDAL >= 1.8", SkipSingle );
#endif
QVERIFY( testZipItemTransparency( mDataDir + "landsat_b1.tar", "gdal", 250 ) );
}

void TestZipLayer::testGzipItemRasterTransparency()
{
QVERIFY( testZipItemTransparency( mDataDir + "landsat_b1.tif.gz", "gdal", 250 ) );
}

void TestZipLayer::testZipItemSubfolder()
{
QSettings settings;
foreach( QString s, mScanZipSettings )
{
myTransparency = getLayerTransparency( mDataDir + "landsat_b1.tif.gz", "gdal", s );
QVERIFY2(( myTransparency == myTarget ), QString( "Transparency is %1, should be %2" ).arg( myTransparency ).arg( myTarget ).toLocal8Bit().data() );
settings.setValue( "/qgis/scanZipInBrowser", s );
QVERIFY( s == settings.value( "/qgis/scanZipInBrowser" ).toString() );
QVERIFY( testZipItem( mDataDir + "testzip.zip", "folder/folder2/landsat_b2.tif" ) );
}
}

void TestZipLayer::testZipItemSubfolder()
void TestZipLayer::testTarItemSubfolder()
{
QSettings settings;
foreach( QString s, mScanZipSettings )
{
settings.setValue( "/qgis/scanZipInBrowser", s );
QVERIFY( s == settings.value( "/qgis/scanZipInBrowser" ).toString() );
QVERIFY( testZipItem( mDataDir + "testzip.zip", "folder/folder2/landsat_b2.tif" ) );
QVERIFY( testZipItem( mDataDir + "testtar.tgz", "folder/folder2/landsat_b2.tif" ) );
}
}

Expand Down
Binary file added tests/testdata/landsat_b1.tar
View file
Binary file not shown.
Binary file added tests/testdata/points2.tar
View file
Binary file not shown.
Binary file added tests/testdata/testtar.tgz
View file
Binary file not shown.