250 changes: 123 additions & 127 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ QRegExp QgsDelimitedTextProvider::CrdDmsRegexp( "^\\s*(?:([-+nsew])\\s*)?(\\d{1,

QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
: QgsVectorDataProvider( uri )
, mLayerValid( false )
, mValid( false )
, mFile( 0 )
, mGeomRep( GeomNone )
, mFieldCount( 0 )
Expand Down Expand Up @@ -165,6 +167,29 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
}
}

QgsDelimitedTextProvider::~QgsDelimitedTextProvider()
{
if ( mActiveIterator )
mActiveIterator->close();

if ( mFile )
{
delete mFile;
mFile = 0;
}

if ( mSubsetExpression )
{
delete mSubsetExpression;
mSubsetExpression = 0;
}
if ( mSpatialIndex )
{
delete mSpatialIndex;
mSpatialIndex = 0;
}
}

QStringList QgsDelimitedTextProvider::readCsvtFieldTypes( QString filename, QString *message )
{
// Look for a file with the same name as the data file, but an extra 't' or 'T' at the end
Expand Down Expand Up @@ -241,15 +266,13 @@ QStringList QgsDelimitedTextProvider::readCsvtFieldTypes( QString filename, QStr

}


void QgsDelimitedTextProvider::resetCachedSubset()
{
mCachedSubsetString = QString();
mCachedUseSubsetIndex = false;
mCachedUseSpatialIndex = false;
}


void QgsDelimitedTextProvider::resetIndexes()
{
resetCachedSubset();
Expand All @@ -276,21 +299,23 @@ bool QgsDelimitedTextProvider::createSpatialIndex()
return true;
}

// buildIndexes parameter of scanFile is to allow for potential rescan - if using
// subset string then rescan follows this to determine subset extents etc.
// Done this way as subset requires fields to be defined, which they are not
// until initial file scan is complete.
//
// Although at this point the subset expression does not apply (if one is defined)
// we still consider a subset index, as this also applies for implicit subsets
// due to filtering on geometry validity.
// Really want to merge scanFile and rescan into single code. Currently the reason
// this is not done is that scanFile is done initially to create field names and, rescan
// file includes building subset expression and assumes field names/types are already
// defined. Merging would not only make code a lot cleaner, but would also avoid
// double scan when loading a file with a subset expression.

// buildIndexes parameter of scanFile is set to false when we know we will be
// immediately rescanning (when the file is loaded and then the subset expression is
// set)

void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
{
QStringList messages;

// assume the layer is invalid until proven otherwise

mLayerValid = false;
mValid = false;

clearInvalidLines();
Expand Down Expand Up @@ -635,34 +660,105 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
mUseSpatialIndex = buildSpatialIndex;

mValid = mGeometryType != QGis::UnknownGeometry;
mLayerValid = mValid;

// If it is valid, then watch for changes to the file
connect( mFile, SIGNAL( fileUpdated() ), this, SLOT( onFileUpdated() ) );


}

QgsDelimitedTextProvider::~QgsDelimitedTextProvider()
// rescanFile. Called if something has changed file definition, such as
// selecting a subset, the file has been changed by another program, etc

void QgsDelimitedTextProvider::rescanFile()
{
if ( mActiveIterator )
mActiveIterator->close();
resetIndexes();

if ( mFile )
bool buildSpatialIndex = mSpatialIndex != 0;
bool buildSubsetIndex = mBuildSubsetIndex && ( mSubsetExpression || mGeomRep != GeomNone );

// In case file has been rewritten check that it is still valid

mValid = mLayerValid && mFile->isValid();
if ( ! mValid ) return;

// Open the file and get number of rows, etc. We assume that the
// file has a header row and process accordingly. Caller should make
// sure that the delimited file is properly formed.

QStringList messages;

if ( mGeomRep == GeomAsWkt )
{
delete mFile;
mFile = 0;
mWktFieldIndex = mFile->fieldIndex( mWktFieldName );
if ( mWktFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Wkt" ).arg( mWktFieldName ) );
}
}
else if ( mGeomRep == GeomAsXy )
{
mXFieldIndex = mFile->fieldIndex( mXFieldName );
mYFieldIndex = mFile->fieldIndex( mYFieldName );
if ( mXFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "X" ).arg( mWktFieldName ) );
}
if ( mYFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Y" ).arg( mWktFieldName ) );
}
}
if ( messages.size() > 0 )
{
reportErrors( messages, false );
QgsDebugMsg( "Delimited text source invalid on rescan - missing geometry fields" );
mValid = false;
}

if ( mSubsetExpression )
// Reset the field columns

for ( int i = 0; i < attributeFields.size(); i++ )
{
delete mSubsetExpression;
mSubsetExpression = 0;
attributeColumns[i] = mFile->fieldIndex( attributeFields[i].name() );
}
if ( mSpatialIndex )

// Scan through the features in the file

mSubsetIndex.clear();
mUseSubsetIndex = false;
QgsFeatureIterator fi = getFeatures( QgsFeatureRequest() );
mNumberFeatures = 0;
mExtent = QgsRectangle();
QgsFeature f;
while ( fi.nextFeature( f ) )
{
delete mSpatialIndex;
mSpatialIndex = 0;
if ( mGeometryType != QGis::NoGeometry )
{
if ( mNumberFeatures == 0 )
{
mExtent = f.geometry()->boundingBox();
}
else
{
QgsRectangle bbox( f.geometry()->boundingBox() );
mExtent.combineExtentWith( &bbox );
}
if ( buildSpatialIndex ) mSpatialIndex->insertFeature( f );
}
if ( buildSubsetIndex ) mSubsetIndex.append(( quintptr ) f.id() );
mNumberFeatures++;
}
if ( buildSubsetIndex )
{
long recordCount = mFile->recordCount();
recordCount -= recordCount / SUBSET_ID_THRESHOLD_FACTOR;
mUseSubsetIndex = recordCount < mSubsetIndex.size();
if ( ! mUseSubsetIndex ) mSubsetIndex.clear();
}

mUseSpatialIndex = buildSpatialIndex;
}

QgsGeometry *QgsDelimitedTextProvider::geomFromWkt( QString &sWkt )
Expand All @@ -688,7 +784,6 @@ QgsGeometry *QgsDelimitedTextProvider::geomFromWkt( QString &sWkt )
return geom;
}


double QgsDelimitedTextProvider::dmsStringToDouble( const QString &sX, bool *xOk )
{
static QString negative( "swSW-" );
Expand Down Expand Up @@ -725,7 +820,6 @@ double QgsDelimitedTextProvider::dmsStringToDouble( const QString &sX, bool *xOk
return x;
}


bool QgsDelimitedTextProvider::pointFromXY( QString &sX, QString &sY, QgsPoint &pt )
{
if ( ! mDecimalPoint.isEmpty() )
Expand Down Expand Up @@ -756,8 +850,6 @@ bool QgsDelimitedTextProvider::pointFromXY( QString &sX, QString &sY, QgsPoint &
return false;
}



QString QgsDelimitedTextProvider::storageType() const
{
return "Delimited text file";
Expand All @@ -770,6 +862,9 @@ void QgsDelimitedTextProvider::resetStream()

QgsFeatureIterator QgsDelimitedTextProvider::getFeatures( const QgsFeatureRequest& request )
{
// If the file has become invalid, check that it is still invalid.
if( mLayerValid && ! mValid ) rescanFile();

return QgsFeatureIterator( new QgsDelimitedTextFeatureIterator( this, request ) );
}

Expand Down Expand Up @@ -845,8 +940,6 @@ void QgsDelimitedTextProvider::reportErrors( QStringList messages , bool showDia
}
}

//

bool QgsDelimitedTextProvider::setSubsetString( QString subset, bool updateFeatureCount )
{
// If not changing string, then oll ok, nothing to do
Expand Down Expand Up @@ -953,100 +1046,6 @@ void QgsDelimitedTextProvider::setUriParameter( QString parameter, QString value
setDataSourceUri( QString::fromAscii( url.toEncoded() ) );
}

// rescanFile. Called if something has changed file definition, such as
// selecting a subset, the file has been changed by another program, etc

void QgsDelimitedTextProvider::rescanFile()
{
resetIndexes();

bool buildSpatialIndex = mSpatialIndex != 0;
bool buildSubsetIndex = mBuildSubsetIndex && ( mSubsetExpression || mGeomRep != GeomNone );

// In case file has been rewritten, check that required fields are still
// valid

mValid = mFile->isValid();
if ( ! mValid ) return;

// Open the file and get number of rows, etc. We assume that the
// file has a header row and process accordingly. Caller should make
// sure that the delimited file is properly formed.

QStringList messages;

if ( mGeomRep == GeomAsWkt )
{
mWktFieldIndex = mFile->fieldIndex( mWktFieldName );
if ( mWktFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Wkt" ).arg( mWktFieldName ) );
}
}
else if ( mGeomRep == GeomAsXy )
{
mXFieldIndex = mFile->fieldIndex( mXFieldName );
mYFieldIndex = mFile->fieldIndex( mYFieldName );
if ( mXFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "X" ).arg( mWktFieldName ) );
}
if ( mYFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Y" ).arg( mWktFieldName ) );
}
}
if ( messages.size() > 0 )
{
reportErrors( messages, false );
QgsDebugMsg( "Delimited text source invalid on rescan - missing geometry fields" );
mValid = false;
}

// Reset the field columns

for ( int i = 0; i < attributeFields.size(); i++ )
{
attributeColumns[i] = mFile->fieldIndex( attributeFields[i].name() );
}

// Scan through the features in the file

mSubsetIndex.clear();
mUseSubsetIndex = false;
QgsFeatureIterator fi = getFeatures( QgsFeatureRequest() );
mNumberFeatures = 0;
mExtent = QgsRectangle();
QgsFeature f;
while ( fi.nextFeature( f ) )
{
if ( mGeometryType != QGis::NoGeometry )
{
if ( mNumberFeatures == 0 )
{
mExtent = f.geometry()->boundingBox();
}
else
{
QgsRectangle bbox( f.geometry()->boundingBox() );
mExtent.combineExtentWith( &bbox );
}
if ( buildSpatialIndex ) mSpatialIndex->insertFeature( f );
}
if ( buildSubsetIndex ) mSubsetIndex.append(( quintptr ) f.id() );
mNumberFeatures++;
}
if ( buildSubsetIndex )
{
long recordCount = mFile->recordCount();
recordCount -= recordCount / SUBSET_ID_THRESHOLD_FACTOR;
mUseSubsetIndex = recordCount < mSubsetIndex.size();
if ( ! mUseSubsetIndex ) mSubsetIndex.clear();
}

mUseSpatialIndex = buildSpatialIndex;
}

void QgsDelimitedTextProvider::onFileUpdated()
{
QStringList messages;
Expand Down Expand Up @@ -1078,6 +1077,7 @@ bool QgsDelimitedTextProvider::nextFeature( QgsFeature& feature, QgsDelimitedTex
QgsDelimitedTextFile::Status status = file->nextRecord( tokens );
if ( status == QgsDelimitedTextFile::RecordEOF ) break;
if ( status != QgsDelimitedTextFile::RecordOk ) continue;

// We ignore empty records, such as added randomly by spreadsheets

if ( recordIsEmpty( tokens ) ) continue;
Expand Down Expand Up @@ -1153,7 +1153,6 @@ bool QgsDelimitedTextProvider::nextFeature( QgsFeature& feature, QgsDelimitedTex
return false;
}


QgsGeometry* QgsDelimitedTextProvider::loadGeometryWkt( const QStringList& tokens, QgsDelimitedTextFeatureIterator *iterator )
{
QgsGeometry* geom = 0;
Expand All @@ -1174,7 +1173,6 @@ QgsGeometry* QgsDelimitedTextProvider::loadGeometryWkt( const QStringList& token
return geom;
}


QgsGeometry* QgsDelimitedTextProvider::loadGeometryXY( const QStringList& tokens, QgsDelimitedTextFeatureIterator *iterator )
{
QString sX = tokens[mXFieldIndex];
Expand All @@ -1189,7 +1187,6 @@ QgsGeometry* QgsDelimitedTextProvider::loadGeometryXY( const QStringList& tokens
return 0;
}


void QgsDelimitedTextProvider::fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens )
{
if ( fieldIdx < 0 || fieldIdx >= attributeColumns.count() ) return;
Expand Down Expand Up @@ -1272,10 +1269,9 @@ const QgsFields & QgsDelimitedTextProvider::fields() const

bool QgsDelimitedTextProvider::isValid()
{
return mValid;
return mLayerValid;
}


int QgsDelimitedTextProvider::capabilities() const
{
return SelectAtId | CreateSpatialIndex;
Expand Down
9 changes: 5 additions & 4 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,8 +239,11 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
bool pointFromXY( QString &sX, QString &sY, QgsPoint &point );
double dmsStringToDouble( const QString &sX, bool *xOk );


QString mUri;
// mLayerValid defines whether the layer has been loaded as a valid layer
bool mLayerValid;
// mValid defines whether the layer is currently valid (may differ from
// mLayerValid if the file has been rewritten)
bool mValid;

//! Text file
QgsDelimitedTextFile *mFile;
Expand Down Expand Up @@ -271,8 +274,6 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
//! Layer extent
QgsRectangle mExtent;

bool mValid;

int mGeomType;

long mNumberFeatures;
Expand Down
26 changes: 19 additions & 7 deletions tests/src/python/test_qgsdelimitedtextprovider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import re
import tempfile
import inspect
import time
import test_qgsdelimitedtextprovider_wanted as want

rebuildTests = 'REBUILD_DELIMITED_TEXT_TESTS' in os.environ;
Expand Down Expand Up @@ -163,10 +164,11 @@ def delimitedTextData( testname, filename, requests, verbose, **params ):
data = {}
if layer.isValid():
for nr,r in enumerate(requests):
if verbose:
print "Processing request",nr+1,repr(r)
if verbose: print "Processing request",nr+1,repr(r)
if callable(r):
r( layer )
if verbose: print "Request function executed"
if callable(r):
continue
rfields,rtypes, rdata = layerData(layer,r,nr*1000)
if len(rfields) > len(fields):
Expand All @@ -175,6 +177,8 @@ def delimitedTextData( testname, filename, requests, verbose, **params ):
data.update(rdata)
if not rdata:
log.append("Request "+str(nr)+" did not return any data")
if verbose:
print "Request returned",len(rdata.keys()),"features"
for msg in logger.messages():
filelogname = 'temp_file' if 'tmp' in filename.lower() else filename
msg = re.sub(r'file\s+.*'+re.escape(filename),'file '+filelogname,msg)
Expand Down Expand Up @@ -534,34 +538,42 @@ def test_029_file_watcher(self):
(filehandle,filename) = tempfile.mkstemp()
with os.fdopen(filehandle,"w") as f:
f.write("id,name\n1,rabbit\n2,pooh\n")
QCoreApplication.instance().processEvents()
def updatefile1( layer ):
def appendfile( layer ):
with file(filename,'a') as f:
f.write('3,tigger\n')
# print "Appended to file - sleeping"
time.sleep(1);
QCoreApplication.instance().processEvents()
def updatefile2( layer ):
def rewritefile( layer ):
with file(filename,'w') as f:
f.write("name,size,id\ntoad,small,5\nmole,medium,6\nbadger,big,7\n")
# print "Rewritten file - sleeping"
time.sleep(1);
QCoreApplication.instance().processEvents()
def deletefile( layer ):
os.remove(filename)
# print "Deleted file - sleeping"
time.sleep(1);
QCoreApplication.instance().processEvents()
params={'geomType': 'none', 'type': 'csv', 'watchFile' : 'yes' }
requests=[
{'fid': 3},
{},
{'fid': 7},
updatefile1,
appendfile,
{'fid': 3},
{'fid': 4},
{},
{'fid': 7},
updatefile2,
rewritefile,
{'fid': 2},
{},
{'fid': 7},
deletefile,
{'fid': 2},
{},
rewritefile,
{'fid': 2},
]
runTest(filename,requests,**params)

Expand Down
20 changes: 5 additions & 15 deletions tests/src/python/test_qgsdelimitedtextprovider_wanted.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1346,32 +1346,22 @@ def test_029_file_watcher():
'#fid': 4L,
'#geometry': 'None',
},
13002L: {
16002L: {
'id': u'5',
'description': u'toad',
'name': u'toad',
'#fid': 2L,
'#geometry': 'None',
},
14003L: {
'id': u'6',
'description': u'mole',
'name': u'mole',
'#fid': 3L,
'#geometry': 'None',
},
14004L: {
'id': u'7',
'description': u'badger',
'name': u'badger',
'#fid': 4L,
'#geometry': 'None',
},
}
wanted['log']=[
'Request 2 did not return any data',
'Request 7 did not return any data',
'Request 11 did not return any data',
'Request 13 did not return any data',
'Request 14 did not return any data',
u'Errors in file temp_file',
u'The file has been updated by another application - reloading',
u'Errors in file temp_file',
u'The file has been updated by another application - reloading',
u'Errors in file temp_file',
Expand Down