Skip to content

Commit

Permalink
Substring working - tests not in place yet
Browse files Browse the repository at this point in the history
  • Loading branch information
ccrook committed Apr 30, 2013
1 parent 1fb1c9c commit ea8617d
Show file tree
Hide file tree
Showing 3 changed files with 310 additions and 2 deletions.
281 changes: 281 additions & 0 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

#include "qgsapplication.h"
#include "qgsdataprovider.h"
#include "qgsexpression.h"
#include "qgsfeature.h"
#include "qgsfield.h"
#include "qgsgeometry.h"
Expand Down Expand Up @@ -60,6 +61,8 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
, mWktHasZM( false )
, mWktHasPrefix( false )
, mXyDms( false )
, mSubsetString("")
, mSubsetExpression( 0 )
, mMaxInvalidLines( 50 )
, mShowInvalidLines( true )
, mCrs()
Expand Down Expand Up @@ -507,6 +510,12 @@ QgsDelimitedTextProvider::~QgsDelimitedTextProvider()
delete mFile;
mFile = 0;
}

if( mSubsetExpression )
{
delete mSubsetExpression;
mSubsetExpression = 0;
}
}


Expand Down Expand Up @@ -597,6 +606,278 @@ void QgsDelimitedTextProvider::reportErrors( QStringList messages )
}
}

//

bool QgsDelimitedTextProvider::setSubsetString( QString subset, bool updateFeatureCount )
{

bool valid = true;

// If there is a new subset string then encode it..

QgsExpression *expression = 0;
if( ! subset.isEmpty() )
{

expression = new QgsExpression( subset );
QString error;
if( expression->hasParserError())
{
error = expression->parserErrorString();
}
else
{
expression->prepare( fields() );
if( expression->hasEvalError() )
{
error = expression->evalErrorString();
}
}
if( ! error.isEmpty() )
{
valid = false;
delete expression;
expression = 0;
QString tag( "DelimitedText" );
QgsMessageLog::logMessage( tr( "Invalid subset string %1 for %2" ).arg(subset).arg( mFile->fileName() ), tag );
}
}


// if the expression is valid, then reset the subset string and data source Uri

if( valid )
{

if( mSubsetExpression ) delete mSubsetExpression;
mSubsetString = subset;
mSubsetExpression = expression;

// Encode the subset string into the data source URI.

QUrl url = QUrl::fromEncoded( dataSourceUri().toAscii() );
if( url.hasQueryItem("subset")) url.removeAllQueryItems("subset");
if( ! subset.isEmpty()) url.addQueryItem("subset",subset);
setDataSourceUri( QString::fromAscii( url.toEncoded() ) );

// Update the feature count and extents if requested
if( updateFeatureCount )
{
resetDataSummary();
}
}

return valid;
}

void QgsDelimitedTextProvider::resetDataSummary()
{
QgsFeatureIterator fi = getFeatures(QgsFeatureRequest());
mNumberFeatures = 0;
mExtent = QgsRectangle();
QgsFeature f;
while( fi.nextFeature(f))
{
if( mGeometryType != QGis::NoGeometry )
{
if( mNumberFeatures == 0 )
{
mExtent = f.geometry()->boundingBox();
}
else
{
QgsRectangle bbox( f.geometry()->boundingBox());
mExtent.combineExtentWith( &bbox);
}
}
mNumberFeatures++;
}
}


bool QgsDelimitedTextProvider::nextFeature( QgsFeature& feature, QgsDelimitedTextFile *file, const QgsFeatureRequest& request )
{
QStringList tokens;
while ( true )
{
// before we do anything else, assume that there's something wrong with
// the feature
feature.setValid( false );
QgsDelimitedTextFile::Status status = file->nextRecord( tokens );
if ( status == QgsDelimitedTextFile::RecordEOF ) break;
if ( status != QgsDelimitedTextFile::RecordOk ) continue;

int fid = file->recordLineNumber();
if( request.filterType() == QgsFeatureRequest::FilterFid && fid != request.filterFid()) continue;
if ( recordIsEmpty( tokens ) ) continue;

while ( tokens.size() < mFieldCount )
tokens.append( QString::null );

QgsGeometry *geom = 0;

if ( mWktFieldIndex >= 0 )
{
geom = loadGeometryWkt( tokens, request );
}
else if ( mXFieldIndex >= 0 && mYFieldIndex >= 0 )
{
geom = loadGeometryXY( tokens,request );
}

if ( !geom && mWkbType != QGis::WKBNoGeometry )
{
// Already dealt with invalid lines in provider - no need to repeat
// removed code (CC 2013-04-13) ...
// mInvalidLines << line;
// In any case it may be a valid line that is excluded because of
// bounds check...
continue;
}

// At this point the current feature values are valid

feature.setValid( true );
feature.setFields( &attributeFields ); // allow name-based attribute lookups
feature.setFeatureId( fid );
feature.initAttributes( attributeFields.count() );

if ( geom )
feature.setGeometry( geom );

// If we have subset expression, then ened attributes
if ( ! mSubsetExpression && request.flags() & QgsFeatureRequest::SubsetOfAttributes )
{
const QgsAttributeList& attrs = request.subsetOfAttributes();
for ( QgsAttributeList::const_iterator i = attrs.begin(); i != attrs.end(); ++i )
{
int fieldIdx = *i;
fetchAttribute( feature, fieldIdx, tokens );
}
}
else
{
for ( int idx = 0; idx < attributeFields.count(); ++idx )
fetchAttribute( feature, idx, tokens );
}

// Are we using a subset expression, if so try and evaluate
// and accept result if passes.

if( mSubsetExpression )
{
QVariant isOk = mSubsetExpression->evaluate( &feature );
if( mSubsetExpression->hasEvalError() ) continue;
if( ! isOk.toBool() ) continue;
}

// We have a good line, so return
return true;

} // !mStream->atEnd()

return false;
}


QgsGeometry* QgsDelimitedTextProvider::loadGeometryWkt( const QStringList& tokens, const QgsFeatureRequest& request )
{
QgsGeometry* geom = 0;
QString sWkt = tokens[mWktFieldIndex];

geom = geomFromWkt( sWkt );

if ( geom && geom->type() != mGeometryType )
{
delete geom;
geom = 0;
}
if ( geom && !boundsCheck( geom, request ) )
{
delete geom;
geom = 0;
}
return geom;
}


QgsGeometry* QgsDelimitedTextProvider::loadGeometryXY( const QStringList& tokens, const QgsFeatureRequest& request )
{
QString sX = tokens[mXFieldIndex];
QString sY = tokens[mYFieldIndex];
QgsPoint pt;
bool ok = pointFromXY( sX, sY, pt );

if ( ok && boundsCheck( pt, request ) )
{
return QgsGeometry::fromPoint( pt );
}
return 0;
}

/**
* Check to see if the point is within the selection rectangle
*/
bool QgsDelimitedTextProvider::boundsCheck( const QgsPoint &pt, const QgsFeatureRequest& request )
{
// no selection rectangle or geometry => always in the bounds
if ( request.filterType() != QgsFeatureRequest::FilterRect || ( request.flags() & QgsFeatureRequest::NoGeometry ) )
return true;

return request.filterRect().contains( pt );
}

/**
* Check to see if the geometry is within the selection rectangle
*/
bool QgsDelimitedTextProvider::boundsCheck( QgsGeometry *geom, const QgsFeatureRequest& request )
{
// no selection rectangle or geometry => always in the bounds
if ( request.filterType() != QgsFeatureRequest::FilterRect || ( request.flags() & QgsFeatureRequest::NoGeometry ) )
return true;

if ( request.flags() & QgsFeatureRequest::ExactIntersect )
return geom->intersects( request.filterRect() );
else
return geom->boundingBox().intersects( request.filterRect() );
}


void QgsDelimitedTextProvider::fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens )
{
if( fieldIdx < 0 || fieldIdx >= attributeColumns.count()) return;
int column = attributeColumns[fieldIdx];
if( column < 0 || column >= tokens.count()) return;
const QString &value = tokens[column];
QVariant val;
switch ( attributeFields[fieldIdx].type() )
{
case QVariant::Int:
if ( value.isEmpty() )
val = QVariant( attributeFields[fieldIdx].type() );
else
val = QVariant( value );
break;
case QVariant::Double:
if ( value.isEmpty() )
{
val = QVariant( attributeFields[fieldIdx].type() );
}
else if ( mDecimalPoint.isEmpty() )
{
val = QVariant( value.toDouble() );
}
else
{
val = QVariant( QString( value ).replace( mDecimalPoint, "." ).toDouble() );
}
break;
default:
val = QVariant( value );
break;
}
feature.setAttribute( fieldIdx, val );
}

void QgsDelimitedTextProvider::resetDataSummary()
{
Expand Down
25 changes: 25 additions & 0 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class QTextStream;

class QgsDelimitedTextFeatureIterator;
class QgsDelimitedTextFile;
class QgsExpression;

/**
\class QgsDelimitedTextProvider
Expand Down Expand Up @@ -142,7 +143,28 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
bool isValid();

virtual QgsCoordinateReferenceSystem crs();
/**
* Set the subset string used to create a subset of features in
* the layer.
*/
virtual bool setSubsetString( QString subset, bool updateFeatureCount = true );

/**
* provider supports setting of subset strings
*/
virtual bool supportsSubsetString() { return true; }

/**
* Returns the subset definition string (typically sql) currently in
* use by the layer and used by the provider to limit the feature set.
* Must be overridden in the dataprovider, otherwise returns a null
* QString.
*/
virtual QString subsetString()
{
return mSubsetString;
}
/* new functions */

/**
Expand Down Expand Up @@ -223,6 +245,9 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
QString mDecimalPoint;
bool mXyDms;

QString mSubsetString;
QgsExpression *mSubsetExpression;

//! Storage for any lines in the file that couldn't be loaded
int mMaxInvalidLines;
int mNExtraInvalidLines;
Expand Down
6 changes: 4 additions & 2 deletions tests/src/python/test_qgsdelimitedtextprovider.py
Original file line number Diff line number Diff line change
Expand Up @@ -1391,13 +1391,14 @@ def test_019_latin1_encoded_file(self):
runTest(description,wanted,filename,requests,**params)



def test_030_filter_rect_xy(self):
description='Filter extents on XY layer'
filename='testextpt.txt'
params={'yField': 'y', 'delimiter': '|', 'type': 'csv', 'xField': 'x'}
requests=[
{'extents': [10, 30, 30, 50]},
{'exact': 1, 'extents': [10, 30, 30, 50]},
{'extents': [10, 30, 30, 50], 'exact': 1},
{'extents': [110, 130, 130, 150]}]
if rebuildTests:
createTest(description,filename,requests,**params)
Expand Down Expand Up @@ -1449,7 +1450,7 @@ def test_031_filter_rect_wkt(self):
params={'delimiter': '|', 'type': 'csv', 'wktField': 'wkt'}
requests=[
{'extents': [10, 30, 30, 50]},
{'exact': 1, 'extents': [10, 30, 30, 50]},
{'extents': [10, 30, 30, 50], 'exact': 1},
{'extents': [110, 130, 130, 150]}]
if rebuildTests:
createTest(description,filename,requests,**params)
Expand Down Expand Up @@ -1569,6 +1570,7 @@ def test_032_filter_fid(self):
runTest(description,wanted,filename,requests,**params)



def test_033_filter_attributes(self):
description='Filter on attributes'
filename='test.csv'
Expand Down

0 comments on commit ea8617d

Please sign in to comment.