140 changes: 115 additions & 25 deletions src/providers/delimitedtext/qgsdelimitedtextfile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <QFile>
#include <QDataStream>
#include <QTextStream>
#include <QFileSystemWatcher>
#include <QTextCodec>
#include <QStringList>
#include <QRegExp>
Expand All @@ -37,15 +38,20 @@ QgsDelimitedTextFile::QgsDelimitedTextFile( QString url ) :
mEncoding( "UTF-8" ),
mFile( 0 ),
mStream( 0 ),
mUseWatcher( true ),
mWatcher( 0 ),
mDefinitionValid( false ),
mUseHeader( true ),
mDiscardEmptyFields( false ),
mTrimFields( false ),
mSkipLines( 0 ),
mMaxFields( 0 ),
mMaxNameLength( 200 ), // Don't want field names to be too unweildy!
mLineNumber( 0 ),
mRecordLineNumber( 0 ),
mLineNumber( -1 ),
mRecordLineNumber( -1 ),
mRecordNumber( -1 ),
mHoldCurrentRecord( false ),
mMaxRecordNumber( -1 ),
mMaxFieldCount( 0 )
{
// The default type is CSV
Expand All @@ -71,6 +77,11 @@ void QgsDelimitedTextFile::close()
delete mFile;
mFile = 0;
}
if ( mWatcher )
{
delete mWatcher;
mWatcher = 0;
}
}

bool QgsDelimitedTextFile::open()
Expand All @@ -92,10 +103,25 @@ bool QgsDelimitedTextFile::open()
QTextCodec *codec = QTextCodec::codecForName( mEncoding.toAscii() );
mStream->setCodec( codec );
}
mMaxRecordNumber = -1;
mHoldCurrentRecord = false;
if ( mWatcher ) delete mWatcher;
if( mUseWatcher )
{
mWatcher = new QFileSystemWatcher( this );
mWatcher->addPath( mFileName );
connect( mWatcher, SIGNAL( fileChanged( QString ) ), this, SLOT( updateFile() ) );
}
}
return true;
}

void QgsDelimitedTextFile::updateFile()
{
close();
emit( fileUpdated() );
}

// Clear information based on current definition of file
void QgsDelimitedTextFile::resetDefinition()
{
Expand Down Expand Up @@ -126,6 +152,12 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
mEncoding = url.queryItemValue( "encoding" );
}

//
if ( url.hasQueryItem( "useWatcher" ) )
{
mUseWatcher = ! url.queryItemValue( "useWatcher" ).toUpper().startsWith( 'N' );;
}

// The default type is csv, to be consistent with the
// previous implementation (except that quoting should be handled properly)

Expand Down Expand Up @@ -181,7 +213,7 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
}
if ( url.hasQueryItem( "skipEmptyFields" ) )
{
mDiscardEmptyFields = ! url.queryItemValue( "skipEmptyFields" ).toUpper().startsWith( 'N' );;
mDiscardEmptyFields = ! url.queryItemValue( "skipEmptyFields" ).toUpper().startsWith( 'N' );
}
if ( url.hasQueryItem( "trimFields" ) )
{
Expand Down Expand Up @@ -231,6 +263,9 @@ QUrl QgsDelimitedTextFile::url()
{
url.addQueryItem( "encoding", mEncoding );
}

if( ! mUseWatcher ) url.addQueryItem( "useWatcher", "no");

url.addQueryItem( "type", type() );
if ( mType == DelimTypeRegexp )
{
Expand Down Expand Up @@ -277,6 +312,12 @@ void QgsDelimitedTextFile::setEncoding( QString encoding )
mEncoding = encoding;
}

void QgsDelimitedTextFile::setUseWatcher(bool useWatcher)
{
resetDefinition();
mUseWatcher = useWatcher;
}

QString QgsDelimitedTextFile::type()
{
if ( mType == DelimTypeWhitespace ) return QString( "whitespace" );
Expand Down Expand Up @@ -372,7 +413,7 @@ void QgsDelimitedTextFile::setDiscardEmptyFields( bool discardEmptyFields )

void QgsDelimitedTextFile::setFieldNames( const QStringList &names )
{
mFieldNames.empty();
mFieldNames.clear();
foreach ( QString name, names )
{
bool nameOk = true;
Expand Down Expand Up @@ -455,9 +496,47 @@ int QgsDelimitedTextFile::fieldIndex( QString name )

}

bool QgsDelimitedTextFile::setNextRecordId(long nextRecordId )
{
mHoldCurrentRecord = nextRecordId == mRecordLineNumber;
if( mHoldCurrentRecord ) return true;
return setNextLineNumber( nextRecordId );
}

QgsDelimitedTextFile::Status QgsDelimitedTextFile::nextRecord( QStringList &record )
{
return ( this->*mParser )( record );

record.clear();
Status status = RecordOk;

if( mHoldCurrentRecord )
{
mHoldCurrentRecord = false;
}
else
{
// Invalidate the record line number, in get EOF
mRecordLineNumber = -1;

// Find the first non-blank line to read
QString buffer;
status = nextLine( buffer, true );
if ( status != RecordOk ) return status;

mCurrentRecord.clear();
mRecordLineNumber = mLineNumber;
if ( mRecordNumber >= 0 )
{
mRecordNumber++;
if ( mRecordNumber > mMaxRecordNumber ) mMaxRecordNumber = mRecordNumber;
}
status = (this->*mParser )( buffer, mCurrentRecord );
}
if( status == RecordOk )
{
record.append(mCurrentRecord);
}
return status;
}


Expand All @@ -469,7 +548,8 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::reset()
// Reset the file pointer
mStream->seek( 0 );
mLineNumber = 0;
mRecordLineNumber = 0;
mRecordNumber = -1;
mRecordLineNumber = -1;

// Skip header lines
for ( int i = mSkipLines; i-- > 0; )
Expand All @@ -478,14 +558,15 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::reset()
mLineNumber++;
}
// Read the column names
Status result = RecordOk;
if ( mUseHeader )
{
QStringList names;
QgsDelimitedTextFile::Status result = nextRecord( names );
result = nextRecord( names );
setFieldNames( names );
return result;
}
return RecordOk;
if( result == RecordOk ) mRecordNumber = 0;
return result;
}

QgsDelimitedTextFile::Status QgsDelimitedTextFile::nextLine( QString &buffer, bool skipBlank )
Expand All @@ -509,6 +590,24 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::nextLine( QString &buffer, bo
return RecordEOF;
}

bool QgsDelimitedTextFile::setNextLineNumber( long nextLineNumber )
{
if ( ! mStream ) return false;
if ( mLineNumber > nextLineNumber-1 )
{
mRecordNumber = -1;
mStream->seek(0);
mLineNumber = 0;
}
QString buffer;
while( mLineNumber < nextLineNumber-1 )
{
if( nextLine(buffer,false) != RecordOk ) return false;
}
return true;

}

void QgsDelimitedTextFile::appendField( QStringList &record, QString field, bool quoted )
{
if ( mMaxFields > 0 && record.size() >= mMaxFields ) return;
Expand All @@ -522,16 +621,14 @@ void QgsDelimitedTextFile::appendField( QStringList &record, QString field, bool
if ( !( mDiscardEmptyFields && field.isEmpty() ) ) record.append( field );
}
// Keep track of maximum number of non-empty fields in a record
if ( record.size() > mMaxFieldCount && ! field.isEmpty() ) mMaxFieldCount = record.size();
if ( record.size() > mMaxFieldCount && ! field.isEmpty() )
{
mMaxFieldCount = record.size();
}
}

QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseRegexp( QStringList &fields )
QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseRegexp( QString &buffer, QStringList &fields )
{
fields.clear();
QString buffer;
Status status = nextLine( buffer, true );
if ( status != RecordOk ) return status;
mRecordLineNumber = mLineNumber;

// If match is anchored, then only interested in records which actually match
// and extract capture groups
Expand Down Expand Up @@ -586,16 +683,9 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseRegexp( QStringList &fie
return RecordOk;
}

QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseQuoted( QStringList &fields )
QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseQuoted( QString &buffer, QStringList &fields )
{
fields.clear();

// Find the first non-blank line to read
QString buffer;
Status status = nextLine( buffer, true );
if ( status != RecordOk ) return status;
mRecordLineNumber = mLineNumber;

Status status = RecordOk;
QString field; // String in which to accumulate next field
bool escaped = false; // Next char is escaped
bool quoted = false; // In quotes
Expand Down
65 changes: 55 additions & 10 deletions src/providers/delimitedtext/qgsdelimitedtextfile.h
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/***************************************************************************
qgsdelimitedtextparser.h - File for delimited text file
qgsdelimitedtextfile.h - File for delimited text file
-------------------
begin : 2004-02-27
copyright : (C) 2013 by Chris Crook
Expand All @@ -15,13 +15,17 @@
* *
***************************************************************************/

#ifndef QGSDELIMITEDTEXTFILE_H
#define QGSDELIMITEDTEXTFILE_H

#include <QStringList>
#include <QRegExp>
#include <QUrl>

class QgsFeature;
class QgsField;
class QFile;
class QFileSystemWatcher;
class QTextStream;


Expand Down Expand Up @@ -66,9 +70,11 @@ class QTextStream;
// on an abstract base class in order to facilitate changing the type of the parser easily
// eg in the provider dialog

class QgsDelimitedTextFile
class QgsDelimitedTextFile : public QObject
{

Q_OBJECT

public:

enum Status
Expand All @@ -84,7 +90,7 @@ class QgsDelimitedTextFile
{
DelimTypeWhitespace,
DelimTypeCSV,
DelimTypeRegexp,
DelimTypeRegexp
};

QgsDelimitedTextFile( QString url = QString() );
Expand Down Expand Up @@ -238,11 +244,22 @@ class QgsDelimitedTextFile
/** Return the line number of the start of the last record read
* @return linenumber The line number of the start of the record
*/
int recordLineNumber()
int recordId()
{
return mRecordLineNumber;
}

/** Set the index of the next record to return.
* @param nextRecordId The id to set the next record to
* @return valid True if the next record can be located
*/
bool setNextRecordId( long nextRecordId );

/** Number record number of records visited. After scanning the file
* serves as a record count.
* @return maxRecordNumber The maximum record number
*/
long recordCount() { return mMaxRecordNumber; }
/** Reset the file to reread from the beginning
*/
Status reset();
Expand Down Expand Up @@ -272,6 +289,22 @@ class QgsDelimitedTextFile
*/
static QString decodeChars( QString string );

/** Set to use or not use a QFileWatcher to notify of changes to the file
* @param useWatcher True to use a watcher, false otherwise
*/

void setUseWatcher( bool useWatcher );

signals:
/** Signal sent when the file is updated by another process
*/
void fileUpdated();

public slots:
/** Slot used by watcher to notify of file updates
*/
void updateFile();

private:

/** Open the file
Expand All @@ -290,29 +323,34 @@ class QgsDelimitedTextFile
void resetDefinition();

/** Parse reqular expression delimited fields */
Status parseRegexp( QStringList &fields );
Status parseRegexp( QString &buffer, QStringList &fields );
/** Parse quote delimited fields, where quote and escape are different */
Status parseQuoted( QStringList &fields );
Status parseQuoted( QString &buffer, QStringList &fields );

/** Return the next line from the data file. If skipBlank is true then
* blank lines will be skipped - this is for compatibility with previous
* delimited text parser implementation.
*/
Status nextLine( QString &buffer, bool skipBlank = false );

/** Set the next line to read from the file.
*/
bool setNextLineNumber( long nextLineNumber );

/** Utility routine to add a field to a record, accounting for trimming
* and discarding, and maximum field count
*/

void appendField( QStringList &record, QString field, bool quoted = false );

// Pointer to the currently selected parser
Status( QgsDelimitedTextFile::*mParser )( QStringList &fields );
Status( QgsDelimitedTextFile::*mParser )( QString &buffer, QStringList &fields );

QString mFileName;
QString mEncoding;
QFile *mFile;
QTextStream *mStream;
bool mUseWatcher;
QFileSystemWatcher *mWatcher;

// Parameters common to parsers
bool mDefinitionValid;
Expand All @@ -333,7 +371,14 @@ class QgsDelimitedTextFile

// Information extracted from file
QStringList mFieldNames;
int mLineNumber;
int mRecordLineNumber;
long mLineNumber;
long mRecordLineNumber;
long mRecordNumber;
QStringList mCurrentRecord;
bool mHoldCurrentRecord;
// Maximum number of record (ie maximum record number visited)
long mMaxRecordNumber;
int mMaxFieldCount;
};

#endif
498 changes: 373 additions & 125 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
View file

Large diffs are not rendered by default.

58 changes: 49 additions & 9 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,12 @@
* *
***************************************************************************/

#ifndef QGSDELIMITEDTEXTPROVIDER_H
#define QGSDELIMITEDTEXTPROVIDER_H

#include "qgsvectordataprovider.h"
#include "qgscoordinatereferencesystem.h"
#include "qgsdelimitedtextfile.h"

#include <QStringList>

Expand All @@ -29,8 +32,8 @@ class QFile;
class QTextStream;

class QgsDelimitedTextFeatureIterator;
class QgsDelimitedTextFile;
class QgsExpression;
class QgsSpatialIndex;

/**
\class QgsDelimitedTextProvider
Expand All @@ -47,7 +50,8 @@ class QgsExpression;
* Example uri = "/home/foo/delim.txt?delimiter=|"*
*
* For detailed information on the uri format see the QGSVectorLayer
* documentation.
* documentation. Note that the interpretation of the URI is split
* between QgsDelimitedTextFile and QgsDelimitedTextProvider.
*
*/
Expand All @@ -64,6 +68,13 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
static QRegExp WktPrefixRegexp;
static QRegExp CrdDmsRegexp;

enum GeomRepresentationType
{
GeomNone,
GeomAsXy,
GeomAsWkt
};

QgsDelimitedTextProvider( QString uri = QString() );

virtual ~QgsDelimitedTextProvider();
Expand Down Expand Up @@ -102,6 +113,10 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
*/
virtual int capabilities() const;

/** Creates a spatial index on the data
* @return indexCreated Returns true if a spatial index is created
*/
virtual bool createSpatialIndex();

/* Implementation of functions from QgsDataProvider */

Expand Down Expand Up @@ -186,23 +201,30 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
*/
bool boundsCheck( QgsGeometry *geom );

private slots:

void onFileUpdated();

private:

static QRegExp WktZMRegexp;
static QRegExp WktCrdRegexp;

void scanFile( bool buildIndexes );
void rescanFile();
void resetCachedSubset();
void resetIndexes();
void clearInvalidLines();
void recordInvalidLine( QString message );
void reportErrors( QStringList messages = QStringList() );
void reportErrors( QStringList messages = QStringList(), bool showDialog = true );
void resetStream();
bool recordIsEmpty( QStringList &record );
bool nextFeature( QgsFeature& feature, QgsDelimitedTextFile *file, const QgsFeatureRequest& request );
QgsGeometry* loadGeometryWkt( const QStringList& tokens, const QgsFeatureRequest& request );
QgsGeometry* loadGeometryXY( const QStringList& tokens, const QgsFeatureRequest& request );
bool boundsCheck( const QgsPoint &pt, const QgsFeatureRequest& request );
bool boundsCheck( QgsGeometry *geom, const QgsFeatureRequest& request );
bool nextFeature( QgsFeature& feature, QgsDelimitedTextFile *file, QgsDelimitedTextFeatureIterator *iterator );
QgsGeometry* loadGeometryWkt( const QStringList& tokens, QgsDelimitedTextFeatureIterator *iterator );
QgsGeometry* loadGeometryXY( const QStringList& tokens, QgsDelimitedTextFeatureIterator *iterator );
void fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens );
void resetDataSummary();
void setUriParameter( QString parameter, QString value );
bool setNextFeatureId( qint64 fid ) { return mFile->setNextRecordId( (long) fid ); }


QgsGeometry *geomFromWkt( QString &sWkt );
Expand All @@ -216,10 +238,15 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
QgsDelimitedTextFile *mFile;

// Fields
GeomRepresentationType mGeomRep;
QList<int> attributeColumns;
QgsFields attributeFields;

int mFieldCount; // Note: this includes field count for wkt field
QString mWktFieldName;
QString mXFieldName;
QString mYFieldName;

int mXFieldIndex;
int mYFieldIndex;
int mWktFieldIndex;
Expand All @@ -246,7 +273,12 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
bool mXyDms;

QString mSubsetString;
QString mCachedSubsetString;
QgsExpression *mSubsetExpression;
bool mBuildSubsetIndex;
QList<quintptr> mSubsetIndex;
bool mUseSubsetIndex;
bool mCachedUseSubsetIndex;

//! Storage for any lines in the file that couldn't be loaded
int mMaxInvalidLines;
Expand All @@ -270,6 +302,14 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
QGis::WkbType mWkbType;
QGis::GeometryType mGeometryType;

// Spatial index
bool mBuildSpatialIndex;
bool mUseSpatialIndex;
bool mCachedUseSpatialIndex;
QgsSpatialIndex *mSpatialIndex;

friend class QgsDelimitedTextFeatureIterator;
QgsDelimitedTextFeatureIterator* mActiveIterator;
};

#endif
1,628 changes: 253 additions & 1,375 deletions tests/src/python/test_qgsdelimitedtextprovider.py
View file

Large diffs are not rendered by default.

2,030 changes: 2,030 additions & 0 deletions tests/src/python/test_qgsdelimitedtextprovider_wanted.py
View file

Large diffs are not rendered by default.