324 changes: 324 additions & 0 deletions src/core/qgsvectorlayerimport.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,324 @@
/***************************************************************************
qgsvectorlayerimport.cpp
vector layer importer
-------------------
begin : Thu Aug 25 2011
copyright : (C) 2011 by Giuseppe Sucameli
email : brush.tyler at gmail.com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#include "qgsfield.h"
#include "qgsfeature.h"
#include "qgsgeometry.h"
#include "qgslogger.h"
#include "qgscoordinatereferencesystem.h"
#include "qgsvectorlayerimport.h"
#include "qgsproviderregistry.h"

#include <QFile>
#include <QSettings>
#include <QFileInfo>
#include <QDir>
#include <QTextCodec>
#include <QTextStream>
#include <QSet>
#include <QMetaType>

#include <cassert>
#include <cstdlib> // size_t
#include <limits> // std::numeric_limits


#define FEATURE_BUFFER_SIZE 200

typedef QgsVectorLayerImport::ImportError createEmptyLayer_t(
const QString &uri,
const QgsFieldMap &fields,
QGis::WkbType geometryType,
const QgsCoordinateReferenceSystem *destCRS,
bool overwrite,
QMap<int, int> *oldToNewAttrIdx,
QString *errorMessage = 0,
const QMap<QString, QVariant> *options = 0
);


QgsVectorLayerImport::QgsVectorLayerImport(
const QString &uri,
const QString &providerKey,
const QgsFieldMap& fields,
QGis::WkbType geometryType,
const QgsCoordinateReferenceSystem* crs,
bool overwrite,
const QMap<QString, QVariant> *options )
{
mProvider = NULL;
QgsProviderRegistry * pReg = QgsProviderRegistry::instance();

QLibrary *myLib = pReg->providerLibrary( providerKey );
if ( !myLib )
{
mError = ErrInvalidProvider;
mErrorMessage = QObject::tr( "Unable to load %1 provider" ).arg( providerKey );
return;
}

createEmptyLayer_t * pCreateEmpty = ( createEmptyLayer_t * ) cast_to_fptr( myLib->resolve( "createEmptyLayer" ) );
if ( !pCreateEmpty )
{
delete myLib;
mError = ErrProviderUnsupportedFeature;
mErrorMessage = QObject::tr( "Provider %1 has no createEmptyLayer method" ).arg( providerKey );
return;
}

delete myLib;

// create an empty layer
QString errMsg;
mError = pCreateEmpty( uri, fields, geometryType, crs, overwrite, &mOldToNewAttrIdx, &errMsg, options );
if ( hasError() )
{
mErrorMessage = errMsg;
return;
}

QgsDebugMsg( "Created empty layer" );

QgsVectorDataProvider *vectorProvider = ( QgsVectorDataProvider* ) pReg->provider( providerKey, uri );
if ( !vectorProvider || !vectorProvider->isValid() )
{
mError = ErrInvalidLayer;
mErrorMessage = QObject::tr( "Loading of layer failed" );

if ( vectorProvider )
delete vectorProvider;
return;
}

mProvider = vectorProvider;
mError = NoError;
}

QgsVectorLayerImport::~QgsVectorLayerImport()
{
flushBuffer();

if ( mProvider )
delete mProvider;
}

QgsVectorLayerImport::ImportError QgsVectorLayerImport::hasError()
{
return mError;
}

QString QgsVectorLayerImport::errorMessage()
{
return mErrorMessage;
}

bool QgsVectorLayerImport::addFeature( QgsFeature& feat )
{
const QgsAttributeMap &attrs = feat.attributeMap();

QgsAttributeMap newAttrs;
for ( QgsAttributeMap::const_iterator it = attrs.begin(); it != attrs.end(); it++ )
{
if ( mOldToNewAttrIdx.contains( it.key() ) )
{
QgsDebugMsgLevel( QString( "moving field from pos %1 to %2" ).arg( it.key() ).arg( mOldToNewAttrIdx.value( it.key() ) ), 3 );
newAttrs.insert( mOldToNewAttrIdx.value( it.key() ), *it );
}
else
{
QgsDebugMsgLevel( QString( "added attr pos %1" ).arg( it.key() ), 3 );
newAttrs.insert( it.key(), *it );
}
}
feat.setAttributeMap( newAttrs );

mFeatureBuffer.append( feat );

if ( mFeatureBuffer.count() >= FEATURE_BUFFER_SIZE )
{
return flushBuffer();
}

return true;
}

bool QgsVectorLayerImport::flushBuffer()
{
if ( mFeatureBuffer.count() <= 0 )
return true;

if ( !mProvider->addFeatures( mFeatureBuffer ) )
{
mErrorMessage = QObject::tr( "Creation error for features from #%1 to #%2" )
.arg( mFeatureBuffer.first().id() )
.arg( mFeatureBuffer.last().id() );
mError = ErrFeatureWriteFailed;

mFeatureBuffer.clear();
QgsDebugMsg( mErrorMessage );
return false;
}

mFeatureBuffer.clear();
return true;
}


QgsVectorLayerImport::ImportError
QgsVectorLayerImport::importLayer( QgsVectorLayer* layer,
const QString& uri,
const QString& providerKey,
const QgsCoordinateReferenceSystem *destCRS,
bool onlySelected,
QString *errorMessage,
bool skipAttributeCreation,
QMap<QString, QVariant> *options )
{
const QgsCoordinateReferenceSystem* outputCRS;
QgsCoordinateTransform* ct = 0;
int shallTransform = false;

if ( layer == NULL )
{
return ErrInvalidLayer;
}

if ( destCRS && destCRS->isValid() )
{
// This means we should transform
outputCRS = destCRS;
shallTransform = true;
}
else
{
// This means we shouldn't transform, use source CRS as output (if defined)
outputCRS = &layer->crs();
}

QgsVectorLayerImport * writer =
new QgsVectorLayerImport( uri, providerKey, skipAttributeCreation ? QgsFieldMap() : layer->pendingFields(), layer->wkbType(), outputCRS, false, options );

// check whether file creation was successful
ImportError err = writer->hasError();
if ( err != NoError )
{
if ( errorMessage )
*errorMessage = writer->errorMessage();
delete writer;
return err;
}

if ( errorMessage )
{
errorMessage->clear();
}

QgsAttributeList allAttr = skipAttributeCreation ? QgsAttributeList() : layer->pendingAllAttributesList();
QgsFeature fet;

layer->select( allAttr, QgsRectangle(), layer->wkbType() != QGis::WKBNoGeometry );

const QgsFeatureIds& ids = layer->selectedFeaturesIds();

// Create our transform
if ( destCRS )
{
ct = new QgsCoordinateTransform( layer->crs(), *destCRS );
}

// Check for failure
if ( ct == NULL )
{
shallTransform = false;
}

int n = 0, errors = 0;

// write all features
while ( layer->nextFeature( fet ) )
{
if ( onlySelected && !ids.contains( fet.id() ) )
continue;

if ( shallTransform )
{
try
{
if ( fet.geometry() )
{
fet.geometry()->transform( *ct );
}
}
catch ( QgsCsException &e )
{
delete ct;
delete writer;

QString msg = QObject::tr( "Failed to transform a point while drawing a feature of type '%1'. Writing stopped. (Exception: %2)" )
.arg( fet.typeName() ).arg( e.what() );
QgsLogger::warning( msg );
if ( errorMessage )
*errorMessage = msg;

return ErrProjection;
}
}
if ( skipAttributeCreation )
{
fet.clearAttributeMap();
}
if ( !writer->addFeature( fet ) )
{
if ( writer->hasError() && errorMessage )
{
if ( errorMessage->isEmpty() )
{
*errorMessage = QObject::tr( "Feature write errors:" );
}
*errorMessage += "\n" + writer->errorMessage();
}
errors++;

if ( errors > 1000 )
{
if ( errorMessage )
{
*errorMessage += QObject::tr( "Stopping after %1 errors" ).arg( errors );
}

n = -1;
break;
}
}
n++;
}

delete writer;

if ( shallTransform )
{
delete ct;
}

if ( errors > 0 && errorMessage && n > 0 )
{
*errorMessage += QObject::tr( "\nOnly %1 of %2 features written." ).arg( n - errors ).arg( n );
}

return errors == 0 ? NoError : ErrFeatureWriteFailed;
}
104 changes: 104 additions & 0 deletions src/core/qgsvectorlayerimport.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/***************************************************************************
qgsvectorlayerimport.cpp
vector layer importer
-------------------
begin : Thu Aug 25 2011
copyright : (C) 2011 by Giuseppe Sucameli
email : brush.tyler at gmail.com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#ifndef _QGSVECTORLAYERIMPORT_H_
#define _QGSVECTORLAYERIMPORT_H_

#include "qgsvectordataprovider.h"
#include "qgsvectorlayer.h"

/** \ingroup core
* A convenience class for writing vector files to disk.
There are two possibilities how to use this class:
1. static call to QgsVectorFileWriter::writeAsShapefile(...) which saves the whole vector layer
2. create an instance of the class and issue calls to addFeature(...)
Currently supports only writing to shapefiles, but shouldn't be a problem to add capability
to support other OGR-writable formats.
*/
class CORE_EXPORT QgsVectorLayerImport
{
public:

enum ImportError
{
NoError = 0,
ErrDriverNotFound,
ErrCreateDataSource,
ErrCreateLayer,
ErrAttributeTypeUnsupported,
ErrAttributeCreationFailed,
ErrProjection,
ErrFeatureWriteFailed,
ErrInvalidLayer,
ErrInvalidProvider,
ErrProviderUnsupportedFeature,
ErrConnectionFailed
};


/** Write contents of vector layer to a different datasource */
static ImportError importLayer( QgsVectorLayer* layer,
const QString& uri,
const QString& providerKey,
const QgsCoordinateReferenceSystem *destCRS,
bool onlySelected = false,
QString *errorMessage = 0,
bool skipAttributeCreation = false,
QMap<QString, QVariant> *options = 0
);

/** create a empty layer and add fields to it */
QgsVectorLayerImport( const QString &uri,
const QString &provider,
const QgsFieldMap& fields,
QGis::WkbType geometryType,
const QgsCoordinateReferenceSystem* crs,
bool overwrite = false,
const QMap<QString, QVariant> *options = 0
);

/** checks whether there were any errors */
ImportError hasError();

/** retrieves error message */
QString errorMessage();

/** add feature to the new created layer */
bool addFeature( QgsFeature& feature );

/** close the new created layer */
~QgsVectorLayerImport();

protected:
/** flush the buffer writing the features to the new layer */
bool flushBuffer();

/** contains error value */
ImportError mError;
QString mErrorMessage;

QgsVectorDataProvider *mProvider;

/** map attribute indexes to new field indexes */
QMap<int, int> mOldToNewAttrIdx;

QgsFeatureList mFeatureBuffer;
};

#endif
133 changes: 131 additions & 2 deletions src/providers/ogr/qgsogrprovider.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@ email : sherman at mrcc.com
#include "qgsfield.h"
#include "qgsgeometry.h"
#include "qgscoordinatereferencesystem.h"
#include "qgsvectorfilewriter.h"
#include "qgsvectorlayer.h"
#include "qgsvectorlayerimport.h"

static const QString TEXT_PROVIDER_KEY = "ogr";
static const QString TEXT_PROVIDER_DESCRIPTION =
Expand Down Expand Up @@ -83,6 +82,121 @@ class QgsCPLErrorHandler
}
};


bool QgsOgrProvider::convertField( QgsField &field, const QTextCodec &encoding )
{
OGRFieldType ogrType = OFTString; //default to string
int ogrWidth = field.length();
int ogrPrecision = field.precision();
switch ( field.type() )
{
case QVariant::LongLong:
ogrType = OFTString;
ogrWidth = ogrWidth > 0 && ogrWidth <= 21 ? ogrWidth : 21;
ogrPrecision = -1;
break;

case QVariant::String:
ogrType = OFTString;
if ( ogrWidth < 0 || ogrWidth > 255 )
ogrWidth = 255;
break;

case QVariant::Int:
ogrType = OFTInteger;
ogrWidth = ogrWidth > 0 && ogrWidth <= 10 ? ogrWidth : 10;
ogrPrecision = 0;
break;

case QVariant::Double:
ogrType = OFTReal;
break;

default:
return false;
}

field.setTypeName( encoding.toUnicode( OGR_GetFieldTypeName( ogrType ) ) );
field.setLength( ogrWidth );
field.setPrecision( ogrPrecision );
return true;
}


QgsVectorLayerImport::ImportError QgsOgrProvider::createEmptyLayer(
const QString& uri,
const QgsFieldMap &fields,
QGis::WkbType wkbType,
const QgsCoordinateReferenceSystem *srs,
bool overwrite,
QMap<int, int> *oldToNewAttrIdxMap,
QString *errorMessage,
const QMap<QString,QVariant> *options )
{
QString encoding;
QString driverName = "ESRI Shapefile";
QStringList dsOptions, layerOptions;

if ( options )
{
if ( options->contains( "fileEncoding" ) )
encoding = options->value( "fileEncoding" ).toString();

if ( options->contains( "driverName" ) )
driverName = options->value( "driverName" ).toString();

if ( options->contains( "datasourceOptions" ) )
dsOptions << options->value( "datasourceOptions" ).toStringList();

if ( options->contains( "layerOptions" ) )
layerOptions << options->value( "layerOptions" ).toStringList();
}

if ( oldToNewAttrIdxMap )
oldToNewAttrIdxMap->clear();
if ( errorMessage )
errorMessage->clear();

if ( !overwrite )
{
QFileInfo fi( uri );
if ( fi.exists() )
{
if ( errorMessage )
*errorMessage += QObject::tr( "Unable to create the datasource. %1 exists and overwrite flag is false." )
.arg( uri );
return QgsVectorLayerImport::ErrCreateDataSource;
}
}

QgsVectorFileWriter *writer = new QgsVectorFileWriter(
uri, encoding, fields, wkbType,
srs, driverName, dsOptions, layerOptions );

QgsVectorFileWriter::WriterError error = writer->hasError();
if ( error )
{
if ( errorMessage )
*errorMessage += writer->errorMessage();

delete writer;
return ( QgsVectorLayerImport::ImportError ) error;
}

if ( oldToNewAttrIdxMap )
{
QMap<int, int> attrIdxMap = writer->attrIdxToOgrIdx();
for ( QMap<int, int>::const_iterator attrIt = attrIdxMap.begin(); attrIt != attrIdxMap.end(); ++attrIt )
{
oldToNewAttrIdxMap->insert( attrIt.key(), *attrIt );
}
}

delete writer;
return QgsVectorLayerImport::NoError;
}


QgsOgrProvider::QgsOgrProvider( QString const & uri )
: QgsVectorDataProvider( uri ),
ogrDataSource( 0 ),
Expand Down Expand Up @@ -2369,3 +2483,18 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
return 0;
}

QGISEXTERN QgsVectorLayerImport::ImportError createEmptyLayer(
const QString& uri,
const QgsFieldMap &fields,
QGis::WkbType wkbType,
const QgsCoordinateReferenceSystem *srs,
bool overwrite,
QMap<int, int> *oldToNewAttrIdxMap,
QString *errorMessage,
const QMap<QString,QVariant> *options )
{
return QgsOgrProvider::createEmptyLayer(
uri, fields, wkbType, srs, overwrite,
oldToNewAttrIdxMap, errorMessage, options
);
}
19 changes: 18 additions & 1 deletion src/providers/ogr/qgsogrprovider.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ email : sherman at mrcc.com
#include "qgsdataitem.h"
#include "qgsrectangle.h"
#include "qgsvectordataprovider.h"
#include "qgsvectorfilewriter.h"
#include "qgsvectorlayerimport.h"

class QgsFeature;
class QgsField;
class QgsVectorLayerImport;

#include <ogr_api.h>

Expand All @@ -34,6 +36,18 @@ class QgsOgrProvider : public QgsVectorDataProvider

public:

/** convert a vector layer to a vector file */
static QgsVectorLayerImport::ImportError createEmptyLayer(
const QString& uri,
const QgsFieldMap &fields,
QGis::WkbType wkbType,
const QgsCoordinateReferenceSystem *srs,
bool overwrite,
QMap<int, int> *oldToNewAttrIdxMap,
QString *errorMessage = 0,
const QMap<QString,QVariant> *options = 0
);

/**
* Constructor of the vector provider
* @param uri uniform resource locator (URI) for a dataset
Expand Down Expand Up @@ -254,6 +268,9 @@ class QgsOgrProvider : public QgsVectorDataProvider
/** tell OGR, which fields to fetch in nextFeature/featureAtId (ie. which not to ignore) */
void setRelevantFields( bool fetchGeometry, const QgsAttributeList& fetchAttributes );

/** convert a QgsField to work with OGR */
static bool convertField( QgsField &field, const QTextCodec &encoding );

private:
bool crsFromWkt( QgsCoordinateReferenceSystem &srs, const char *wkt );
unsigned char *getGeometryPointer( OGRFeatureH fet );
Expand Down
414 changes: 410 additions & 4 deletions src/providers/postgres/qgspostgresprovider.cpp
View file

Large diffs are not rendered by default.

30 changes: 28 additions & 2 deletions src/providers/postgres/qgspostgresprovider.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ extern "C"
#include "qgsvectordataprovider.h"
#include "qgsdataitem.h"
#include "qgsrectangle.h"
#include "qgsvectorlayerimport.h"

#include <list>
#include <queue>
Expand Down Expand Up @@ -66,6 +67,19 @@ class QgsPostgresProvider : public QgsVectorDataProvider
Q_OBJECT

public:

/** Import a vector layer into the database */
static QgsVectorLayerImport::ImportError createEmptyLayer(
const QString& uri,
const QgsFieldMap &fields,
QGis::WkbType wkbType,
const QgsCoordinateReferenceSystem *srs,
bool overwrite,
QMap<int, int> *oldToNewAttrIdxMap,
QString *errorMessage = 0,
const QMap<QString,QVariant> *options = 0
);

/**
* Constructor for the provider. The uri must be in the following format:
* host=localhost user=gsherman dbname=test password=xxx table=test.alaska (the_geom)
Expand Down Expand Up @@ -319,6 +333,7 @@ class QgsPostgresProvider : public QgsVectorDataProvider
*/
QString description() const;


signals:
/**
* This is emitted whenever the worker thread has fully calculated the
Expand All @@ -340,6 +355,7 @@ class QgsPostgresProvider : public QgsVectorDataProvider
void repaintRequested();

private:

int providerId; // id to append to provider specific identified (like cursors)

bool declareCursor( const QString &cursorName,
Expand Down Expand Up @@ -367,11 +383,11 @@ class QgsPostgresProvider : public QgsVectorDataProvider

/** Double quote a PostgreSQL identifier for placement in a SQL string.
*/
QString quotedIdentifier( QString ident ) const;
static QString quotedIdentifier( QString ident );

/** Quote a value for placement in a SQL string.
*/
QString quotedValue( QString value ) const;
static QString quotedValue( QString value );

/** expression to retrieve value
*/
Expand All @@ -381,6 +397,9 @@ class QgsPostgresProvider : public QgsVectorDataProvider
*/
bool loadFields();

/** convert a QgsField to work with PG */
static bool convertField( QgsField &field );

/**Parses the enum_range of an attribute and inserts the possible values into a stringlist
@param enumValues the stringlist where the values are appended
@param attributeName the name of the enum attribute
Expand Down Expand Up @@ -745,6 +764,13 @@ class QgsPostgresProvider : public QgsVectorDataProvider
* Default value for primary key
*/
QString mPrimaryKeyDefault;

#if 0
/** used to cache the lastest fetched features */
QHash<QgsFeatureId, QgsFeature> mFeatureMap;
QList<QgsFeatureId> mPriorityIds;
#endif

};

class QgsPGConnectionItem : public QgsDataCollectionItem
Expand Down
441 changes: 413 additions & 28 deletions src/providers/spatialite/qgsspatialiteprovider.cpp
View file

Large diffs are not rendered by default.

46 changes: 44 additions & 2 deletions src/providers/spatialite/qgsspatialiteprovider.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ extern "C"

#include "qgsvectordataprovider.h"
#include "qgsrectangle.h"
#include "qgsvectorlayerimport.h"
#include <list>
#include <queue>
#include <fstream>
Expand All @@ -45,6 +46,19 @@ class QgsField;
class QgsSpatiaLiteProvider: public QgsVectorDataProvider
{
Q_OBJECT public:

/** Import a vector layer into the database */
static QgsVectorLayerImport::ImportError createEmptyLayer(
const QString& uri,
const QgsFieldMap &fields,
QGis::WkbType wkbType,
const QgsCoordinateReferenceSystem *srs,
bool overwrite,
QMap<int, int> *oldToNewAttrIdxMap,
QString *errorMessage = 0,
const QMap<QString,QVariant> *options = 0
);

/**
* Constructor of the vector provider
* @param uri uniform resource locator (URI) for a dataset
Expand Down Expand Up @@ -262,6 +276,9 @@ class QgsSpatiaLiteProvider: public QgsVectorDataProvider
/** loads fields from input file to member attributeFields */
void loadFields();

/** convert a QgsField to work with SL */
static bool convertField( QgsField &field );

QgsFieldMap attributeFields;
/**
* Flag indicating if the layer data source is a valid SpatiaLite layer
Expand Down Expand Up @@ -370,8 +387,8 @@ class QgsSpatiaLiteProvider: public QgsVectorDataProvider
*/
//void sqliteOpen();
void closeDb();
QString quotedIdentifier( QString id ) const;
QString quotedValue( QString value ) const;
static QString quotedIdentifier( QString id );
static QString quotedValue( QString value );
bool checkLayerType();
bool getGeometryDetails();
bool getTableGeometryDetails();
Expand Down Expand Up @@ -458,6 +475,31 @@ class QgsSpatiaLiteProvider: public QgsVectorDataProvider
static QMap < QString, SqliteHandles * >handles;
};

struct SLException
{
SLException( char *msg ) : errMsg( msg )
{
}

SLException( const SLException &e ) : errMsg( e.errMsg )
{
}

~SLException()
{
if ( errMsg )
sqlite3_free( errMsg );
}

QString errorMessage() const
{
return errMsg ? QString::fromUtf8( errMsg ) : "unknown cause";
}

private:
char *errMsg;
};

/**
* sqlite3 handles pointer
*/
Expand Down