Skip to content

Commit

Permalink
Ignore FID field when merging vector layers to geopackage
Browse files Browse the repository at this point in the history
  • Loading branch information
m-kuhn committed Oct 31, 2018
1 parent b91c04a commit 8a46059
Show file tree
Hide file tree
Showing 18 changed files with 103 additions and 35 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -611,7 +611,7 @@ Evaluates the parameter with matching ``name`` to a static boolean value.
%End

QgsFeatureSink *parameterAsSink( const QVariantMap &parameters, const QString &name, QgsProcessingContext &context, QString &destinationIdentifier /Out/,
const QgsFields &fields, QgsWkbTypes::Type geometryType = QgsWkbTypes::NoGeometry, const QgsCoordinateReferenceSystem &crs = QgsCoordinateReferenceSystem() ) const /Factory/;
const QgsFields &fields, QgsWkbTypes::Type geometryType = QgsWkbTypes::NoGeometry, const QgsCoordinateReferenceSystem &crs = QgsCoordinateReferenceSystem(), QgsFeatureSink::SinkFlags options = 0 ) const /Factory/;
%Docstring
Evaluates the parameter with matching ``name`` to a feature sink.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -588,7 +588,7 @@ Evaluates the parameter with matching ``definition`` and ``value`` to a static b

static QgsFeatureSink *parameterAsSink( const QgsProcessingParameterDefinition *definition, const QVariantMap &parameters,
const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs,
QgsProcessingContext &context, QString &destinationIdentifier /Out/ ) /Factory/;
QgsProcessingContext &context, QString &destinationIdentifier /Out/, QgsFeatureSink::SinkFlags options = 0 ) /Factory/;
%Docstring
Evaluates the parameter with matching ``definition`` to a feature sink.

Expand All @@ -605,7 +605,7 @@ This function creates a new object and the caller takes responsibility for delet

static QgsFeatureSink *parameterAsSink( const QgsProcessingParameterDefinition *definition, const QVariant &value,
const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs,
QgsProcessingContext &context, QString &destinationIdentifier /Out/ ) /Factory/;
QgsProcessingContext &context, QString &destinationIdentifier /Out/, QgsFeatureSink::SinkFlags options = 0 ) /Factory/;
%Docstring
Evaluates the parameter with matching ``definition`` and ``value`` to a feature sink.

Expand Down
8 changes: 8 additions & 0 deletions python/core/auto_generated/qgsfeaturesink.sip.in
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,14 @@ An interface for objects which accept features via addFeature(s) methods.
%End
public:

enum SinkFlag
{

RegeneratePrimaryKey,
};
typedef QFlags<QgsFeatureSink::SinkFlag> SinkFlags;


enum Flag
{

Expand Down
3 changes: 2 additions & 1 deletion python/core/auto_generated/qgsvectorfilewriter.sip.in
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,8 @@ Writes a layer out to a vector file.
const QStringList &datasourceOptions = QStringList(),
const QStringList &layerOptions = QStringList(),
QString *newFilename = 0,
QgsVectorFileWriter::SymbologyExport symbologyExport = QgsVectorFileWriter::NoSymbology
QgsVectorFileWriter::SymbologyExport symbologyExport = QgsVectorFileWriter::NoSymbology,
QgsFeatureSink::SinkFlags sinkFlags = 0
);


Expand Down
3 changes: 2 additions & 1 deletion python/core/auto_generated/qgsvectorlayerexporter.sip.in
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ Writes the contents of vector layer to a different datasource.
QgsWkbTypes::Type geometryType,
const QgsCoordinateReferenceSystem &crs,
bool overwrite = false,
const QMap<QString, QVariant> &options = QMap<QString, QVariant>() );
const QMap<QString, QVariant> &options = QMap<QString, QVariant>(),
QgsFeatureSink::SinkFlags sinkOptions = 0 );
%Docstring
Constructor for QgsVectorLayerExporter.

Expand Down
2 changes: 1 addition & 1 deletion src/analysis/processing/qgsalgorithmmergevector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ QVariantMap QgsMergeVectorAlgorithm::processAlgorithm( const QVariantMap &parame
}

QString dest;
std::unique_ptr< QgsFeatureSink > sink( parameterAsSink( parameters, QStringLiteral( "OUTPUT" ), context, dest, outputFields, outputType, outputCrs ) );
std::unique_ptr< QgsFeatureSink > sink( parameterAsSink( parameters, QStringLiteral( "OUTPUT" ), context, dest, outputFields, outputType, outputCrs, QgsFeatureSink::RegeneratePrimaryKey ) );
if ( !sink )
throw QgsProcessingException( invalidSinkError( parameters, QStringLiteral( "OUTPUT" ) ) );

Expand Down
5 changes: 4 additions & 1 deletion src/app/qgsgeometryvalidationservice.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,9 @@ void QgsGeometryValidationService::processFeature( QgsVectorLayer *layer, QgsFea
if ( !allErrors.empty() )
mLayerChecks[layer].singleFeatureCheckErrors.insert( fid, allErrors );

if ( !mLayerChecks[layer].singleFeatureCheckErrors.empty() )
layer->setAllowCommit( false );

emit geometryCheckCompleted( layer, fid, allErrors );
}

Expand Down Expand Up @@ -432,7 +435,7 @@ void QgsGeometryValidationService::triggerTopologyChecks( QgsVectorLayer *layer
connect( futureWatcher, &QFutureWatcherBase::finished, this, [&allErrors, layer, feedbacks, futureWatcher, this]()
{
QgsReadWriteLocker errorLocker( mTopologyCheckLock, QgsReadWriteLocker::Read );
layer->setAllowCommit( allErrors.empty() );
layer->setAllowCommit( allErrors.empty() && mLayerChecks[layer].singleFeatureCheckErrors.empty() );
errorLocker.unlock();
qDeleteAll( feedbacks.values() );
futureWatcher->deleteLater();
Expand Down
4 changes: 2 additions & 2 deletions src/core/processing/qgsprocessingalgorithm.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -586,9 +586,9 @@ bool QgsProcessingAlgorithm::parameterAsBool( const QVariantMap &parameters, con
return QgsProcessingParameters::parameterAsBool( parameterDefinition( name ), parameters, context );
}

QgsFeatureSink *QgsProcessingAlgorithm::parameterAsSink( const QVariantMap &parameters, const QString &name, QgsProcessingContext &context, QString &destinationIdentifier, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs ) const
QgsFeatureSink *QgsProcessingAlgorithm::parameterAsSink( const QVariantMap &parameters, const QString &name, QgsProcessingContext &context, QString &destinationIdentifier, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, QgsFeatureSink::SinkFlags options ) const
{
return QgsProcessingParameters::parameterAsSink( parameterDefinition( name ), parameters, fields, geometryType, crs, context, destinationIdentifier );
return QgsProcessingParameters::parameterAsSink( parameterDefinition( name ), parameters, fields, geometryType, crs, context, destinationIdentifier, options );
}

QgsProcessingFeatureSource *QgsProcessingAlgorithm::parameterAsSource( const QVariantMap &parameters, const QString &name, QgsProcessingContext &context ) const
Expand Down
2 changes: 1 addition & 1 deletion src/core/processing/qgsprocessingalgorithm.h
Original file line number Diff line number Diff line change
Expand Up @@ -619,7 +619,7 @@ class CORE_EXPORT QgsProcessingAlgorithm
* This function creates a new object and the caller takes responsibility for deleting the returned object.
*/
QgsFeatureSink *parameterAsSink( const QVariantMap &parameters, const QString &name, QgsProcessingContext &context, QString &destinationIdentifier SIP_OUT,
const QgsFields &fields, QgsWkbTypes::Type geometryType = QgsWkbTypes::NoGeometry, const QgsCoordinateReferenceSystem &crs = QgsCoordinateReferenceSystem() ) const SIP_FACTORY;
const QgsFields &fields, QgsWkbTypes::Type geometryType = QgsWkbTypes::NoGeometry, const QgsCoordinateReferenceSystem &crs = QgsCoordinateReferenceSystem(), QgsFeatureSink::SinkFlags options = nullptr ) const SIP_FACTORY;

/**
* Evaluates the parameter with matching \a name to a feature source.
Expand Down
9 changes: 5 additions & 4 deletions src/core/processing/qgsprocessingparameters.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -350,18 +350,18 @@ bool QgsProcessingParameters::parameterAsBool( const QgsProcessingParameterDefin

QgsFeatureSink *QgsProcessingParameters::parameterAsSink( const QgsProcessingParameterDefinition *definition, const QVariantMap &parameters, const QgsFields &fields,
QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs,
QgsProcessingContext &context, QString &destinationIdentifier )
QgsProcessingContext &context, QString &destinationIdentifier, QgsFeatureSink::SinkFlags options )
{
QVariant val;
if ( definition )
{
val = parameters.value( definition->name() );
}

return parameterAsSink( definition, val, fields, geometryType, crs, context, destinationIdentifier );
return parameterAsSink( definition, val, fields, geometryType, crs, context, destinationIdentifier, options );
}

QgsFeatureSink *QgsProcessingParameters::parameterAsSink( const QgsProcessingParameterDefinition *definition, const QVariant &value, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, QgsProcessingContext &context, QString &destinationIdentifier )
QgsFeatureSink *QgsProcessingParameters::parameterAsSink( const QgsProcessingParameterDefinition *definition, const QVariant &value, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, QgsProcessingContext &context, QString &destinationIdentifier, QgsFeatureSink::SinkFlags options )
{
QVariant val = value;

Expand All @@ -374,6 +374,7 @@ QgsFeatureSink *QgsProcessingParameters::parameterAsSink( const QgsProcessingPar
QgsProcessingOutputLayerDefinition fromVar = qvariant_cast<QgsProcessingOutputLayerDefinition>( val );
destinationProject = fromVar.destinationProject;
createOptions = fromVar.createOptions;

val = fromVar.sink;
destName = fromVar.destinationName;
}
Expand Down Expand Up @@ -401,7 +402,7 @@ QgsFeatureSink *QgsProcessingParameters::parameterAsSink( const QgsProcessingPar
if ( dest.isEmpty() )
return nullptr;

std::unique_ptr< QgsFeatureSink > sink( QgsProcessingUtils::createFeatureSink( dest, context, fields, geometryType, crs, createOptions ) );
std::unique_ptr< QgsFeatureSink > sink( QgsProcessingUtils::createFeatureSink( dest, context, fields, geometryType, crs, createOptions, options ) );
destinationIdentifier = dest;

if ( destinationProject )
Expand Down
5 changes: 3 additions & 2 deletions src/core/processing/qgsprocessingparameters.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include "qgsproperty.h"
#include "qgscoordinatereferencesystem.h"
#include "qgsfeaturesource.h"
#include "qgsprocessingutils.h"
#include <QMap>
#include <limits>

Expand Down Expand Up @@ -662,7 +663,7 @@ class CORE_EXPORT QgsProcessingParameters
*/
static QgsFeatureSink *parameterAsSink( const QgsProcessingParameterDefinition *definition, const QVariantMap &parameters,
const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs,
QgsProcessingContext &context, QString &destinationIdentifier SIP_OUT ) SIP_FACTORY;
QgsProcessingContext &context, QString &destinationIdentifier SIP_OUT, QgsFeatureSink::SinkFlags options = nullptr ) SIP_FACTORY;

/**
* Evaluates the parameter with matching \a definition and \a value to a feature sink.
Expand All @@ -681,7 +682,7 @@ class CORE_EXPORT QgsProcessingParameters
*/
static QgsFeatureSink *parameterAsSink( const QgsProcessingParameterDefinition *definition, const QVariant &value,
const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs,
QgsProcessingContext &context, QString &destinationIdentifier SIP_OUT ) SIP_FACTORY;
QgsProcessingContext &context, QString &destinationIdentifier SIP_OUT, QgsFeatureSink::SinkFlags options = nullptr ) SIP_FACTORY;

/**
* Evaluates the parameter with matching \a definition to a feature source.
Expand Down
10 changes: 6 additions & 4 deletions src/core/processing/qgsprocessingutils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,7 @@ void QgsProcessingUtils::parseDestinationString( QString &destination, QString &
options.insert( QStringLiteral( "layerName" ), layerName );
}
uri = dsUri.database();
format = QgsVectorFileWriter::driverForExtension( QFileInfo( uri ).completeSuffix() );
}
options.insert( QStringLiteral( "update" ), true );
}
Expand Down Expand Up @@ -402,7 +403,7 @@ void QgsProcessingUtils::parseDestinationString( QString &destination, QString &
}
}

QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, QgsProcessingContext &context, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, const QVariantMap &createOptions )
QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, QgsProcessingContext &context, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, const QVariantMap &createOptions, QgsFeatureSink::SinkFlags sinkFlags )
{
QVariantMap options = createOptions;
if ( !options.contains( QStringLiteral( "fileEncoding" ) ) )
Expand Down Expand Up @@ -445,13 +446,14 @@ QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, Qgs
bool useWriter = false;
parseDestinationString( destination, providerKey, uri, layerName, format, options, useWriter );

QgsFields newFields = fields;
if ( useWriter && providerKey == QLatin1String( "ogr" ) )
{
// use QgsVectorFileWriter for OGR destinations instead of QgsVectorLayerImport, as that allows
// us to use any OGR format which supports feature addition
QString finalFileName;
std::unique_ptr< QgsVectorFileWriter > writer = qgis::make_unique< QgsVectorFileWriter >( destination, options.value( QStringLiteral( "fileEncoding" ) ).toString(), fields, geometryType, crs, format, QgsVectorFileWriter::defaultDatasetOptions( format ),
QgsVectorFileWriter::defaultLayerOptions( format ), &finalFileName );
std::unique_ptr< QgsVectorFileWriter > writer = qgis::make_unique< QgsVectorFileWriter >( destination, options.value( QStringLiteral( "fileEncoding" ) ).toString(), newFields, geometryType, crs, format, QgsVectorFileWriter::defaultDatasetOptions( format ),
QgsVectorFileWriter::defaultLayerOptions( format ), &finalFileName, QgsVectorFileWriter::NoSymbology, sinkFlags );

if ( writer->hasError() )
{
Expand All @@ -463,7 +465,7 @@ QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, Qgs
else
{
//create empty layer
std::unique_ptr< QgsVectorLayerExporter > exporter( new QgsVectorLayerExporter( uri, providerKey, fields, geometryType, crs, true, options ) );
std::unique_ptr< QgsVectorLayerExporter > exporter( new QgsVectorLayerExporter( uri, providerKey, newFields, geometryType, crs, true, options, sinkFlags ) );
if ( exporter->errorCode() )
{
throw QgsProcessingException( QObject::tr( "Could not create layer %1: %2" ).arg( destination, exporter->errorMessage() ) );
Expand Down
15 changes: 7 additions & 8 deletions src/core/processing/qgsprocessingutils.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ class QgsProcessingFeatureSource;
*/
class CORE_EXPORT QgsProcessingUtils
{

public:

/**
Expand Down Expand Up @@ -154,13 +153,13 @@ class CORE_EXPORT QgsProcessingUtils
* The caller takes responsibility for deleting the returned sink.
*/
#ifndef SIP_RUN
static QgsFeatureSink *createFeatureSink(
QString &destination,
QgsProcessingContext &context,
const QgsFields &fields,
QgsWkbTypes::Type geometryType,
const QgsCoordinateReferenceSystem &crs,
const QVariantMap &createOptions = QVariantMap() ) SIP_FACTORY;
static QgsFeatureSink *createFeatureSink( QString &destination,
QgsProcessingContext &context,
const QgsFields &fields,
QgsWkbTypes::Type geometryType,
const QgsCoordinateReferenceSystem &crs,
const QVariantMap &createOptions = QVariantMap(),
QgsFeatureSink::SinkFlags options = nullptr ) SIP_FACTORY;
#endif

/**
Expand Down
22 changes: 22 additions & 0 deletions src/core/qgsfeaturesink.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,28 @@ class CORE_EXPORT QgsFeatureSink
{
public:

/**
* Flags that can be set on a QgsFeatureSink. Not all sinks may implement all flags.
*
* \since QGIS 3.4
*/
enum SinkFlag
{

/**
* This flag indicates, that a primary key field cannot be guaranteed to be unique and
* the sink should ignore it if somehow possible.
* This should for example be set for a geopackage file if the field "fid" has a risk
* to contain duplicate entries. In this case sinks like QgsVectorFileWriter or
* QgsVectorLayerExporter will prefer to regenerate the fid instead of trying to reuse
* the fids provided in addFeature calls.
*
* \since QGIS 3.4
*/
RegeneratePrimaryKey = 1 << 1,
};
Q_DECLARE_FLAGS( SinkFlags, SinkFlag )

//! Flags controlling how features are added to a sink.
enum Flag
{
Expand Down
18 changes: 15 additions & 3 deletions src/core/qgsvectorfilewriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ QgsVectorFileWriter::QgsVectorFileWriter(
const QStringList &layerOptions,
QString *newFilename,
SymbologyExport symbologyExport,
QgsFeatureSink::SinkFlags sinkFlags,
QString *newLayer
)
: mError( NoError )
Expand All @@ -111,7 +112,7 @@ QgsVectorFileWriter::QgsVectorFileWriter(
{
init( vectorFileName, fileEncoding, fields, geometryType,
srs, driverName, datasourceOptions, layerOptions, newFilename, nullptr,
QString(), CreateOrOverwriteFile, newLayer );
QString(), CreateOrOverwriteFile, newLayer, sinkFlags );
}

QgsVectorFileWriter::QgsVectorFileWriter( const QString &vectorFileName,
Expand All @@ -135,7 +136,7 @@ QgsVectorFileWriter::QgsVectorFileWriter( const QString &vectorFileName,
{
init( vectorFileName, fileEncoding, fields, geometryType, srs, driverName,
datasourceOptions, layerOptions, newFilename, fieldValueConverter,
layerName, action, newLayer );
layerName, action, newLayer, nullptr );
}

bool QgsVectorFileWriter::supportsFeatureStyles( const QString &driverName )
Expand Down Expand Up @@ -171,7 +172,7 @@ void QgsVectorFileWriter::init( QString vectorFileName,
FieldValueConverter *fieldValueConverter,
const QString &layerNameIn,
ActionOnExistingFile action,
QString *newLayer )
QString *newLayer, SinkFlags sinkFlags )
{
mRenderContext.setRendererScale( mSymbologyScale );

Expand Down Expand Up @@ -310,6 +311,7 @@ void QgsVectorFileWriter::init( QString vectorFileName,
}
options[ datasourceOptions.size()] = nullptr;
}
mAttrIdxToOgrIdx.remove( 0 );

// create the data source
if ( action == CreateOrOverwriteFile )
Expand Down Expand Up @@ -687,6 +689,16 @@ void QgsVectorFileWriter::init( QString vectorFileName,
}
}

// Geopackages require a unique feature id. If the input feature stream cannot guarantee
// the uniqueness of the FID column, we drop it and let OGR generate new ones
if ( sinkFlags.testFlag( QgsFeatureSink::RegeneratePrimaryKey ) && driverName == QLatin1String( "GPKG" ) )
{
int fidIdx = fields.lookupField( QStringLiteral( "FID" ) );

if ( fidIdx >= 0 )
mAttrIdxToOgrIdx.remove( fidIdx );
}

QgsDebugMsg( QStringLiteral( "Done creating fields" ) );

mWkbType = geometryType;
Expand Down
5 changes: 3 additions & 2 deletions src/core/qgsvectorfilewriter.h
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,8 @@ class CORE_EXPORT QgsVectorFileWriter : public QgsFeatureSink
const QStringList &datasourceOptions = QStringList(),
const QStringList &layerOptions = QStringList(),
QString *newFilename = nullptr,
QgsVectorFileWriter::SymbologyExport symbologyExport = QgsVectorFileWriter::NoSymbology
QgsVectorFileWriter::SymbologyExport symbologyExport = QgsVectorFileWriter::NoSymbology,
QgsFeatureSink::SinkFlags sinkFlags = nullptr
#ifndef SIP_RUN
, QString *newLayer = nullptr
#endif
Expand Down Expand Up @@ -880,7 +881,7 @@ class CORE_EXPORT QgsVectorFileWriter : public QgsFeatureSink
QStringList layerOptions, QString *newFilename,
QgsVectorFileWriter::FieldValueConverter *fieldValueConverter,
const QString &layerName,
QgsVectorFileWriter::ActionOnExistingFile action, QString *newLayer );
QgsVectorFileWriter::ActionOnExistingFile action, QString *newLayer, QgsFeatureSink::SinkFlags sinkFlags );
void resetMap( const QgsAttributeList &attributes );

std::unique_ptr< QgsFeatureRenderer > mRenderer;
Expand Down

0 comments on commit 8a46059

Please sign in to comment.