Skip to content
Permalink
Browse files

[processing] Add algorithms for dealing with shapefile encoding issues

- "Extract Shapefile encoding": extracts the embedded shapefile encoding
information and lists it for the user (and storing it in output strings
for use in models)
- "Set layer encoding": allows users to set the encoding for a vector
layer to a different encoding (applies to the layer only, no permanent
changes are made to the data source). For use in handling encoding
issues in Processing models
  • Loading branch information
nyalldawson committed Feb 10, 2020
1 parent 8c10e08 commit 8cec5d0686c0e2800a1c63935ff8ac6608056a1f
@@ -131,11 +131,13 @@ SET(QGIS_ANALYSIS_SRCS
processing/qgsalgorithmsegmentize.cpp
processing/qgsalgorithmserviceareafromlayer.cpp
processing/qgsalgorithmserviceareafrompoint.cpp
processing/qgsalgorithmsetlayerencoding.cpp
processing/qgsalgorithmsetmvalue.cpp
processing/qgsalgorithmsetzvalue.cpp
processing/qgsalgorithmshortestpathlayertopoint.cpp
processing/qgsalgorithmshortestpathpointtolayer.cpp
processing/qgsalgorithmshortestpathpointtopoint.cpp
processing/qgsalgorithmshpencodinginfo.cpp
processing/qgsalgorithmsimplify.cpp
processing/qgsalgorithmsinglesidedbuffer.cpp
processing/qgsalgorithmslope.cpp
@@ -0,0 +1,106 @@
/***************************************************************************
qgsalgorithmsetlayerencoding.cpp
------------------------------
begin : February 2020
copyright : (C) 2020 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#include "qgsalgorithmsetlayerencoding.h"
#include "qgsvectorlayer.h"
#include "qgsvectordataprovider.h"

///@cond PRIVATE

QString QgsSetLayerEncodingAlgorithm::name() const
{
return QStringLiteral( "setlayerencoding" );
}

QString QgsSetLayerEncodingAlgorithm::displayName() const
{
return QObject::tr( "Set layer encoding" );
}

QStringList QgsSetLayerEncodingAlgorithm::tags() const
{
return QObject::tr( "change,alter,attribute,codepage" ).split( ',' );
}

QString QgsSetLayerEncodingAlgorithm::group() const
{
return QObject::tr( "Vector general" );
}

QString QgsSetLayerEncodingAlgorithm::groupId() const
{
return QStringLiteral( "vectorgeneral" );
}

QString QgsSetLayerEncodingAlgorithm::shortHelpString() const
{
return QObject::tr( "This algorithm sets the encoding used for reading a layer's attributes. No permanant changes "
"are made to the layer, rather it affects only how the layer is read during the current session.\n\n"
"Changing the encoding is only supported for some vector layer data sources." );
}

QString QgsSetLayerEncodingAlgorithm::shortDescription() const
{
return QObject::tr( "Sets the encoding used for reading a layer's attributes" );
}

QgsSetLayerEncodingAlgorithm *QgsSetLayerEncodingAlgorithm::createInstance() const
{
return new QgsSetLayerEncodingAlgorithm();
}

void QgsSetLayerEncodingAlgorithm::initAlgorithm( const QVariantMap & )
{
addParameter( new QgsProcessingParameterVectorLayer( QStringLiteral( "INPUT" ), QObject::tr( "Input layer" ) ) );
addParameter( new QgsProcessingParameterString( QStringLiteral( "ENCODING" ), QObject::tr( "Encoding" ) ) );

addOutput( new QgsProcessingOutputVectorLayer( QStringLiteral( "OUTPUT" ), QObject::tr( "Output layer" ) ) );
}

bool QgsSetLayerEncodingAlgorithm::prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
{
QgsVectorLayer *layer = parameterAsVectorLayer( parameters, QStringLiteral( "INPUT" ), context );

if ( !layer )
throw QgsProcessingException( QObject::tr( "Could not load source layer for %1." ).arg( QStringLiteral( "INPUT" ) ) );

const QString encoding = parameterAsString( parameters, QStringLiteral( "ENCODING" ), context );

mOutputId = layer->id();
QgsVectorDataProvider *provider = layer->dataProvider();

if ( provider->capabilities() & QgsVectorDataProvider::SelectEncoding )
{
layer->setProviderEncoding( encoding );
}
else
{
feedback->pushInfo( QObject::tr( "Layer's data provider does not support changing the attribute encoding" ) );
// we don't return false here -- rather we allow the algorithm to gracefully handle an attempt to be flexible to different layer sources,
// otherwise we potentially break model input flexibility!
}
return true;
}

QVariantMap QgsSetLayerEncodingAlgorithm::processAlgorithm( const QVariantMap &, QgsProcessingContext &, QgsProcessingFeedback * )
{
QVariantMap outputs;
outputs.insert( QStringLiteral( "OUTPUT" ), mOutputId );
return outputs;
}

///@endcond
@@ -0,0 +1,59 @@
/***************************************************************************
qgsalgorithmsetlayerencoding.h
------------------------------
begin : February 2020
copyright : (C) 2020 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#ifndef QGSALGORITHMSETLAYERENCODING_H
#define QGSALGORITHMSETLAYERENCODING_H

#define SIP_NO_FILE

#include "qgis_sip.h"
#include "qgsprocessingalgorithm.h"

///@cond PRIVATE

/**
* Native constant raster algorithm.
*/
class QgsSetLayerEncodingAlgorithm : public QgsProcessingAlgorithm
{

public:

QgsSetLayerEncodingAlgorithm() = default;
void initAlgorithm( const QVariantMap &configuration = QVariantMap() ) override;
QString name() const override;
QString displayName() const override;
QStringList tags() const override;
QString group() const override;
QString groupId() const override;
QString shortHelpString() const override;
QString shortDescription() const override;
QgsSetLayerEncodingAlgorithm *createInstance() const override SIP_FACTORY;

protected:

bool prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;
QVariantMap processAlgorithm( const QVariantMap &parameters,
QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;

private:
QString mOutputId;
};

///@endcond PRIVATE

#endif // QGSALGORITHMSETLAYERENCODING_H
@@ -0,0 +1,111 @@
/***************************************************************************
qgsalgorithmshpencodinginfo.cpp
-----------------------------
begin : February 2020
copyright : (C) 2020 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#include "qgsalgorithmshpencodinginfo.h"
#include "qgsogrutils.h"

///@cond PRIVATE

QString QgsShapefileEncodingInfoAlgorithm::name() const
{
return QStringLiteral( "shpencodinginfo" );
}

QString QgsShapefileEncodingInfoAlgorithm::displayName() const
{
return QObject::tr( "Extract Shapefile encoding" );
}

QStringList QgsShapefileEncodingInfoAlgorithm::tags() const
{
return QObject::tr( "shp,codepage,cpg,ldid,information,list,show" ).split( ',' );
}

QString QgsShapefileEncodingInfoAlgorithm::group() const
{
return QObject::tr( "Vector general" );
}

QString QgsShapefileEncodingInfoAlgorithm::groupId() const
{
return QStringLiteral( "vectorgeneral" );
}

QString QgsShapefileEncodingInfoAlgorithm::shortHelpString() const
{
return QObject::tr( "This algorithm extracts the attribute encoding information embedded in a Shapefile.\n\n"
"Both the encoding specified by an optional .cpg file and any encoding details present in the "
".dbf LDID header block are considered." );
}

QString QgsShapefileEncodingInfoAlgorithm::shortDescription() const
{
return QObject::tr( "Extracts the attribute encoding information embedded in a Shapefile." );
}

QgsShapefileEncodingInfoAlgorithm *QgsShapefileEncodingInfoAlgorithm::createInstance() const
{
return new QgsShapefileEncodingInfoAlgorithm();
}

void QgsShapefileEncodingInfoAlgorithm::initAlgorithm( const QVariantMap & )
{
addParameter( new QgsProcessingParameterFile( QStringLiteral( "INPUT" ), QObject::tr( "Input layer" ), QgsProcessingParameterFile::File,
QString(), QVariant(), false, QObject::tr( "Shapefiles (%1)" ).arg( QStringLiteral( "*.shp *.SHP)" ) ) ) );

addOutput( new QgsProcessingOutputString( QStringLiteral( "ENCODING" ), QObject::tr( "Shapefile Encoding" ) ) );
addOutput( new QgsProcessingOutputString( QStringLiteral( "CPG_ENCODING" ), QObject::tr( "CPG Encoding" ) ) );
addOutput( new QgsProcessingOutputString( QStringLiteral( "LDID_ENCODING" ), QObject::tr( "LDID Encoding" ) ) );
}

bool QgsShapefileEncodingInfoAlgorithm::prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
{
const QString path = parameterAsFile( parameters, QStringLiteral( "INPUT" ), context );
if ( !QFile::exists( path ) )
{
feedback->reportError( QObject::tr( "Input Shapefile %1 does not exist" ).arg( path ) );
return false;
}
else
{
mCpgEncoding = QgsOgrUtils::readShapefileEncodingFromCpg( path );
if ( mCpgEncoding.isEmpty() )
feedback->pushInfo( QObject::tr( "No encoding information present in CPG file" ) );
else
feedback->pushInfo( QObject::tr( "Detected encoding from CPG file: %1" ).arg( mCpgEncoding ) );

mLdidEncoding = QgsOgrUtils::readShapefileEncodingFromLdid( path );
if ( mLdidEncoding.isEmpty() )
feedback->pushInfo( QObject::tr( "No encoding information present in DBF LDID header" ) );
else
feedback->pushInfo( QObject::tr( "Detected encoding from DBF LDID header: %1" ).arg( mLdidEncoding ) );

}
return true;
}


QVariantMap QgsShapefileEncodingInfoAlgorithm::processAlgorithm( const QVariantMap &, QgsProcessingContext &, QgsProcessingFeedback * )
{
QVariantMap outputs;
outputs.insert( QStringLiteral( "ENCODING" ), mCpgEncoding.isEmpty() ? mLdidEncoding : mCpgEncoding );
outputs.insert( QStringLiteral( "CPG_ENCODING" ), mCpgEncoding );
outputs.insert( QStringLiteral( "LDID_ENCODING" ), mLdidEncoding );
return outputs;
}

///@endcond
@@ -0,0 +1,62 @@
/***************************************************************************
qgsalgorithmshpencodinginfo.h
-----------------------------
begin : February 2020
copyright : (C) 2020 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#ifndef QGSALGORITHMSHPENCODINGINFO_H
#define QGSALGORITHMSHPENCODINGINFO_H

#define SIP_NO_FILE

#include "qgis_sip.h"
#include "qgsprocessingalgorithm.h"

///@cond PRIVATE

/**
* Native constant raster algorithm.
*/
class QgsShapefileEncodingInfoAlgorithm : public QgsProcessingAlgorithm
{

public:

QgsShapefileEncodingInfoAlgorithm() = default;
void initAlgorithm( const QVariantMap &configuration = QVariantMap() ) override;
QString name() const override;
QString displayName() const override;
QStringList tags() const override;
QString group() const override;
QString groupId() const override;
QString shortHelpString() const override;
QString shortDescription() const override;
QgsShapefileEncodingInfoAlgorithm *createInstance() const override SIP_FACTORY;

protected:
bool prepareAlgorithm( const QVariantMap &parameters,
QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;

QVariantMap processAlgorithm( const QVariantMap &parameters,
QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;

private:

QString mCpgEncoding;
QString mLdidEncoding;
};

///@endcond PRIVATE

#endif // QGSALGORITHMSHPENCODINGINFO_H
@@ -125,11 +125,13 @@
#include "qgsalgorithmsegmentize.h"
#include "qgsalgorithmserviceareafromlayer.h"
#include "qgsalgorithmserviceareafrompoint.h"
#include "qgsalgorithmsetlayerencoding.h"
#include "qgsalgorithmsetmvalue.h"
#include "qgsalgorithmsetzvalue.h"
#include "qgsalgorithmshortestpathlayertopoint.h"
#include "qgsalgorithmshortestpathpointtolayer.h"
#include "qgsalgorithmshortestpathpointtopoint.h"
#include "qgsalgorithmshpencodinginfo.h"
#include "qgsalgorithmsimplify.h"
#include "qgsalgorithmsinglesidedbuffer.h"
#include "qgsalgorithmslope.h"
@@ -325,8 +327,10 @@ void QgsNativeAlgorithms::loadAlgorithms()
addAlgorithm( new QgsSelectByLocationAlgorithm() );
addAlgorithm( new QgsServiceAreaFromLayerAlgorithm() );
addAlgorithm( new QgsServiceAreaFromPointAlgorithm() );
addAlgorithm( new QgsSetLayerEncodingAlgorithm() );
addAlgorithm( new QgsSetMValueAlgorithm() );
addAlgorithm( new QgsSetZValueAlgorithm() );
addAlgorithm( new QgsShapefileEncodingInfoAlgorithm() );
addAlgorithm( new QgsShortestPathLayerToPointAlgorithm() );
addAlgorithm( new QgsShortestPathPointToLayerAlgorithm() );
addAlgorithm( new QgsShortestPathPointToPointAlgorithm() );

0 comments on commit 8cec5d0

Please sign in to comment.
You can’t perform that action at this time.