[processing] Add algorithms for dealing with shapefile encoding issues

- "Extract Shapefile encoding": extracts the embedded shapefile encoding information and lists it for the user (and storing it in output strings for use in models) - "Set layer encoding": allows users to set the encoding for a vector layer to a different encoding (applies to the layer only, no permanent changes are made to the data source). For use in handling encoding issues in Processing models
qgis · Feb 12, 2020 · 8cec5d0 · 8cec5d0
1 parent 8c10e08
commit 8cec5d0
Show file tree

Hide file tree

Showing 7 changed files with 416 additions and 0 deletions.
diff --git a/src/analysis/CMakeLists.txt b/src/analysis/CMakeLists.txt
@@ -131,11 +131,13 @@ SET(QGIS_ANALYSIS_SRCS
   processing/qgsalgorithmsegmentize.cpp
   processing/qgsalgorithmserviceareafromlayer.cpp
   processing/qgsalgorithmserviceareafrompoint.cpp
+  processing/qgsalgorithmsetlayerencoding.cpp
   processing/qgsalgorithmsetmvalue.cpp
   processing/qgsalgorithmsetzvalue.cpp
   processing/qgsalgorithmshortestpathlayertopoint.cpp
   processing/qgsalgorithmshortestpathpointtolayer.cpp
   processing/qgsalgorithmshortestpathpointtopoint.cpp
+  processing/qgsalgorithmshpencodinginfo.cpp
   processing/qgsalgorithmsimplify.cpp
   processing/qgsalgorithmsinglesidedbuffer.cpp
   processing/qgsalgorithmslope.cpp

diff --git a/src/analysis/processing/qgsalgorithmsetlayerencoding.cpp b/src/analysis/processing/qgsalgorithmsetlayerencoding.cpp
@@ -0,0 +1,106 @@
+/***************************************************************************
+                         qgsalgorithmsetlayerencoding.cpp
+                         ------------------------------
+    begin                : February 2020
+    copyright            : (C) 2020 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsalgorithmsetlayerencoding.h"
+#include "qgsvectorlayer.h"
+#include "qgsvectordataprovider.h"
+
+///@cond PRIVATE
+
+QString QgsSetLayerEncodingAlgorithm::name() const
+{
+  return QStringLiteral( "setlayerencoding" );
+}
+
+QString QgsSetLayerEncodingAlgorithm::displayName() const
+{
+  return QObject::tr( "Set layer encoding" );
+}
+
+QStringList QgsSetLayerEncodingAlgorithm::tags() const
+{
+  return QObject::tr( "change,alter,attribute,codepage" ).split( ',' );
+}
+
+QString QgsSetLayerEncodingAlgorithm::group() const
+{
+  return QObject::tr( "Vector general" );
+}
+
+QString QgsSetLayerEncodingAlgorithm::groupId() const
+{
+  return QStringLiteral( "vectorgeneral" );
+}
+
+QString QgsSetLayerEncodingAlgorithm::shortHelpString() const
+{
+  return QObject::tr( "This algorithm sets the encoding used for reading a layer's attributes. No permanant changes "
+                      "are made to the layer, rather it affects only how the layer is read during the current session.\n\n"
+                      "Changing the encoding is only supported for some vector layer data sources." );
+}
+
+QString QgsSetLayerEncodingAlgorithm::shortDescription() const
+{
+  return QObject::tr( "Sets the encoding used for reading a layer's attributes" );
+}
+
+QgsSetLayerEncodingAlgorithm *QgsSetLayerEncodingAlgorithm::createInstance() const
+{
+  return new QgsSetLayerEncodingAlgorithm();
+}
+
+void QgsSetLayerEncodingAlgorithm::initAlgorithm( const QVariantMap & )
+{
+  addParameter( new QgsProcessingParameterVectorLayer( QStringLiteral( "INPUT" ), QObject::tr( "Input layer" ) ) );
+  addParameter( new QgsProcessingParameterString( QStringLiteral( "ENCODING" ), QObject::tr( "Encoding" ) ) );
+
+  addOutput( new QgsProcessingOutputVectorLayer( QStringLiteral( "OUTPUT" ), QObject::tr( "Output layer" ) ) );
+}
+
+bool QgsSetLayerEncodingAlgorithm::prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
+{
+  QgsVectorLayer *layer = parameterAsVectorLayer( parameters, QStringLiteral( "INPUT" ), context );
+
+  if ( !layer )
+    throw QgsProcessingException( QObject::tr( "Could not load source layer for %1." ).arg( QStringLiteral( "INPUT" ) ) );
+
+  const QString encoding = parameterAsString( parameters, QStringLiteral( "ENCODING" ), context );
+
+  mOutputId = layer->id();
+  QgsVectorDataProvider *provider = layer->dataProvider();
+
+  if ( provider->capabilities() & QgsVectorDataProvider::SelectEncoding )
+  {
+    layer->setProviderEncoding( encoding );
+  }
+  else
+  {
+    feedback->pushInfo( QObject::tr( "Layer's data provider does not support changing the attribute encoding" ) );
+    // we don't return false here -- rather we allow the algorithm to gracefully handle an attempt to be flexible to different layer sources,
+    // otherwise we potentially break model input flexibility!
+  }
+  return true;
+}
+
+QVariantMap QgsSetLayerEncodingAlgorithm::processAlgorithm( const QVariantMap &, QgsProcessingContext &, QgsProcessingFeedback * )
+{
+  QVariantMap outputs;
+  outputs.insert( QStringLiteral( "OUTPUT" ), mOutputId );
+  return outputs;
+}
+
+///@endcond
diff --git a/src/analysis/processing/qgsalgorithmsetlayerencoding.h b/src/analysis/processing/qgsalgorithmsetlayerencoding.h
@@ -0,0 +1,59 @@
+/***************************************************************************
+                         qgsalgorithmsetlayerencoding.h
+                         ------------------------------
+    begin                : February 2020
+    copyright            : (C) 2020 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSALGORITHMSETLAYERENCODING_H
+#define QGSALGORITHMSETLAYERENCODING_H
+
+#define SIP_NO_FILE
+
+#include "qgis_sip.h"
+#include "qgsprocessingalgorithm.h"
+
+///@cond PRIVATE
+
+/**
+ * Native constant raster algorithm.
+ */
+class QgsSetLayerEncodingAlgorithm : public QgsProcessingAlgorithm
+{
+
+  public:
+
+    QgsSetLayerEncodingAlgorithm() = default;
+    void initAlgorithm( const QVariantMap &configuration = QVariantMap() ) override;
+    QString name() const override;
+    QString displayName() const override;
+    QStringList tags() const override;
+    QString group() const override;
+    QString groupId() const override;
+    QString shortHelpString() const override;
+    QString shortDescription() const override;
+    QgsSetLayerEncodingAlgorithm *createInstance() const override SIP_FACTORY;
+
+  protected:
+
+    bool prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;
+    QVariantMap processAlgorithm( const QVariantMap &parameters,
+                                  QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;
+
+  private:
+    QString mOutputId;
+};
+
+///@endcond PRIVATE
+
+#endif // QGSALGORITHMSETLAYERENCODING_H
diff --git a/src/analysis/processing/qgsalgorithmshpencodinginfo.cpp b/src/analysis/processing/qgsalgorithmshpencodinginfo.cpp
@@ -0,0 +1,111 @@
+/***************************************************************************
+                         qgsalgorithmshpencodinginfo.cpp
+                         -----------------------------
+    begin                : February 2020
+    copyright            : (C) 2020 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsalgorithmshpencodinginfo.h"
+#include "qgsogrutils.h"
+
+///@cond PRIVATE
+
+QString QgsShapefileEncodingInfoAlgorithm::name() const
+{
+  return QStringLiteral( "shpencodinginfo" );
+}
+
+QString QgsShapefileEncodingInfoAlgorithm::displayName() const
+{
+  return QObject::tr( "Extract Shapefile encoding" );
+}
+
+QStringList QgsShapefileEncodingInfoAlgorithm::tags() const
+{
+  return QObject::tr( "shp,codepage,cpg,ldid,information,list,show" ).split( ',' );
+}
+
+QString QgsShapefileEncodingInfoAlgorithm::group() const
+{
+  return QObject::tr( "Vector general" );
+}
+
+QString QgsShapefileEncodingInfoAlgorithm::groupId() const
+{
+  return QStringLiteral( "vectorgeneral" );
+}
+
+QString QgsShapefileEncodingInfoAlgorithm::shortHelpString() const
+{
+  return QObject::tr( "This algorithm extracts the attribute encoding information embedded in a Shapefile.\n\n"
+                      "Both the encoding specified by an optional .cpg file and any encoding details present in the "
+                      ".dbf LDID header block are considered." );
+}
+
+QString QgsShapefileEncodingInfoAlgorithm::shortDescription() const
+{
+  return QObject::tr( "Extracts the attribute encoding information embedded in a Shapefile." );
+}
+
+QgsShapefileEncodingInfoAlgorithm *QgsShapefileEncodingInfoAlgorithm::createInstance() const
+{
+  return new QgsShapefileEncodingInfoAlgorithm();
+}
+
+void QgsShapefileEncodingInfoAlgorithm::initAlgorithm( const QVariantMap & )
+{
+  addParameter( new QgsProcessingParameterFile( QStringLiteral( "INPUT" ), QObject::tr( "Input layer" ), QgsProcessingParameterFile::File,
+                QString(), QVariant(), false, QObject::tr( "Shapefiles (%1)" ).arg( QStringLiteral( "*.shp *.SHP)" ) ) ) );
+
+  addOutput( new QgsProcessingOutputString( QStringLiteral( "ENCODING" ), QObject::tr( "Shapefile Encoding" ) ) );
+  addOutput( new QgsProcessingOutputString( QStringLiteral( "CPG_ENCODING" ), QObject::tr( "CPG Encoding" ) ) );
+  addOutput( new QgsProcessingOutputString( QStringLiteral( "LDID_ENCODING" ), QObject::tr( "LDID Encoding" ) ) );
+}
+
+bool QgsShapefileEncodingInfoAlgorithm::prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
+{
+  const QString path = parameterAsFile( parameters, QStringLiteral( "INPUT" ), context );
+  if ( !QFile::exists( path ) )
+  {
+    feedback->reportError( QObject::tr( "Input Shapefile %1 does not exist" ).arg( path ) );
+    return false;
+  }
+  else
+  {
+    mCpgEncoding = QgsOgrUtils::readShapefileEncodingFromCpg( path );
+    if ( mCpgEncoding.isEmpty() )
+      feedback->pushInfo( QObject::tr( "No encoding information present in CPG file" ) );
+    else
+      feedback->pushInfo( QObject::tr( "Detected encoding from CPG file: %1" ).arg( mCpgEncoding ) );
+
+    mLdidEncoding = QgsOgrUtils::readShapefileEncodingFromLdid( path );
+    if ( mLdidEncoding.isEmpty() )
+      feedback->pushInfo( QObject::tr( "No encoding information present in DBF LDID header" ) );
+    else
+      feedback->pushInfo( QObject::tr( "Detected encoding from DBF LDID header: %1" ).arg( mLdidEncoding ) );
+
+  }
+  return true;
+}
+
+
+QVariantMap QgsShapefileEncodingInfoAlgorithm::processAlgorithm( const QVariantMap &, QgsProcessingContext &, QgsProcessingFeedback * )
+{
+  QVariantMap outputs;
+  outputs.insert( QStringLiteral( "ENCODING" ), mCpgEncoding.isEmpty() ? mLdidEncoding : mCpgEncoding );
+  outputs.insert( QStringLiteral( "CPG_ENCODING" ), mCpgEncoding );
+  outputs.insert( QStringLiteral( "LDID_ENCODING" ), mLdidEncoding );
+  return outputs;
+}
+
+///@endcond
diff --git a/src/analysis/processing/qgsalgorithmshpencodinginfo.h b/src/analysis/processing/qgsalgorithmshpencodinginfo.h
@@ -0,0 +1,62 @@
+/***************************************************************************
+                         qgsalgorithmshpencodinginfo.h
+                         -----------------------------
+    begin                : February 2020
+    copyright            : (C) 2020 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSALGORITHMSHPENCODINGINFO_H
+#define QGSALGORITHMSHPENCODINGINFO_H
+
+#define SIP_NO_FILE
+
+#include "qgis_sip.h"
+#include "qgsprocessingalgorithm.h"
+
+///@cond PRIVATE
+
+/**
+ * Native constant raster algorithm.
+ */
+class QgsShapefileEncodingInfoAlgorithm : public QgsProcessingAlgorithm
+{
+
+  public:
+
+    QgsShapefileEncodingInfoAlgorithm() = default;
+    void initAlgorithm( const QVariantMap &configuration = QVariantMap() ) override;
+    QString name() const override;
+    QString displayName() const override;
+    QStringList tags() const override;
+    QString group() const override;
+    QString groupId() const override;
+    QString shortHelpString() const override;
+    QString shortDescription() const override;
+    QgsShapefileEncodingInfoAlgorithm *createInstance() const override SIP_FACTORY;
+
+  protected:
+    bool prepareAlgorithm( const QVariantMap &parameters,
+                           QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;
+
+    QVariantMap processAlgorithm( const QVariantMap &parameters,
+                                  QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;
+
+  private:
+
+    QString mCpgEncoding;
+    QString mLdidEncoding;
+};
+
+///@endcond PRIVATE
+
+#endif // QGSALGORITHMSHPENCODINGINFO_H
diff --git a/src/analysis/processing/qgsnativealgorithms.cpp b/src/analysis/processing/qgsnativealgorithms.cpp
@@ -125,11 +125,13 @@
 #include "qgsalgorithmsegmentize.h"
 #include "qgsalgorithmserviceareafromlayer.h"
 #include "qgsalgorithmserviceareafrompoint.h"
+#include "qgsalgorithmsetlayerencoding.h"
 #include "qgsalgorithmsetmvalue.h"
 #include "qgsalgorithmsetzvalue.h"
 #include "qgsalgorithmshortestpathlayertopoint.h"
 #include "qgsalgorithmshortestpathpointtolayer.h"
 #include "qgsalgorithmshortestpathpointtopoint.h"
+#include "qgsalgorithmshpencodinginfo.h"
 #include "qgsalgorithmsimplify.h"
 #include "qgsalgorithmsinglesidedbuffer.h"
 #include "qgsalgorithmslope.h"
@@ -325,8 +327,10 @@ void QgsNativeAlgorithms::loadAlgorithms()
   addAlgorithm( new QgsSelectByLocationAlgorithm() );
   addAlgorithm( new QgsServiceAreaFromLayerAlgorithm() );
   addAlgorithm( new QgsServiceAreaFromPointAlgorithm() );
+  addAlgorithm( new QgsSetLayerEncodingAlgorithm() );
   addAlgorithm( new QgsSetMValueAlgorithm() );
   addAlgorithm( new QgsSetZValueAlgorithm() );
+  addAlgorithm( new QgsShapefileEncodingInfoAlgorithm() );
   addAlgorithm( new QgsShortestPathLayerToPointAlgorithm() );
   addAlgorithm( new QgsShortestPathPointToLayerAlgorithm() );
   addAlgorithm( new QgsShortestPathPointToPointAlgorithm() );