Skip to content
Permalink
Browse files

[feature] Automatically translate layer data from ESRI Filegeodatabases

to QGIS layer metadata

Translate as much as possible of the original ESRI metadata across
to the QGIS metadata, so that it's immediately available for
use by users after loading a .gdb file.
  • Loading branch information
nyalldawson committed Apr 29, 2021
1 parent 77696a8 commit 1a9a533fca584689805f94702c443cf1aa0c0e41
Showing with 1,516 additions and 7 deletions.
  1. +39 −0 python/core/auto_generated/metadata/qgsmetadatautils.sip.in
  2. +1 −0 python/core/core_auto.sip
  3. +2 −0 src/core/CMakeLists.txt
  4. +264 −0 src/core/metadata/qgsmetadatautils.cpp
  5. +45 −0 src/core/metadata/qgsmetadatautils.h
  6. +33 −7 src/core/providers/ogr/qgsogrprovider.cpp
  7. +1 −0 tests/src/python/CMakeLists.txt
  8. +21 −0 tests/src/python/test_provider_ogr.py
  9. +83 −0 tests/src/python/test_qgsmetadatautils.py
  10. +1,027 −0 tests/testdata/esri_metadata.xml
  11. 0 tests/testdata/gdb_metadata.gdb/Test.DESKTOP-D021O0E.880.4500.sr.lock
  12. 0 tests/testdata/gdb_metadata.gdb/_gdb.DESKTOP-D021O0E.880.4500.sr.lock
  13. BIN tests/testdata/gdb_metadata.gdb/a00000001.TablesByName.atx
  14. BIN tests/testdata/gdb_metadata.gdb/a00000001.gdbindexes
  15. BIN tests/testdata/gdb_metadata.gdb/a00000001.gdbtable
  16. BIN tests/testdata/gdb_metadata.gdb/a00000001.gdbtablx
  17. BIN tests/testdata/gdb_metadata.gdb/a00000002.gdbtable
  18. BIN tests/testdata/gdb_metadata.gdb/a00000002.gdbtablx
  19. BIN tests/testdata/gdb_metadata.gdb/a00000003.gdbindexes
  20. BIN tests/testdata/gdb_metadata.gdb/a00000003.gdbtable
  21. BIN tests/testdata/gdb_metadata.gdb/a00000003.gdbtablx
  22. BIN tests/testdata/gdb_metadata.gdb/a00000004.CatItemsByPhysicalName.atx
  23. BIN tests/testdata/gdb_metadata.gdb/a00000004.CatItemsByType.atx
  24. BIN tests/testdata/gdb_metadata.gdb/a00000004.FDO_UUID.atx
  25. BIN tests/testdata/gdb_metadata.gdb/a00000004.gdbindexes
  26. BIN tests/testdata/gdb_metadata.gdb/a00000004.gdbtable
  27. BIN tests/testdata/gdb_metadata.gdb/a00000004.gdbtablx
  28. BIN tests/testdata/gdb_metadata.gdb/a00000004.spx
  29. BIN tests/testdata/gdb_metadata.gdb/a00000005.CatItemTypesByName.atx
  30. BIN tests/testdata/gdb_metadata.gdb/a00000005.CatItemTypesByParentTypeID.atx
  31. BIN tests/testdata/gdb_metadata.gdb/a00000005.CatItemTypesByUUID.atx
  32. BIN tests/testdata/gdb_metadata.gdb/a00000005.gdbindexes
  33. BIN tests/testdata/gdb_metadata.gdb/a00000005.gdbtable
  34. BIN tests/testdata/gdb_metadata.gdb/a00000005.gdbtablx
  35. BIN tests/testdata/gdb_metadata.gdb/a00000006.CatRelsByDestinationID.atx
  36. BIN tests/testdata/gdb_metadata.gdb/a00000006.CatRelsByOriginID.atx
  37. BIN tests/testdata/gdb_metadata.gdb/a00000006.CatRelsByType.atx
  38. BIN tests/testdata/gdb_metadata.gdb/a00000006.FDO_UUID.atx
  39. BIN tests/testdata/gdb_metadata.gdb/a00000006.gdbindexes
  40. BIN tests/testdata/gdb_metadata.gdb/a00000006.gdbtable
  41. BIN tests/testdata/gdb_metadata.gdb/a00000006.gdbtablx
  42. BIN tests/testdata/gdb_metadata.gdb/a00000007.CatRelTypesByBackwardLabel.atx
  43. BIN tests/testdata/gdb_metadata.gdb/a00000007.CatRelTypesByDestItemTypeID.atx
  44. BIN tests/testdata/gdb_metadata.gdb/a00000007.CatRelTypesByForwardLabel.atx
  45. BIN tests/testdata/gdb_metadata.gdb/a00000007.CatRelTypesByName.atx
  46. BIN tests/testdata/gdb_metadata.gdb/a00000007.CatRelTypesByOriginItemTypeID.atx
  47. BIN tests/testdata/gdb_metadata.gdb/a00000007.CatRelTypesByUUID.atx
  48. BIN tests/testdata/gdb_metadata.gdb/a00000007.gdbindexes
  49. BIN tests/testdata/gdb_metadata.gdb/a00000007.gdbtable
  50. BIN tests/testdata/gdb_metadata.gdb/a00000007.gdbtablx
  51. BIN tests/testdata/gdb_metadata.gdb/a00000009.gdbindexes
  52. BIN tests/testdata/gdb_metadata.gdb/a00000009.gdbtable
  53. BIN tests/testdata/gdb_metadata.gdb/a00000009.gdbtablx
  54. BIN tests/testdata/gdb_metadata.gdb/a00000009.spx
  55. BIN tests/testdata/gdb_metadata.gdb/gdb
  56. BIN tests/testdata/gdb_metadata.gdb/timestamps
@@ -0,0 +1,39 @@
/************************************************************************
* This file has been generated automatically from *
* *
* src/core/metadata/qgsmetadatautils.h *
* *
* Do not edit manually ! Edit header and run scripts/sipify.pl again *
************************************************************************/





class QgsMetadataUtils
{
%Docstring(signature="appended")
Contains utility functions for working with metadata.

.. versionadded:: 3.20
%End

%TypeHeaderCode
#include "qgsmetadatautils.h"
%End
public:

static QgsLayerMetadata convertFromEsri( const QDomDocument &document );
%Docstring
Converts ESRI layer metadata to :py:class:`QgsLayerMetadata`.
%End

};

/************************************************************************
* This file has been generated automatically from *
* *
* src/core/metadata/qgsmetadatautils.h *
* *
* Do not edit manually ! Edit header and run scripts/sipify.pl again *
************************************************************************/
@@ -438,6 +438,7 @@
%Include auto_generated/metadata/qgslayermetadata.sip
%Include auto_generated/metadata/qgslayermetadataformatter.sip
%Include auto_generated/metadata/qgslayermetadatavalidator.sip
%Include auto_generated/metadata/qgsmetadatautils.sip
%Include auto_generated/metadata/qgsprojectmetadata.sip
%Include auto_generated/network/qgsblockingnetworkrequest.sip
%Include auto_generated/network/qgsfiledownloader.sip
@@ -120,6 +120,7 @@ set(QGIS_CORE_SRCS
metadata/qgslayermetadata.cpp
metadata/qgslayermetadatavalidator.cpp
metadata/qgslayermetadataformatter.cpp
metadata/qgsmetadatautils.cpp
metadata/qgsprojectmetadata.cpp

numericformats/qgsbasicnumericformat.cpp
@@ -1363,6 +1364,7 @@ set(QGIS_CORE_HDRS
metadata/qgslayermetadata.h
metadata/qgslayermetadataformatter.h
metadata/qgslayermetadatavalidator.h
metadata/qgsmetadatautils.h
metadata/qgsprojectmetadata.h

network/qgsblockingnetworkrequest.h
@@ -0,0 +1,264 @@
/***************************************************************************
qgsmetadatautils.cpp
-------------------
begin : April 2021
copyright : (C) 2021 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#include "qgsmetadatautils.h"
#include "qgslayermetadata.h"

#include <QDomDocument>
#include <QTextDocumentFragment>

QgsLayerMetadata QgsMetadataUtils::convertFromEsri( const QDomDocument &document )
{
QgsLayerMetadata metadata;
const QDomElement metadataElem = document.firstChildElement( QStringLiteral( "metadata" ) );

const QDomElement esri = metadataElem.firstChildElement( QStringLiteral( "Esri" ) );
const QDomElement dataProperties = esri.firstChildElement( QStringLiteral( "DataProperties" ) );
const QDomElement itemProps = dataProperties.firstChildElement( QStringLiteral( "itemProps" ) );
metadata.setIdentifier( itemProps.firstChildElement( QStringLiteral( "itemName" ) ).text() );

const QDomElement dataIdInfo = metadataElem.firstChildElement( QStringLiteral( "dataIdInfo" ) );

// title
const QDomElement idCitation = dataIdInfo.firstChildElement( QStringLiteral( "idCitation" ) );
const QString title = idCitation.firstChildElement( QStringLiteral( "resTitle" ) ).text();
metadata.setTitle( title );

// abstract
const QDomElement idAbs = dataIdInfo.firstChildElement( QStringLiteral( "idAbs" ) );
const QString abstractPlainText = QTextDocumentFragment::fromHtml( idAbs.text() ).toPlainText();
metadata.setAbstract( abstractPlainText );

// purpose
const QDomElement idPurp = dataIdInfo.firstChildElement( QStringLiteral( "idPurp" ) );
const QString purposePlainText = QTextDocumentFragment::fromHtml( idPurp.text() ).toPlainText();
if ( !metadata.abstract().isEmpty() )
metadata.setAbstract( metadata.abstract() + QStringLiteral( "\n\n" ) + purposePlainText );
else
metadata.setAbstract( purposePlainText );

// supplementary info
const QDomElement suppInfo = dataIdInfo.firstChildElement( QStringLiteral( "suppInfo" ) );
const QString suppInfoPlainText = QTextDocumentFragment::fromHtml( suppInfo.text() ).toPlainText();
if ( !metadata.abstract().isEmpty() )
metadata.setAbstract( metadata.abstract() + QStringLiteral( "\n\n" ) + suppInfoPlainText );
else
metadata.setAbstract( suppInfoPlainText );

// language
const QDomElement dataLang = dataIdInfo.firstChildElement( QStringLiteral( "dataLang" ) );
const QDomElement languageCode = dataLang.firstChildElement( QStringLiteral( "languageCode" ) );
const QString language = languageCode.attribute( QStringLiteral( "value" ) ).toUpper();
metadata.setLanguage( language );

// keywords
QDomElement searchKeys = dataIdInfo.firstChildElement( QStringLiteral( "searchKeys" ) );
QStringList keywords;
while ( !searchKeys.isNull() )
{
QDomElement keyword = searchKeys.firstChildElement( QStringLiteral( "keyword" ) );
while ( !keyword.isNull() )
{
keywords << keyword.text();
keyword = keyword.nextSiblingElement( QStringLiteral( "keyword" ) );
}

searchKeys = searchKeys.nextSiblingElement( QStringLiteral( "searchKeys" ) );
}
if ( !keywords.empty() )
metadata.addKeywords( QObject::tr( "Search keys" ), keywords );

// categories
QDomElement themeKeys = dataIdInfo.firstChildElement( QStringLiteral( "themeKeys" ) );
QStringList categories;
while ( !themeKeys.isNull() )
{
QDomElement themeKeyword = themeKeys.firstChildElement( QStringLiteral( "keyword" ) );
while ( !themeKeyword.isNull() )
{
categories << themeKeyword.text();
themeKeyword = themeKeyword.nextSiblingElement( QStringLiteral( "keyword" ) );
}
themeKeys = themeKeys.nextSiblingElement( QStringLiteral( "themeKeys" ) );
}
if ( !categories.isEmpty() )
metadata.setCategories( categories );

QgsLayerMetadata::Extent extent;

// pubDate
const QDomElement date = idCitation.firstChildElement( QStringLiteral( "date" ) );
const QString pubDate = date.firstChildElement( QStringLiteral( "pubDate" ) ).text();
const QDateTime publicationDate = QDateTime::fromString( pubDate, Qt::ISODate );
if ( publicationDate.isValid() )
{
extent.setTemporalExtents( { publicationDate, QDateTime() } );
}

//crs
QgsCoordinateReferenceSystem crs;
const QDomElement refSysInfo = metadataElem.firstChildElement( QStringLiteral( "refSysInfo" ) );
if ( !refSysInfo.isNull() )
{
const QDomElement refSystem = refSysInfo.firstChildElement( QStringLiteral( "RefSystem" ) );
const QDomElement refSysID = refSystem.firstChildElement( QStringLiteral( "refSysID" ) );
const QString code = refSysID.firstChildElement( QStringLiteral( "identCode" ) ).attribute( QStringLiteral( "code" ) );
const QString auth = refSysID.firstChildElement( QStringLiteral( "idCodeSpace" ) ).text();

crs = QgsCoordinateReferenceSystem( QStringLiteral( "%1:%2" ).arg( auth, code ) );
if ( crs.isValid() )
metadata.setCrs( crs );
}

// extent
QDomElement dataExt = dataIdInfo.firstChildElement( QStringLiteral( "dataExt" ) );
while ( !dataExt.isNull() )
{
const QDomElement geoEle = dataExt.firstChildElement( QStringLiteral( "geoEle" ) );
if ( !geoEle.isNull() )
{
const QDomElement geoBndBox = geoEle.firstChildElement( QStringLiteral( "GeoBndBox" ) );
const double west = geoBndBox.firstChildElement( QStringLiteral( "westBL" ) ).text().toDouble();
const double east = geoBndBox.firstChildElement( QStringLiteral( "eastBL" ) ).text().toDouble();
const double south = geoBndBox.firstChildElement( QStringLiteral( "northBL" ) ).text().toDouble();
const double north = geoBndBox.firstChildElement( QStringLiteral( "southBL" ) ).text().toDouble();

QgsLayerMetadata::SpatialExtent spatialExtent;
spatialExtent.extentCrs = crs.isValid() ? crs : QgsCoordinateReferenceSystem( QStringLiteral( "EPSG:4326" ) );
spatialExtent.bounds = QgsBox3d( west, south, 0, east, north, 0 );

extent.setSpatialExtents( { spatialExtent } );
break;
}
dataExt = dataExt.nextSiblingElement( QStringLiteral( "dataExt" ) );
}

metadata.setExtent( extent );

// licenses, constraints
QStringList licenses;
QStringList rights;
QgsLayerMetadata::ConstraintList constraints;
QDomElement resConst = dataIdInfo.firstChildElement( QStringLiteral( "resConst" ) );
while ( !resConst.isNull() )
{
QDomElement legConsts = resConst.firstChildElement( QStringLiteral( "LegConsts" ) );
while ( !legConsts.isNull() )
{
const QString restrictCd = legConsts.firstChildElement( QStringLiteral( "useConsts" ) ).firstChildElement( QStringLiteral( "RestrictCd" ) ).attribute( QStringLiteral( "value" ) );

if ( restrictCd.compare( QLatin1String( "005" ) ) == 0 )
{
licenses << QTextDocumentFragment::fromHtml( legConsts.firstChildElement( QStringLiteral( "useLimit" ) ).text() ).toPlainText();
}
else if ( restrictCd.compare( QLatin1String( "006" ) ) == 0 )
{
rights << QTextDocumentFragment::fromHtml( legConsts.firstChildElement( QStringLiteral( "useLimit" ) ).text() ).toPlainText();
}
legConsts = legConsts.nextSiblingElement( QStringLiteral( "LegConsts" ) );
}

QDomElement secConsts = resConst.firstChildElement( QStringLiteral( "SecConsts" ) );
while ( !secConsts.isNull() )
{
QgsLayerMetadata::Constraint constraint;
constraint.type = QObject::tr( "Security constraints" );
constraint.constraint = QTextDocumentFragment::fromHtml( secConsts.firstChildElement( QStringLiteral( "userNote" ) ).text() ).toPlainText();
constraints << constraint;
secConsts = secConsts.nextSiblingElement( QStringLiteral( "SecConsts" ) );
}

QDomElement consts = resConst.firstChildElement( QStringLiteral( "Consts" ) );
while ( !consts.isNull() )
{
QgsLayerMetadata::Constraint constraint;
constraint.type = QObject::tr( "Limitations of use" );
constraint.constraint = QTextDocumentFragment::fromHtml( consts.firstChildElement( QStringLiteral( "useLimit" ) ).text() ).toPlainText();
constraints << constraint;
consts = consts.nextSiblingElement( QStringLiteral( "Consts" ) );
}

resConst = resConst.nextSiblingElement( QStringLiteral( "resConst" ) );
}

const QDomElement idCredit = dataIdInfo.firstChildElement( QStringLiteral( "idCredit" ) );
const QString credit = idCredit.text();
if ( !credit.isEmpty() )
rights << credit;

metadata.setLicenses( licenses );
metadata.setRights( rights );
metadata.setConstraints( constraints );

// links
const QDomElement distInfo = metadataElem.firstChildElement( QStringLiteral( "distInfo" ) );
const QDomElement distributor = distInfo.firstChildElement( QStringLiteral( "distributor" ) );

QDomElement distorTran = distributor.firstChildElement( QStringLiteral( "distorTran" ) );
while ( !distorTran.isNull() )
{
const QDomElement onLineSrc = distorTran.firstChildElement( QStringLiteral( "onLineSrc" ) );
if ( !onLineSrc.isNull() )
{
QgsAbstractMetadataBase::Link link;
link.url = onLineSrc.firstChildElement( QStringLiteral( "linkage" ) ).text();

const QDomElement distorFormat = distributor.firstChildElement( QStringLiteral( "distorFormat" ) );
link.name = distorFormat.firstChildElement( QStringLiteral( "formatName" ) ).text();
link.type = distorFormat.firstChildElement( QStringLiteral( "formatSpec" ) ).text();
metadata.addLink( link );
}

distorTran = distorTran.nextSiblingElement( QStringLiteral( "distorTran" ) );
}

// lineage
const QDomElement dqInfo = metadataElem.firstChildElement( QStringLiteral( "dqInfo" ) );
const QDomElement dataLineage = dqInfo.firstChildElement( QStringLiteral( "dataLineage" ) );
const QString statement = QTextDocumentFragment::fromHtml( dataLineage.firstChildElement( QStringLiteral( "statement" ) ).text() ).toPlainText();
if ( !statement.isEmpty() )
metadata.addHistoryItem( statement );

QDomElement dataSource = dataLineage.firstChildElement( QStringLiteral( "dataSource" ) );
while ( !dataSource.isNull() )
{
metadata.addHistoryItem( QObject::tr( "Data source: %1" ).arg( QTextDocumentFragment::fromHtml( dataSource.firstChildElement( QStringLiteral( "srcDesc" ) ).text() ).toPlainText() ) );
dataSource = dataSource.nextSiblingElement( QStringLiteral( "dataSource" ) );
}

// contacts
const QDomElement mdContact = metadataElem.firstChildElement( QStringLiteral( "mdContact" ) );
if ( !mdContact.isNull() )
{
QgsAbstractMetadataBase::Contact contact;
contact.name = mdContact.firstChildElement( QStringLiteral( "rpIndName" ) ).text();
contact.organization = mdContact.firstChildElement( QStringLiteral( "rpOrgName" ) ).text();
contact.position = mdContact.firstChildElement( QStringLiteral( "rpPosName" ) ).text();

const QString role = mdContact.firstChildElement( QStringLiteral( "role" ) ).firstChildElement( QStringLiteral( "RoleCd" ) ).attribute( QStringLiteral( "value" ) );
if ( role == QLatin1String( "007" ) )
contact.role = QObject::tr( "Point of contact" );

const QDomElement rpCntInfo = mdContact.firstChildElement( QStringLiteral( "rpCntInfo" ) );
contact.email = rpCntInfo.firstChildElement( QStringLiteral( "cntAddress" ) ).firstChildElement( QStringLiteral( "eMailAdd" ) ).text();
contact.voice = rpCntInfo.firstChildElement( QStringLiteral( "cntPhone" ) ).firstChildElement( QStringLiteral( "voiceNum" ) ).text();

metadata.addContact( contact );
}

return metadata;
}
@@ -0,0 +1,45 @@
/***************************************************************************
qgsmetadatautils.h
-------------------
begin : April 2021
copyright : (C) 2021 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#ifndef QGSMETADATAUTILS_H
#define QGSMETADATAUTILS_H

#include "qgis_sip.h"
#include "qgis_core.h"

class QgsLayerMetadata;
class QDomDocument;

/**
* \ingroup core
* \class QgsMetadataUtils
* \brief Contains utility functions for working with metadata.
*
* \since QGIS 3.20
*/
class CORE_EXPORT QgsMetadataUtils
{
public:

/**
* Converts ESRI layer metadata to QgsLayerMetadata.
*/
static QgsLayerMetadata convertFromEsri( const QDomDocument &document );

};

#endif // QGSMETADATAUTILS_H

0 comments on commit 1a9a533

Please sign in to comment.