Skip to content
Permalink
Browse files

[FEATURE] Binary blob support for OGR provider

Instead of converting binary fields to truncated strings, we instead
store their contents as QByteArray values, allowing the original binary
content to be retrieved.

This allows for plugins and scripts to utilise binary fields,
such as extracting their contents.
  • Loading branch information
nyalldawson committed Nov 9, 2018
1 parent 8ae1880 commit 5c27b7da5005e055bbe8b1f7191c4ca85ab3f0a9
Showing with 59 additions and 1 deletion.
  1. +4 −0 src/core/qgsfield.cpp
  2. +11 −0 src/core/qgsogrutils.cpp
  3. +20 −0 src/providers/ogr/qgsogrprovider.cpp
  4. +5 −0 tests/src/core/testqgsfield.cpp
  5. +19 −1 tests/src/python/test_provider_ogr.py
  6. BIN tests/testdata/attachments.gdb/a00000001.TablesByName.atx
  7. BIN tests/testdata/attachments.gdb/a00000001.freelist
  8. BIN tests/testdata/attachments.gdb/a00000001.gdbindexes
  9. BIN tests/testdata/attachments.gdb/a00000001.gdbtable
  10. BIN tests/testdata/attachments.gdb/a00000001.gdbtablx
  11. BIN tests/testdata/attachments.gdb/a00000002.gdbtable
  12. BIN tests/testdata/attachments.gdb/a00000002.gdbtablx
  13. BIN tests/testdata/attachments.gdb/a00000003.gdbindexes
  14. BIN tests/testdata/attachments.gdb/a00000003.gdbtable
  15. BIN tests/testdata/attachments.gdb/a00000003.gdbtablx
  16. BIN tests/testdata/attachments.gdb/a00000004.CatItemsByPhysicalName.atx
  17. BIN tests/testdata/attachments.gdb/a00000004.CatItemsByType.atx
  18. BIN tests/testdata/attachments.gdb/a00000004.FDO_UUID.atx
  19. BIN tests/testdata/attachments.gdb/a00000004.freelist
  20. BIN tests/testdata/attachments.gdb/a00000004.gdbindexes
  21. BIN tests/testdata/attachments.gdb/a00000004.gdbtable
  22. BIN tests/testdata/attachments.gdb/a00000004.gdbtablx
  23. BIN tests/testdata/attachments.gdb/a00000004.spx
  24. BIN tests/testdata/attachments.gdb/a00000005.CatItemTypesByName.atx
  25. BIN tests/testdata/attachments.gdb/a00000005.CatItemTypesByParentTypeID.atx
  26. BIN tests/testdata/attachments.gdb/a00000005.CatItemTypesByUUID.atx
  27. BIN tests/testdata/attachments.gdb/a00000005.gdbindexes
  28. BIN tests/testdata/attachments.gdb/a00000005.gdbtable
  29. BIN tests/testdata/attachments.gdb/a00000005.gdbtablx
  30. BIN tests/testdata/attachments.gdb/a00000006.CatRelsByDestinationID.atx
  31. BIN tests/testdata/attachments.gdb/a00000006.CatRelsByOriginID.atx
  32. BIN tests/testdata/attachments.gdb/a00000006.CatRelsByType.atx
  33. BIN tests/testdata/attachments.gdb/a00000006.FDO_UUID.atx
  34. BIN tests/testdata/attachments.gdb/a00000006.freelist
  35. BIN tests/testdata/attachments.gdb/a00000006.gdbindexes
  36. BIN tests/testdata/attachments.gdb/a00000006.gdbtable
  37. BIN tests/testdata/attachments.gdb/a00000006.gdbtablx
  38. BIN tests/testdata/attachments.gdb/a00000007.CatRelTypesByBackwardLabel.atx
  39. BIN tests/testdata/attachments.gdb/a00000007.CatRelTypesByDestItemTypeID.atx
  40. BIN tests/testdata/attachments.gdb/a00000007.CatRelTypesByForwardLabel.atx
  41. BIN tests/testdata/attachments.gdb/a00000007.CatRelTypesByName.atx
  42. BIN tests/testdata/attachments.gdb/a00000007.CatRelTypesByOriginItemTypeID.atx
  43. BIN tests/testdata/attachments.gdb/a00000007.CatRelTypesByUUID.atx
  44. BIN tests/testdata/attachments.gdb/a00000007.gdbindexes
  45. BIN tests/testdata/attachments.gdb/a00000007.gdbtable
  46. BIN tests/testdata/attachments.gdb/a00000007.gdbtablx
  47. BIN tests/testdata/attachments.gdb/a00000016.gdbindexes
  48. BIN tests/testdata/attachments.gdb/a00000016.gdbtable
  49. BIN tests/testdata/attachments.gdb/a00000016.gdbtablx
  50. BIN tests/testdata/attachments.gdb/a00000016.spx
  51. BIN tests/testdata/attachments.gdb/a00000017.GDB_5_REL_OBJECTID.atx
  52. BIN tests/testdata/attachments.gdb/a00000017.gdbindexes
  53. BIN tests/testdata/attachments.gdb/a00000017.gdbtable
  54. BIN tests/testdata/attachments.gdb/a00000017.gdbtablx
  55. BIN tests/testdata/attachments.gdb/gdb
  56. BIN tests/testdata/attachments.gdb/timestamps
@@ -259,6 +259,10 @@ QString QgsField::displayString( const QVariant &v ) const
QJsonDocument doc = QJsonDocument::fromVariant( v );
return QString::fromUtf8( doc.toJson().data() );
}
else if ( d->type == QVariant::ByteArray )
{
return QObject::tr( "BLOB" );
}
// Fallback if special rules do not apply
return v.toString();
}
@@ -221,6 +221,17 @@ QVariant QgsOgrUtils::getOgrFeatureAttribute( OGRFeatureH ogrFet, const QgsField
value = QDateTime( QDate( year, month, day ), QTime( hour, minute, second ) );
}
break;

case QVariant::ByteArray:
{
int size = 0;
const GByte *b = OGR_F_GetFieldAsBinary( ogrFet, attIndex, &size );
QByteArray ba = QByteArray::fromRawData( reinterpret_cast<const char *>( b ), size );
ba.detach();
value = ba;
break;
}

default:
Q_ASSERT_X( false, "QgsOgrUtils::getOgrFeatureAttribute", "unsupported field type" );
if ( ok )
@@ -1028,6 +1028,11 @@ void QgsOgrProvider::loadFields()
case OFTDateTime:
varType = QVariant::DateTime;
break;

case OFTBinary:
varType = QVariant::ByteArray;
break;

case OFTString:
default:
varType = QVariant::String; // other unsupported, leave it as a string
@@ -1504,6 +1509,13 @@ bool QgsOgrProvider::addFeaturePrivate( QgsFeature &f, Flags flags )
OGR_F_SetFieldString( feature.get(), ogrAttId, textEncoding()->fromUnicode( attrVal.toString() ).constData() );
break;

case OFTBinary:
{
const QByteArray ba = attrVal.toByteArray();
OGR_F_SetFieldBinary( feature.get(), ogrAttId, ba.size(), const_cast< GByte * >( reinterpret_cast< const GByte * >( ba.data() ) ) );
break;
}

default:
QgsMessageLog::logMessage( tr( "type %1 for attribute %2 not found" ).arg( type ).arg( qgisAttId ), tr( "OGR" ) );
break;
@@ -2059,6 +2071,14 @@ bool QgsOgrProvider::changeAttributeValues( const QgsChangedAttributesMap &attr_
case OFTString:
OGR_F_SetFieldString( of.get(), f, textEncoding()->fromUnicode( it2->toString() ).constData() );
break;

case OFTBinary:
{
const QByteArray ba = it2->toByteArray();
OGR_F_SetFieldBinary( of.get(), f, ba.size(), const_cast< GByte * >( reinterpret_cast< const GByte * >( ba.data() ) ) );
break;
}

default:
pushError( tr( "Type %1 of attribute %2 of feature %3 unknown." ).arg( type ).arg( fid ).arg( f ) );
break;
@@ -384,6 +384,11 @@ void TestQgsField::displayString()
QCOMPARE( longField.displayString( 5 ), QString( "5" ) );
QCOMPARE( longField.displayString( 599999898999LL ), QString( "599999898999" ) );

// binary field
QgsField binaryField( QStringLiteral( "binary" ), QVariant::ByteArray, QStringLiteral( "Binary" ) );
QString testBAString( QStringLiteral( "test string" ) );
QByteArray testBA( testBAString.toLocal8Bit() );
QCOMPARE( binaryField.displayString( testBA ), QStringLiteral( "BLOB" ) );
}

void TestQgsField::convertCompatible()
@@ -16,9 +16,10 @@
import shutil
import sys
import tempfile
import hashlib

from osgeo import gdal, ogr # NOQA
from qgis.PyQt.QtCore import QVariant
from qgis.PyQt.QtCore import QVariant, QByteArray
from qgis.core import (QgsApplication,
QgsRectangle,
QgsProviderRegistry,
@@ -489,6 +490,23 @@ def testOSM(self):
self.assertEqual(f.id(), 8)
del it

def testBinaryField(self):
source = os.path.join(TEST_DATA_DIR, 'attachments.gdb')
vl = QgsVectorLayer(source + "|layername=points__ATTACH")
self.assertTrue(vl.isValid())

fields = vl.fields()
data_field = fields[fields.lookupField('DATA')]
self.assertEqual(data_field.type(), QVariant.ByteArray)
self.assertEqual(data_field.typeName(), 'Binary')

features = {f['ATTACHMENTID']: f for f in vl.getFeatures()}
self.assertEqual(len(features), 2)
self.assertIsInstance(features[1]['DATA'], QByteArray)
self.assertEqual(hashlib.md5(features[1]['DATA'].data()).hexdigest(), 'ef3dbc530cc39a545832a6c82aac57b6')
self.assertIsInstance(features[2]['DATA'], QByteArray)
self.assertEqual(hashlib.md5(features[2]['DATA'].data()).hexdigest(), '4b952b80e4288ca5111be2f6dd5d6809')


if __name__ == '__main__':
unittest.main()
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.

0 comments on commit 5c27b7d

Please sign in to comment.
You can’t perform that action at this time.