Skip to content
Permalink
Browse files

Greatly decrease the number of round-trips to the postgres server per…

…formed when reading and creating fields in the postgres provider
  • Loading branch information
manisandro authored and jef-n committed Mar 6, 2015
1 parent 914ecc9 commit 09f410cab089be96e4bad73513357d0f9f80b349
Showing with 84 additions and 41 deletions.
  1. +76 −41 src/providers/postgres/qgspostgresprovider.cpp
  2. +8 −0 src/providers/postgres/qgspostgresprovider.h
@@ -658,8 +658,64 @@ bool QgsPostgresProvider::loadFields()

QSet<QString> fields;

// The queries inside this loop could possibly be combined into one
// single query - this would make the code run faster.
/* Collect type info */
sql = "SELECT oid, typname,typtype,typelem,typlen FROM pg_type";
QgsPostgresResult typeResult = connectionRO()->PQexec( sql );
QMap<int, PGTypeInfo> typeMap;
for ( int i = 0; i < typeResult.PQntuples(); ++i )
{
PGTypeInfo typeInfo =
{
/*typeName = */ typeResult.PQgetvalue( i, 1 ),
/*typeType = */ typeResult.PQgetvalue( i, 2 ),
/*typeElem = */ typeResult.PQgetvalue( i, 3 ),
/*typeLen = */ typeResult.PQgetvalue( i, 4 ).toInt()
};
typeMap.insert( typeResult.PQgetvalue( i, 0 ).toInt(), typeInfo );
}

/* Collect table oids */
QSet<int> tableoids;
for ( int i = 0; i < result.PQnfields(); i++ )
{
int tableoid = result.PQftable( i );
if ( tableoid > 0 )
{
tableoids.insert( tableoid );
}
}
QStringList tableoidsList;
foreach ( int tableoid, tableoids )
{
tableoidsList.append( QString::number( tableoid ) );
}

QString tableoidsFilter = "(" + tableoidsList.join( "," ) + ")";

/* Collect formatted field types */
sql = "SELECT attrelid, attnum, pg_catalog.format_type(atttypid,atttypmod) FROM pg_attribute WHERE attrelid IN " + tableoidsFilter;
QgsPostgresResult fmtFieldTypeResult = connectionRO()->PQexec( sql );
QMap<int, QMap<int, QString> > fmtFieldTypeMap;
for ( int i = 0; i < fmtFieldTypeResult.PQntuples(); ++i )
{
int attrelid = fmtFieldTypeResult.PQgetvalue( i, 0 ).toInt();
int attnum = fmtFieldTypeResult.PQgetvalue( i, 1 ).toInt();
QString formatType = fmtFieldTypeResult.PQgetvalue( i, 2 );
fmtFieldTypeMap[attrelid][attnum] = formatType;
}

/* Collect descriptions */
sql = "SELECT objoid, objsubid, description FROM pg_description WHERE objoid IN " + tableoidsFilter;
QgsPostgresResult descrResult = connectionRO()->PQexec( sql );
QMap<int, QMap<int, QString> > descrMap;
for ( int i = 0; i < descrResult.PQntuples(); ++i )
{
int objoid = descrResult.PQgetvalue( i, 0 ).toInt();
int objsubid = descrResult.PQgetvalue( i, 1 ).toInt();
QString descr = descrResult.PQgetvalue( i, 2 );
descrMap[objoid][objsubid] = descr;
}

mAttributeFields.clear();
for ( int i = 0; i < result.PQnfields(); i++ )
{
@@ -668,40 +724,17 @@ bool QgsPostgresProvider::loadFields()
continue;

int fldtyp = result.PQftype( i );
QString typOid = QString().setNum( fldtyp );
int fieldPrec = -1;
QString fieldComment( "" );
int tableoid = result.PQftable( i );
int attnum = result.PQftablecol( i );

sql = QString( "SELECT typname,typtype,typelem,typlen FROM pg_type WHERE oid=%1" ).arg( typOid );
// just oid; needs more work to support array type
// "oid = (SELECT Distinct typelem FROM pg_type WHERE " //needs DISTINCT to guard against 2 or more rows on int2
// "typelem = " + typOid + " AND typlen = -1)";

QgsPostgresResult oidResult = connectionRO()->PQexec( sql );
QString fieldTypeName = oidResult.PQgetvalue( 0, 0 );
QString fieldTType = oidResult.PQgetvalue( 0, 1 );
QString fieldElem = oidResult.PQgetvalue( 0, 2 );
int fieldSize = oidResult.PQgetvalue( 0, 3 ).toInt();

QString formattedFieldType;
if ( tableoid > 0 )
{
sql = QString( "SELECT pg_catalog.format_type(atttypid,atttypmod) FROM pg_attribute WHERE attrelid=%1 AND attnum=%2" )
.arg( tableoid ).arg( quotedValue( attnum ) );

QgsPostgresResult tresult = connectionRO()->PQexec( sql );
if ( tresult.PQntuples() > 0 )
formattedFieldType = tresult.PQgetvalue( 0, 0 );

sql = QString( "SELECT description FROM pg_description WHERE objoid=%1 AND objsubid=%2" )
.arg( tableoid ).arg( attnum );
const PGTypeInfo& typeInfo = typeMap.value( fldtyp );
QString fieldTypeName = typeInfo.typeName;
QString fieldTType = typeInfo.typeType;
int fieldSize = typeInfo.typeLen;

tresult = connectionRO()->PQexec( sql );
if ( tresult.PQntuples() > 0 )
fieldComment = tresult.PQgetvalue( 0, 0 );
}
QString formattedFieldType = fmtFieldTypeMap[tableoid][attnum];
QString fieldComment = descrMap[tableoid][attnum];

QVariant::Type fieldType;

@@ -1919,6 +1952,7 @@ bool QgsPostgresProvider::addAttributes( const QList<QgsField> &attributes )
{
conn->begin();

QString sql = QString( "ALTER TABLE %1 " ).arg( mQuery );
for ( QList<QgsField>::const_iterator iter = attributes.begin(); iter != attributes.end(); ++iter )
{
QString type = iter->typeName();
@@ -1932,18 +1966,19 @@ bool QgsPostgresProvider::addAttributes( const QList<QgsField> &attributes )
if ( iter->length() > 0 && iter->precision() >= 0 )
type = QString( "%1(%2,%3)" ).arg( type ).arg( iter->length() ).arg( iter->precision() );
}
sql.append( QString( "ADD COLUMN %1 %2, " ).arg( quotedIdentifier( iter->name() ) ).arg( type ) );
}
sql.chop( 2 ); /* ", " */
sql.append( ";" );
QgsDebugMsg( sql );

QString sql = QString( "ALTER TABLE %1 ADD COLUMN %2 %3" )
.arg( mQuery )
.arg( quotedIdentifier( iter->name() ) )
.arg( type );
QgsDebugMsg( sql );

//send sql statement and do error handling
QgsPostgresResult result = conn->PQexec( sql );
if ( result.PQresultStatus() != PGRES_COMMAND_OK )
throw PGException( result );
//send sql statement and do error handling
QgsPostgresResult result = conn->PQexec( sql );
if ( result.PQresultStatus() != PGRES_COMMAND_OK )
throw PGException( result );

for ( QList<QgsField>::const_iterator iter = attributes.begin(); iter != attributes.end(); ++iter )
{
if ( !iter->comment().isEmpty() )
{
sql = QString( "COMMENT ON COLUMN %1.%2 IS %3" )
@@ -455,6 +455,14 @@ class QgsPostgresProvider : public QgsVectorDataProvider
QString mWhat;
};

struct PGTypeInfo
{
QString typeName;
QString typeType;
QString typeElem;
int typeLen;
};

// A function that determines if the given schema.table.column
// contains unqiue entries
bool uniqueData( QString query, QString colName );

0 comments on commit 09f410c

Please sign in to comment.
You can’t perform that action at this time.