Skip to content
Permalink
Browse files

[delimitedtext] Fix datetime,date field type ignored (in CSVT et cie)

  • Loading branch information
nirvn committed Jun 10, 2020
1 parent 5385ae6 commit 8192be7659087b37a41895491a0457791de306aa
@@ -501,6 +501,21 @@ void QgsDelimitedTextFeatureIterator::fetchAttribute( QgsFeature &feature, int f
}
break;
}
case QVariant::DateTime:
{
val = QVariant( QDateTime::fromString( value, Qt::ISODate ) );
break;
}
case QVariant::Date:
{
val = QVariant( QDate::fromString( value, Qt::ISODate ) );
break;
}
case QVariant::Time:
{
val = QVariant( QTime::fromString( value ) );
break;
}
default:
val = QVariant( value );
break;
@@ -68,6 +68,11 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr
<< QgsVectorDataProvider::NativeType( tr( "Whole number (integer - 64 bit)" ), QStringLiteral( "int8" ), QVariant::LongLong )
<< QgsVectorDataProvider::NativeType( tr( "Decimal number (double)" ), QStringLiteral( "double precision" ), QVariant::Double, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Text, unlimited length (text)" ), QStringLiteral( "text" ), QVariant::String, -1, -1, -1, -1 )

// date type
<< QgsVectorDataProvider::NativeType( tr( "Date" ), QStringLiteral( "date" ), QVariant::Date, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Time" ), QStringLiteral( "time" ), QVariant::Time, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Date & Time" ), QStringLiteral( "datetime" ), QVariant::DateTime, -1, -1, -1, -1 )
);

QgsDebugMsgLevel( "Delimited text file uri is " + uri, 2 );
@@ -399,6 +404,10 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
QList<bool> couldBeInt;
QList<bool> couldBeLongLong;
QList<bool> couldBeDouble;
QList<bool> couldBeDateTime;
QList<bool> couldBeDate;
QList<bool> couldBeTime;

bool foundFirstGeometry = false;

while ( true )
@@ -578,6 +587,9 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
couldBeInt.append( false );
couldBeLongLong.append( false );
couldBeDouble.append( false );
couldBeDateTime.append( false );
couldBeDate.append( false );
couldBeTime.append( false );
}

// If this column has been empty so far then initiallize it
@@ -589,6 +601,9 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
couldBeInt[i] = true;
couldBeLongLong[i] = true;
couldBeDouble[i] = true;
couldBeDateTime[i] = true;
couldBeDate[i] = true;
couldBeTime[i] = true;
}

if ( ! mDetectTypes )
@@ -604,25 +619,46 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
( void )value.toInt( &couldBeInt[i] );
}

if ( couldBeLongLong[i] && ! couldBeInt[i] )
if ( couldBeLongLong[i] && !couldBeInt[i] )
{
( void )value.toLongLong( &couldBeLongLong[i] );
}

if ( couldBeDouble[i] && ! couldBeLongLong[i] )
if ( couldBeDouble[i] && !couldBeLongLong[i] )
{
if ( ! mDecimalPoint.isEmpty() )
{
value.replace( mDecimalPoint, QLatin1String( "." ) );
}
( void )value.toDouble( &couldBeDouble[i] );
}

if ( couldBeDateTime[i] )
{
QDateTime dt;
if ( value.length() > 10 )
{
dt = QDateTime::fromString( value, Qt::ISODate );
}
couldBeDateTime[i] = ( dt.isValid() );
}

if ( couldBeDate[i] && !couldBeDateTime[i] )
{
QDate d = QDate::fromString( value, Qt::ISODate );
couldBeDate[i] = d.isValid();
}

if ( couldBeTime[i] && !couldBeDateTime[i] )
{
QTime t = QTime::fromString( value );
couldBeTime[i] = t.isValid();
}
}
}

// Now create the attribute fields. Field types are integer by preference,
// failing that double, failing that text.

// Now create the attribute fields. Field types are determined by prioritizing
// integer, failing that double, datetime, date, time, and finally text.
QStringList fieldNames = mFile->fieldNames();
mFieldCount = fieldNames.size();
attributeColumns.clear();
@@ -659,7 +695,20 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
{
typeName = QStringLiteral( "double" );
}
else if ( couldBeDateTime[i] )
{
typeName = QStringLiteral( "datetime" );
}
else if ( couldBeDate[i] )
{
typeName = QStringLiteral( "date" );
}
else if ( couldBeTime[i] )
{
typeName = QStringLiteral( "time" );
}
}

if ( typeName == QStringLiteral( "integer" ) )
{
fieldType = QVariant::Int;
@@ -673,10 +722,23 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
typeName = QStringLiteral( "double" );
fieldType = QVariant::Double;
}
else if ( typeName == QStringLiteral( "datetime" ) )
{
fieldType = QVariant::DateTime;
}
else if ( typeName == QStringLiteral( "date" ) )
{
fieldType = QVariant::Date;
}
else if ( typeName == QStringLiteral( "time" ) )
{
fieldType = QVariant::Time;
}
else
{
typeName = QStringLiteral( "text" );
}

attributeFields.append( QgsField( fieldNames[i], fieldType, typeName ) );
}

@@ -33,7 +33,7 @@

rebuildTests = 'REBUILD_DELIMITED_TEXT_TESTS' in os.environ

from qgis.PyQt.QtCore import QCoreApplication, QUrl, QObject
from qgis.PyQt.QtCore import QCoreApplication, QVariant, QUrl, QObject

from qgis.core import (
QgsProviderRegistry,
@@ -156,13 +156,13 @@ def tearDownClass(cls):
"""Run after all tests"""

def treat_time_as_string(self):
return True
return False

def treat_date_as_string(self):
return True
return False

def treat_datetime_as_string(self):
return True
return False


class TestQgsDelimitedTextProviderWKT(unittest.TestCase, ProviderTestCase):
@@ -205,13 +205,13 @@ def tearDownClass(cls):
"""Run after all tests"""

def treat_time_as_string(self):
return True
return False

def treat_date_as_string(self):
return True
return False

def treat_datetime_as_string(self):
return True
return False


class TestQgsDelimitedTextProviderOther(unittest.TestCase):
@@ -907,6 +907,26 @@ def test_046_M(self):
assert vl.wkbType() == QgsWkbTypes.PointM, "wrong wkb type, should be PointM"
assert vl.getFeature(2).geometry().asWkt() == "PointM (-71.12300000000000466 78.23000000000000398 2)", "wrong PointM geometry"

def test_047_datetime(self):
# Create test layer
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
basetestfile = os.path.join(srcpath, 'delimited_datetime.csv')

url = MyUrl.fromLocalFile(basetestfile)
url.addQueryItem("crs", "epsg:4326")
url.addQueryItem("type", "csv")
url.addQueryItem("xField", "X")
url.addQueryItem("yField", "Y")
url.addQueryItem("spatialIndex", "no")
url.addQueryItem("subsetIndex", "no")
url.addQueryItem("watchFile", "no")

vl = QgsVectorLayer(url.toString(), 'test', 'delimitedtext')
assert vl.isValid(), "{} is invalid".format(basetestfile)
assert vl.fields().at(4).type() == QVariant.DateTime
assert vl.fields().at(5).type() == QVariant.Date
assert vl.fields().at(6).type() == QVariant.Time

def testSpatialIndex(self):
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
basetestfile = os.path.join(srcpath, 'delimited_xyzm.csv')
@@ -1 +1 @@
integer,string,integer,real,string,string,datetime,date,time,long,longlong
integer,string,integer,real,string,string,string,string,string,long,longlong
@@ -0,0 +1,3 @@
point_id,X,Y,ele,time,date,t,Ds,len
"371",676239.6644,212505.7907,486,2020/04/07 09:58:07,2020/04/07,09:58:07,0,0
"372",676238.6797,212497.3033,486.2,2020/04/07 09:58:08,2020/04/07,09:58:08,8.544,8.54
@@ -0,0 +1 @@
"Integer","Real","Real","Real","DateTime","Date","Time","Real","Real"

0 comments on commit 8192be7

Please sign in to comment.
You can’t perform that action at this time.