Skip to content
Permalink
Browse files
Merge pull request #3816 from nyalldawson/processing_datetime
[processing][FEATURE] New unified basic stats algorithm
  • Loading branch information
nyalldawson committed Dec 1, 2016
2 parents 2c4eb3c + b30a1ff commit a17facf6cbc05ec2a0aaae7c01beed686bcce407
Showing with 525 additions and 116 deletions.
  1. +7 −0 python/core/qgsstringstatisticalsummary.sip
  2. +3 −7 python/plugins/processing/algs/help/qgis.yaml
  3. +251 −0 python/plugins/processing/algs/qgis/BasicStatistics.py
  4. +18 −31 python/plugins/processing/algs/qgis/BasicStatisticsNumbers.py
  5. +42 −59 python/plugins/processing/algs/qgis/BasicStatisticsStrings.py
  6. +3 −1 python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py
  7. +6 −0 python/plugins/processing/core/parameters.py
  8. +1 −1 python/plugins/processing/gui/ResultsDialog.py
  9. +4 −1 python/plugins/processing/gui/wrappers.py
  10. +1 −0 python/plugins/processing/modeler/ModelerParameterDefinitionDialog.py
  11. BIN python/plugins/processing/tests/testdata/custom/datetimes.dat
  12. BIN python/plugins/processing/tests/testdata/custom/datetimes.id
  13. BIN python/plugins/processing/tests/testdata/custom/datetimes.map
  14. +10 −0 python/plugins/processing/tests/testdata/custom/datetimes.tab
  15. +10 −0 python/plugins/processing/tests/testdata/expected/basic_statistics_date.html
  16. +10 −0 python/plugins/processing/tests/testdata/expected/basic_statistics_datetime.html
  17. +6 −4 python/plugins/processing/tests/testdata/expected/basic_statistics_string.html
  18. +10 −0 python/plugins/processing/tests/testdata/expected/basic_statistics_time.html
  19. +74 −3 python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml
  20. +16 −3 src/core/qgsdatetimestatisticalsummary.cpp
  21. +1 −0 src/core/qgsdatetimestatisticalsummary.h
  22. +9 −2 src/core/qgsstringstatisticalsummary.cpp
  23. +10 −1 src/core/qgsstringstatisticalsummary.h
  24. +30 −3 tests/src/python/test_qgsdatetimestatisticalsummary.py
  25. +3 −0 tests/src/python/test_qgsstringstatisticalsummary.py
@@ -31,6 +31,7 @@ class QgsStringStatisticalSummary
Max, //!< Maximum string value
MinimumLength, //!< Minimum length of string
MaximumLength, //!< Maximum length of string
MeanLength, //!< Mean length of strings
All, //! All statistics
};
typedef QFlags<QgsStringStatisticalSummary::Statistic> Statistics;
@@ -147,6 +148,12 @@ class QgsStringStatisticalSummary
*/
int maxLength() const;

/**
* Returns the mean length of strings.
* @note added in QGIS 3.0
*/
double meanLength() const;

/** Returns the friendly display name for a statistic
* @param statistic statistic to return name for
*/
@@ -22,14 +22,10 @@ qgis:advancedpythonfieldcalculator: >

qgis:barplot:

qgis:basicstatisticsforfields: >
This algorithm generates basic statistics from the analysis of a values in a field in the attribute table of a vector layer. Numeric, date, time and string fields are supported.

qgis:basicstatisticsfornumericfields: >
This algorithm generates basic statistics from the analysis of a numeric field in the attribute table of a vector layer.

Statistics are generated as an HTML file.

qgis:basicstatisticsfortextfields: >
This algorithm generates basic statistics from the analysis of a text field in the attribute table of a vector layer.
The statistics returned will depend on the field type.

Statistics are generated as an HTML file.

@@ -0,0 +1,251 @@
# -*- coding: utf-8 -*-

"""
***************************************************************************
BasicStatistics.py
---------------------
Date : November 2016
Copyright : (C) 2016 by Nyall Dawson
Email : nyall dot dawson at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""

__author__ = 'Nyall Dawson'
__date__ = 'November 2016'
__copyright__ = '(C) 2016, Nyall Dawson'

# This will get replaced with a git SHA1 when you do a git archive

__revision__ = '$Format:%H$'

import os
import codecs

from qgis.PyQt.QtCore import QVariant
from qgis.PyQt.QtGui import QIcon

from qgis.core import (QgsStatisticalSummary,
QgsStringStatisticalSummary,
QgsDateTimeStatisticalSummary,
QgsFeatureRequest)

from processing.core.GeoAlgorithm import GeoAlgorithm
from processing.core.parameters import ParameterTable
from processing.core.parameters import ParameterTableField
from processing.core.outputs import OutputHTML
from processing.core.outputs import OutputNumber
from processing.tools import dataobjects, vector


pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0]


class BasicStatisticsForField(GeoAlgorithm):

INPUT_LAYER = 'INPUT_LAYER'
FIELD_NAME = 'FIELD_NAME'
OUTPUT_HTML_FILE = 'OUTPUT_HTML_FILE'

MIN = 'MIN'
MAX = 'MAX'
COUNT = 'COUNT'
UNIQUE = 'UNIQUE'
EMPTY = 'EMPTY'
FILLED = 'FILLED'
MIN_LENGTH = 'MIN_LENGTH'
MAX_LENGTH = 'MAX_LENGTH'
MEAN_LENGTH = 'MEAN_LENGTH'
CV = 'CV'
SUM = 'SUM'
MEAN = 'MEAN'
STD_DEV = 'STD_DEV'
RANGE = 'RANGE'
MEDIAN = 'MEDIAN'
MINORITY = 'MINORITY'
MAJORITY = 'MAJORITY'
FIRSTQUARTILE = 'FIRSTQUARTILE'
THIRDQUARTILE = 'THIRDQUARTILE'
IQR = 'IQR'

def getIcon(self):
return QIcon(os.path.join(pluginPath, 'images', 'ftools', 'basic_statistics.png'))

def defineCharacteristics(self):
self.name, self.i18n_name = self.trAlgorithm('Basic statistics for fields')
self.group, self.i18n_group = self.trAlgorithm('Vector table tools')
self.tags = self.tr('stats,statistics,date,time,datetime,string,number,text,table,layer,maximum,minimum,mean,average,standard,deviation,'
'count,distinct,unique,variance,median,quartile,range,majority,minority')

self.addParameter(ParameterTable(self.INPUT_LAYER,
self.tr('Input table')))
self.addParameter(ParameterTableField(self.FIELD_NAME,
self.tr('Field to calculate statistics on'),
self.INPUT_LAYER))

self.addOutput(OutputHTML(self.OUTPUT_HTML_FILE,
self.tr('Statistics')))

self.addOutput(OutputNumber(self.COUNT, self.tr('Count')))
self.addOutput(OutputNumber(self.UNIQUE, self.tr('Number of unique values')))
self.addOutput(OutputNumber(self.EMPTY, self.tr('Number of empty (null) values')))
self.addOutput(OutputNumber(self.FILLED, self.tr('Number of non-empty values')))
self.addOutput(OutputNumber(self.MIN, self.tr('Minimum value')))
self.addOutput(OutputNumber(self.MAX, self.tr('Maximum value')))
self.addOutput(OutputNumber(self.MIN_LENGTH, self.tr('Minimum length')))
self.addOutput(OutputNumber(self.MAX_LENGTH, self.tr('Maximum length')))
self.addOutput(OutputNumber(self.MEAN_LENGTH, self.tr('Mean length')))
self.addOutput(OutputNumber(self.CV, self.tr('Coefficient of Variation')))
self.addOutput(OutputNumber(self.SUM, self.tr('Sum')))
self.addOutput(OutputNumber(self.MEAN, self.tr('Mean value')))
self.addOutput(OutputNumber(self.STD_DEV, self.tr('Standard deviation')))
self.addOutput(OutputNumber(self.RANGE, self.tr('Range')))
self.addOutput(OutputNumber(self.MEDIAN, self.tr('Median')))
self.addOutput(OutputNumber(self.MINORITY, self.tr('Minority (rarest occurring value)')))
self.addOutput(OutputNumber(self.MAJORITY, self.tr('Majority (most frequently occurring value)')))
self.addOutput(OutputNumber(self.FIRSTQUARTILE, self.tr('First quartile')))
self.addOutput(OutputNumber(self.THIRDQUARTILE, self.tr('Third quartile')))
self.addOutput(OutputNumber(self.IQR, self.tr('Interquartile Range (IQR)')))

def processAlgorithm(self, progress):
layer = dataobjects.getObjectFromUri(
self.getParameterValue(self.INPUT_LAYER))
field_name = self.getParameterValue(self.FIELD_NAME)
field = layer.fields().at(layer.fields().lookupField(field_name))

output_file = self.getOutputValue(self.OUTPUT_HTML_FILE)

request = QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry).setSubsetOfAttributes([field_name], layer.fields())
features = vector.features(layer, request)

data = []
data.append(self.tr('Analyzed layer: {}').format(layer.name()))
data.append(self.tr('Analyzed field: {}').format(field_name))

if field.isNumeric():
data.extend(self.calcNumericStats(features, progress, field))
elif field.type() in (QVariant.Date, QVariant.Time, QVariant.DateTime):
data.extend(self.calcDateTimeStats(features, progress, field))
else:
data.extend(self.calcStringStats(features, progress, field))

self.createHTML(output_file, data)

def calcNumericStats(self, features, progress, field):
count = len(features)
total = 100.0 / float(count)
stat = QgsStatisticalSummary()
for current, ft in enumerate(features):
stat.addVariant(ft[field.name()])
progress.setPercentage(int(current * total))
stat.finalize()

cv = stat.stDev() / stat.mean() if stat.mean() != 0 else 0

self.setOutputValue(self.COUNT, stat.count())
self.setOutputValue(self.UNIQUE, stat.variety())
self.setOutputValue(self.EMPTY, stat.countMissing())
self.setOutputValue(self.FILLED, count - stat.countMissing())
self.setOutputValue(self.MIN, stat.min())
self.setOutputValue(self.MAX, stat.max())
self.setOutputValue(self.RANGE, stat.range())
self.setOutputValue(self.SUM, stat.sum())
self.setOutputValue(self.MEAN, stat.mean())
self.setOutputValue(self.MEDIAN, stat.median())
self.setOutputValue(self.STD_DEV, stat.stDev())
self.setOutputValue(self.CV, cv)
self.setOutputValue(self.MINORITY, stat.minority())
self.setOutputValue(self.MAJORITY, stat.majority())
self.setOutputValue(self.FIRSTQUARTILE, stat.firstQuartile())
self.setOutputValue(self.THIRDQUARTILE, stat.thirdQuartile())
self.setOutputValue(self.IQR, stat.interQuartileRange())

data = []
data.append(self.tr('Count: {}').format(stat.count()))
data.append(self.tr('Unique values: {}').format(stat.variety()))
data.append(self.tr('NULL (missing) values: {}').format(stat.countMissing()))
data.append(self.tr('Minimum value: {}').format(stat.min()))
data.append(self.tr('Maximum value: {}').format(stat.max()))
data.append(self.tr('Range: {}').format(stat.range()))
data.append(self.tr('Sum: {}').format(stat.sum()))
data.append(self.tr('Mean value: {}').format(stat.mean()))
data.append(self.tr('Median value: {}').format(stat.median()))
data.append(self.tr('Standard deviation: {}').format(stat.stDev()))
data.append(self.tr('Coefficient of Variation: {}').format(cv))
data.append(self.tr('Minority (rarest occurring value): {}').format(stat.minority()))
data.append(self.tr('Majority (most frequently occurring value): {}').format(stat.majority()))
data.append(self.tr('First quartile: {}').format(stat.firstQuartile()))
data.append(self.tr('Third quartile: {}').format(stat.thirdQuartile()))
data.append(self.tr('Interquartile Range (IQR): {}').format(stat.interQuartileRange()))
return data

def calcStringStats(self, features, progress, field):
count = len(features)
total = 100.0 / float(count)
stat = QgsStringStatisticalSummary()
for current, ft in enumerate(features):
stat.addValue(ft[field.name()])
progress.setPercentage(int(current * total))
stat.finalize()

self.setOutputValue(self.COUNT, stat.count())
self.setOutputValue(self.UNIQUE, stat.countDistinct())
self.setOutputValue(self.EMPTY, stat.countMissing())
self.setOutputValue(self.FILLED, stat.count() - stat.countMissing())
self.setOutputValue(self.MIN, stat.min())
self.setOutputValue(self.MAX, stat.max())
self.setOutputValue(self.MIN_LENGTH, stat.minLength())
self.setOutputValue(self.MAX_LENGTH, stat.maxLength())
self.setOutputValue(self.MEAN_LENGTH, stat.meanLength())

data = []
data.append(self.tr('Count: {}').format(count))
data.append(self.tr('Unique values: {}').format(stat.countDistinct()))
data.append(self.tr('NULL (missing) values: {}').format(stat.countMissing()))
data.append(self.tr('Minimum value: {}').format(stat.min()))
data.append(self.tr('Maximum value: {}').format(stat.max()))
data.append(self.tr('Minimum length: {}').format(stat.minLength()))
data.append(self.tr('Maximum length: {}').format(stat.maxLength()))
data.append(self.tr('Mean length: {}').format(stat.meanLength()))

return data

def calcDateTimeStats(self, features, progress, field):
count = len(features)
total = 100.0 / float(count)
stat = QgsDateTimeStatisticalSummary()
for current, ft in enumerate(features):
stat.addValue(ft[field.name()])
progress.setPercentage(int(current * total))
stat.finalize()

self.setOutputValue(self.COUNT, stat.count())
self.setOutputValue(self.UNIQUE, stat.countDistinct())
self.setOutputValue(self.EMPTY, stat.countMissing())
self.setOutputValue(self.FILLED, stat.count() - stat.countMissing())
self.setOutputValue(self.MIN, stat.statistic(QgsDateTimeStatisticalSummary.Min))
self.setOutputValue(self.MAX, stat.statistic(QgsDateTimeStatisticalSummary.Max))

data = []
data.append(self.tr('Count: {}').format(count))
data.append(self.tr('Unique values: {}').format(stat.countDistinct()))
data.append(self.tr('NULL (missing) values: {}').format(stat.countMissing()))
data.append(self.tr('Minimum value: {}').format(field.displayString(stat.statistic(QgsDateTimeStatisticalSummary.Min))))
data.append(self.tr('Maximum value: {}').format(field.displayString(stat.statistic(QgsDateTimeStatisticalSummary.Max))))

return data

def createHTML(self, outputFile, algData):
with codecs.open(outputFile, 'w', encoding='utf-8') as f:
f.write('<html><head>\n')
f.write('<meta http-equiv="Content-Type" content="text/html; \
charset=utf-8" /></head><body>\n')
for s in algData:
f.write('<p>' + str(s) + '</p>\n')
f.write('</body></html>\n')
@@ -31,10 +31,11 @@

from qgis.PyQt.QtGui import QIcon

from qgis.core import QgsStatisticalSummary
from qgis.core import (QgsStatisticalSummary,
QgsFeatureRequest)

from processing.core.GeoAlgorithm import GeoAlgorithm
from processing.core.parameters import ParameterVector
from processing.core.parameters import ParameterTable
from processing.core.parameters import ParameterTableField
from processing.core.outputs import OutputHTML
from processing.core.outputs import OutputNumber
@@ -67,15 +68,21 @@ class BasicStatisticsNumbers(GeoAlgorithm):
NULLVALUES = 'NULLVALUES'
IQR = 'IQR'

def __init__(self):
GeoAlgorithm.__init__(self)
# this algorithm is deprecated - use BasicStatistics instead
self.showInToolbox = False

def getIcon(self):
return QIcon(os.path.join(pluginPath, 'images', 'ftools', 'basic_statistics.png'))

def defineCharacteristics(self):
self.name, self.i18n_name = self.trAlgorithm('Basic statistics for numeric fields')
self.group, self.i18n_group = self.trAlgorithm('Vector table tools')
self.tags = self.tr('stats,statistics,number,table,layer')

self.addParameter(ParameterVector(self.INPUT_LAYER,
self.tr('Input vector layer')))
self.addParameter(ParameterTable(self.INPUT_LAYER,
self.tr('Input vector layer')))
self.addParameter(ParameterTableField(self.FIELD_NAME,
self.tr('Field to calculate statistics on'),
self.INPUT_LAYER, ParameterTableField.DATA_TYPE_NUMBER))
@@ -107,36 +114,16 @@ def processAlgorithm(self, progress):

outputFile = self.getOutputValue(self.OUTPUT_HTML_FILE)

cvValue = 0
minValue = 0
maxValue = 0
sumValue = 0
meanValue = 0
medianValue = 0
stdDevValue = 0
minority = 0
majority = 0
firstQuartile = 0
thirdQuartile = 0
nullValues = 0
iqr = 0

values = []

features = vector.features(layer)
request = QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry).setSubsetOfAttributes([fieldName], layer.fields())
stat = QgsStatisticalSummary()
features = vector.features(layer, request)
count = len(features)
total = 100.0 / float(count)
for current, ft in enumerate(features):
value = ft[fieldName]
if value or value == 0:
values.append(float(value))
else:
nullValues += 1

stat.addVariant(ft[fieldName])
progress.setPercentage(int(current * total))

stat = QgsStatisticalSummary()
stat.calculate(values)
stat.finalize()

count = stat.count()
uniqueValue = stat.variety()
@@ -147,13 +134,13 @@ def processAlgorithm(self, progress):
meanValue = stat.mean()
medianValue = stat.median()
stdDevValue = stat.stDev()
if meanValue != 0.00:
cvValue = stdDevValue / meanValue
cvValue = stdDevValue / meanValue if meanValue != 0 else 0
minority = stat.minority()
majority = stat.majority()
firstQuartile = stat.firstQuartile()
thirdQuartile = stat.thirdQuartile()
iqr = stat.interQuartileRange()
nullValues = stat.countMissing()

data = []
data.append(self.tr('Analyzed layer: {}').format(layer.name()))

0 comments on commit a17facf

Please sign in to comment.