Skip to content

Commit

Permalink
[processing] TauDEM provider overhaul
Browse files Browse the repository at this point in the history
 - add new TauDEM 5.1.2/5.2 tools: Gage watershed, TWI and Select GT
   Threshold
 - implement support for multifile TauDEM version
 - allow to use single- and multifile versions simultaneously

Work done for Faunalia (http://faunalia.eu)

(cherry-picked from 427adf7)
  • Loading branch information
alexbruy committed Mar 10, 2015
1 parent e1b08fb commit 0b1a67b
Show file tree
Hide file tree
Showing 56 changed files with 1,432 additions and 38 deletions.
6 changes: 4 additions & 2 deletions python/plugins/processing/algs/taudem/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
FILE(GLOB PY_FILES *.py)
FILE(GLOB DESCR_FILES description/*.txt)
FILE(GLOB SINGLE_DESCR_FILES description/single/*.txt)
FILE(GLOB MULTI_DESCR_FILES description/multi/*.txt)

PLUGIN_INSTALL(processing algs/taudem ${PY_FILES})
PLUGIN_INSTALL(processing algs/taudem/description ${DESCR_FILES})
PLUGIN_INSTALL(processing algs/taudem/description/single ${SINGLE_DESCR_FILES})
PLUGIN_INSTALL(processing algs/taudem/description/multi ${MULTI_DESCR_FILES})
5 changes: 4 additions & 1 deletion python/plugins/processing/algs/taudem/TauDEMAlgorithm.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,21 @@

import os
from PyQt4.QtGui import QIcon

from processing.core.GeoAlgorithm import GeoAlgorithm
from processing.core.ProcessingLog import ProcessingLog
from processing.core.ProcessingConfig import ProcessingConfig
from processing.core.GeoAlgorithmExecutionException import \
GeoAlgorithmExecutionException
from processing.core.parameters import getParameterFromString

from processing.core.parameters import ParameterRaster
from processing.core.parameters import ParameterVector
from processing.core.parameters import ParameterBoolean
from processing.core.parameters import ParameterString
from processing.core.parameters import ParameterNumber
from processing.core.parameters import getParameterFromString
from processing.core.outputs import getOutputFromString

from TauDEMUtils import TauDEMUtils


Expand Down
113 changes: 82 additions & 31 deletions python/plugins/processing/algs/taudem/TauDEMAlgorithmProvider.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,13 @@
from PyQt4.QtGui import QIcon

from processing.core.AlgorithmProvider import AlgorithmProvider
from processing.core.ProcessingConfig import ProcessingConfig
from processing.core.ProcessingConfig import Setting
from processing.core.ProcessingConfig import ProcessingConfig, Setting
from processing.core.ProcessingLog import ProcessingLog

from TauDEMAlgorithm import TauDEMAlgorithm
from TauDEMMultifileAlgorithm import TauDEMMultifileAlgorithm
from TauDEMUtils import TauDEMUtils

from peukerdouglas import PeukerDouglas
from slopearea import SlopeArea
from lengtharea import LengthArea
Expand All @@ -45,13 +47,22 @@
from dinftranslimaccum import DinfTransLimAccum
from dinftranslimaccum2 import DinfTransLimAccum2

from peukerdouglas_multi import PeukerDouglasMulti
from slopearea_multi import SlopeAreaMulti
from lengtharea_multi import LengthAreaMulti
from dropanalysis_multi import DropAnalysisMulti
from dinfdistdown_multi import DinfDistDownMulti
from dinfdistup_multi import DinfDistUpMulti
from gridnet_multi import GridNetMulti
from dinftranslimaccum_multi import DinfTransLimAccumMulti
from dinftranslimaccum2_multi import DinfTransLimAccum2Multi


class TauDEMAlgorithmProvider(AlgorithmProvider):

def __init__(self):
AlgorithmProvider.__init__(self)
self.activate = False
self.createAlgsList()

def getDescription(self):
return self.tr('TauDEM (hydrologic analysis)')
Expand All @@ -64,10 +75,21 @@ def getIcon(self):

def initializeSettings(self):
AlgorithmProvider.initializeSettings(self)

ProcessingConfig.addSetting(Setting(self.getDescription(),
TauDEMUtils.TAUDEM_FOLDER,
self.tr('TauDEM command line tools folder'),
TauDEMUtils.taudemPath()))
ProcessingConfig.addSetting(Setting(self.getDescription(),
TauDEMUtils.TAUDEM_MULTIFILE_FOLDER,
self.tr('TauDEM multifile command line tools folder'),
TauDEMUtils.taudemMultifilePath()))
ProcessingConfig.addSetting(Setting(self.getDescription(),
TauDEMUtils.TAUDEM_USE_SINGLEFILE,
self.tr('Enable singlefile TauDEM tools'), True))
ProcessingConfig.addSetting(Setting(self.getDescription(),
TauDEMUtils.TAUDEM_USE_MULTIFILE,
self.tr('Enable multifile TauDEM tools'), False))
ProcessingConfig.addSetting(Setting(self.getDescription(),
TauDEMUtils.MPIEXEC_FOLDER,
self.tr('MPICH2/OpenMPI bin directory'),
Expand All @@ -78,36 +100,65 @@ def initializeSettings(self):

def unload(self):
AlgorithmProvider.unload(self)

ProcessingConfig.removeSetting(TauDEMUtils.TAUDEM_FOLDER)
ProcessingConfig.removeSetting(TauDEMUtils.TAUDEM_MULTIFILE_FOLDER)
ProcessingConfig.removeSetting(TauDEMUtils.TAUDEM_USE_SINGLEFILE)
ProcessingConfig.removeSetting(TauDEMUtils.TAUDEM_USE_MULTIFILE)
ProcessingConfig.removeSetting(TauDEMUtils.MPIEXEC_FOLDER)
ProcessingConfig.removeSetting(TauDEMUtils.MPI_PROCESSES)

def _loadAlgorithms(self):
self.algs = self.preloadedAlgs

def createAlgsList(self):
self.preloadedAlgs = []
folder = TauDEMUtils.taudemDescriptionPath()
for descriptionFile in os.listdir(folder):
if descriptionFile.endswith('txt'):
try:
alg = TauDEMAlgorithm(os.path.join(folder,
descriptionFile))
if alg.name.strip() != '':
self.preloadedAlgs.append(alg)
else:
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
self.tr('Could not open TauDEM algorithm: %s' % descriptionFile))
except Exception, e:
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
self.tr('Could not open TauDEM algorithm: %s' % descriptionFile))

self.preloadedAlgs.append(PeukerDouglas())
self.preloadedAlgs.append(SlopeArea())
self.preloadedAlgs.append(LengthArea())
self.preloadedAlgs.append(DropAnalysis())
self.preloadedAlgs.append(DinfDistDown())
self.preloadedAlgs.append(DinfDistUp())
self.preloadedAlgs.append(GridNet())
self.preloadedAlgs.append(DinfTransLimAccum())
self.preloadedAlgs.append(DinfTransLimAccum2())
self.algs = []
basePath = TauDEMUtils.taudemDescriptionPath()

if ProcessingConfig.getSetting(TauDEMUtils.TAUDEM_USE_SINGLEFILE):
folder = os.path.join(basePath, 'single')

for descriptionFile in os.listdir(folder):
if descriptionFile.endswith('txt'):
descriptionFile = os.path.join(folder, descriptionFile)
self._algFromDescription(descriptionFile)

self.algs.append(PeukerDouglas())
self.algs.append(SlopeArea())
self.algs.append(LengthArea())
self.algs.append(DropAnalysis())
self.algs.append(DinfDistDown())
self.algs.append(DinfDistUp())
self.algs.append(GridNet())
self.algs.append(DinfTransLimAccum())
self.algs.append(DinfTransLimAccum2())

if ProcessingConfig.getSetting(TauDEMUtils.TAUDEM_USE_MULTIFILE):
folder = os.path.join(basePath, 'multi')

for descriptionFile in os.listdir(folder):
if descriptionFile.endswith('txt'):
descriptionFile = os.path.join(folder, descriptionFile)
self._algFromDescription(descriptionFile, True)

self.algs.append(PeukerDouglasMulti())
self.algs.append(SlopeAreaMulti())
self.algs.append(LengthAreaMulti())
self.algs.append(DropAnalysisMulti())
self.algs.append(DinfDistDownMulti())
self.algs.append(DinfDistUpMulti())
self.algs.append(GridNetMulti())
self.algs.append(DinfTransLimAccumMulti())
self.algs.append(DinfTransLimAccum2Multi())

def _algFromDescription(self, descriptionFile, multifile=False):
try:
if multifile:
alg = TauDEMMultifileAlgorithm(descriptionFile)
else:
alg = TauDEMAlgorithm(descriptionFile)
if alg.name.strip() != '':
self.algs.append(alg)
else:
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
self.tr('Could not open TauDEM algorithm: %s' % descriptionFile))
except Exception, e:
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
self.tr('Could not open TauDEM algorithm %s:\n%s' % (descriptionFile, str(e))))
122 changes: 122 additions & 0 deletions python/plugins/processing/algs/taudem/TauDEMMultifileAlgorithm.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
# -*- coding: utf-8 -*-

"""
***************************************************************************
TauDEMMultifileAlgorithm.py
---------------------
Date : March 2015
Copyright : (C) 2012 by Alexander Bruy
Email : alexander dot bruy at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""

__author__ = 'Alexander Bruy'
__date__ = 'March 2015'
__copyright__ = '(C) 2015, Alexander Bruy'

# This will get replaced with a git SHA1 when you do a git archive

__revision__ = '$Format:%H$'

import os
from PyQt4.QtGui import QIcon

from processing.core.GeoAlgorithm import GeoAlgorithm
from processing.core.ProcessingLog import ProcessingLog
from processing.core.ProcessingConfig import ProcessingConfig
from processing.core.GeoAlgorithmExecutionException import \
GeoAlgorithmExecutionException

from processing.core.parameters import ParameterFile
from processing.core.parameters import ParameterVector
from processing.core.parameters import ParameterBoolean
from processing.core.parameters import ParameterString
from processing.core.parameters import ParameterNumber
from processing.core.parameters import getParameterFromString
from processing.core.outputs import getOutputFromString

from TauDEMUtils import TauDEMUtils


class TauDEMMultifileAlgorithm(GeoAlgorithm):

def __init__(self, descriptionfile):
GeoAlgorithm.__init__(self)
self.descriptionFile = descriptionfile
self.defineCharacteristicsFromFile()

def getCopy(self):
newone = TauDEMMultifileAlgorithm(self.descriptionFile)
newone.provider = self.provider
return newone

def getIcon(self):
return QIcon(os.path.dirname(__file__) + '/../../images/taudem.png')

def defineCharacteristicsFromFile(self):
lines = open(self.descriptionFile)
line = lines.readline().strip('\n').strip()
self.name = line
line = lines.readline().strip('\n').strip()
self.cmdName = line
line = lines.readline().strip('\n').strip()
self.group = line

line = lines.readline().strip('\n').strip()
while line != '':
try:
line = line.strip('\n').strip()
if line.startswith('Parameter'):
param = getParameterFromString(line)
self.addParameter(param)
else:
self.addOutput(getOutputFromString(line))
line = lines.readline().strip('\n').strip()
except Exception, e:
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
self.tr('Could not load TauDEM algorithm: %s\n%s' % (self.descriptionFile, line)))
raise e
lines.close()

def processAlgorithm(self, progress):
commands = []
commands.append(os.path.join(TauDEMUtils.mpiexecPath(), 'mpiexec'))

processNum = int(ProcessingConfig.getSetting(TauDEMUtils.MPI_PROCESSES))
if processNum <= 0:
raise GeoAlgorithmExecutionException(
self.tr('Wrong number of MPI processes used. Please set '
'correct number before running TauDEM algorithms.'))

commands.append('-n')
commands.append(str(processNum))
commands.append(os.path.join(TauDEMUtils.taudemMultifilePath(), self.cmdName))

for param in self.parameters:
if param.value is None or param.value == '':
continue
if isinstance(param, ParameterNumber):
commands.append(param.name)
commands.append(str(param.value))
if isinstance(param, (ParameterFile, ParameterVector)):
commands.append(param.name)
commands.append(param.value)
elif isinstance(param, ParameterBoolean):
if param.value and str(param.value).lower() == 'false':
commands.append(param.name)
elif isinstance(param, ParameterString):
commands.append(param.name)
commands.append(str(param.value))

for out in self.outputs:
commands.append(out.name)
commands.append(out.value)

TauDEMUtils.executeTauDEM(commands, progress)
28 changes: 24 additions & 4 deletions python/plugins/processing/algs/taudem/TauDEMUtils.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,22 @@
__revision__ = '$Format:%H$'

import os
from qgis.core import QgsApplication
import subprocess

from PyQt4.QtCore import QCoreApplication
from qgis.core import QgsApplication

from processing.core.ProcessingConfig import ProcessingConfig
from processing.core.ProcessingLog import ProcessingLog
from processing.tools.system import isMac
from PyQt4.QtCore import QCoreApplication


class TauDEMUtils:

TAUDEM_FOLDER = 'TAUDEM_FOLDER'
TAUDEM_MULTIFILE_FOLDER = 'TAUDEM_MULTIFILE_FOLDER'
TAUDEM_USE_SINGLEFILE = 'TAUDEM_USE_SINGLEFILE'
TAUDEM_USE_MULTIFILE = 'TAUDEM_USE_MULTIFILE'
MPIEXEC_FOLDER = 'MPIEXEC_FOLDER'
MPI_PROCESSES = 'MPI_PROCESSES'

Expand All @@ -57,6 +61,22 @@ def taudemPath():
folder = testfolder
return folder

@staticmethod
def taudemMultifilePath():
folder = ProcessingConfig.getSetting(TauDEMUtils.TAUDEM_MULTIFILE_FOLDER)
if folder is None:
folder = ''

if isMac():
testfolder = os.path.join(QgsApplication.prefixPath(), 'bin')
if os.path.exists(os.path.join(testfolder, 'slopearea')):
folder = testfolder
else:
testfolder = '/usr/local/bin'
if os.path.exists(os.path.join(testfolder, 'slopearea')):
folder = testfolder
return folder

@staticmethod
def mpiexecPath():
folder = ProcessingConfig.getSetting(TauDEMUtils.MPIEXEC_FOLDER)
Expand All @@ -75,8 +95,8 @@ def mpiexecPath():

@staticmethod
def taudemDescriptionPath():
return os.path.normpath(os.path.join(os.path.dirname(__file__),
'description'))
return os.path.normpath(
os.path.join(os.path.dirname(__file__), 'description'))

@staticmethod
def executeTauDEM(command, progress):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
D8 Contributing Area (multifile)
aread8
Basic Grid Analysis tools
ParameterFile|-p|D8 Flow Direction Grid|True|False
ParameterVector|-o|Outlets Shapefile|0|True
ParameterFile|-wg|Weight Grid|True|True
ParameterBoolean|-nc|Check for edge contamination|True
OutputDirectory|-ad8|D8 Contributing Area Grid
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
D-Infinity Contributing Area (multifile)
areadinf
Basic Grid Analysis tools
ParameterFile|-ang|D-Infinity Flow Direction Grid|True|False
ParameterVector|-o|Outlets Shapefile|0|True
ParameterFile|-wg|Weight Grid|True|True
ParameterBoolean|-nc|Check for edge contamination|True
OutputDirectory|-sca|D-Infinity Specific Catchment Area Grid
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
D8 Flow Directions (multifile)
d8flowdir
Basic Grid Analysis tools
ParameterFile|-fel|Pit Filled Elevation Grid|True|False
OutputDirectory|-p|D8 Flow Direction Grid
OutputDirectory|-sd8|D8 Slope Grid

0 comments on commit 0b1a67b

Please sign in to comment.