ModelerAlgorithm.py

# -*- coding: utf-8 -*-

"""
***************************************************************************
    ModelerAlgorithm.py
    ---------------------
    Date                 : August 2012
    Copyright            : (C) 2012 by Victor Olaya
    Email                : volayaf at gmail dot com
***************************************************************************
*                                                                         *
*   This program is free software; you can redistribute it and/or modify  *
*   it under the terms of the GNU General Public License as published by  *
*   the Free Software Foundation; either version 2 of the License, or     *
*   (at your option) any later version.                                   *
*                                                                         *
***************************************************************************
"""

__author__ = 'Victor Olaya'
__date__ = 'August 2012'
__copyright__ = '(C) 2012, Victor Olaya'

# This will get replaced with a git SHA1 when you do a git archive

__revision__ = '$Format:%H$'

import os.path
import sys
import copy
import time
import json
import codecs
import traceback
from PyQt4 import QtCore, QtGui
from qgis.core import *
from processing.core.GeoAlgorithm import GeoAlgorithm
from processing.modeler.WrongModelException import WrongModelException
from processing.core.GeoAlgorithmExecutionException import \
        GeoAlgorithmExecutionException
from processing.modeler.ModelerUtils import ModelerUtils
from processing.core.parameters import *
from processing.tools import dataobjects
from processing.gui.Help2Html import getHtmlFromDescriptionsDict

class ModelerParameter():

    def __init__(self, param=None, pos=None):
        self.param = param
        self.pos = pos

    def todict(self):
        return self.__dict__

    @staticmethod
    def fromdict(d):
        return ModelerParameter(d["param"], d["pos"])


class ModelerOutput():

    def __init__(self, description=""):
        self.description = description
        self.pos = None

    def todict(self):
        return self.__dict__


class Algorithm():

    def __init__(self, consoleName=""):

        self.name = None
        self.description = ""

        #The type of the algorithm, indicated as a string, which corresponds
        #to the string used to refer to it in the python console
        self.consoleName = consoleName

        self._algInstance = None

        #A dict of Input object. keys are param names
        self.params = {}

        #A dict of Output with final output descriptions. Keys are output names.
        #Outputs not final are not stored in this dict
        self.outputs = {}

        self.pos = None

        self.dependencies = []

        self.paramsFolded = True
        self.outputsFolded = True
        self.active = True


    def todict(self):
        return {k:v for k,v in self.__dict__.iteritems() if  not k.startswith("_")}

    @property
    def algorithm(self):
        if self._algInstance is None:
            self._algInstance = ModelerUtils.getAlgorithm(self.consoleName).getCopy();
        return self._algInstance

    def setName(self, model):
        if self.name is None:
            i = 1
            name = self.consoleName + "_" + str(i)
            while name in model.algs:
                i += 1
                name = self.consoleName + "_" + str(i)
            self.name = name


class ValueFromInput():

    def __init__(self, name=""):
        self.name = name

    def todict(self):
        return self.__dict__

    def __str__(self):
        return self.name

    def __eq__(self, other):
        try:
            return self.name == other.name
        except:
            return False


class ValueFromOutput():

    def __init__(self, alg="", output=""):
        self.alg = alg
        self.output = output

    def todict(self):
        return self.__dict__

    def __eq__(self, other):
        try:
            return self.alg == other.alg and self.output == other.output
        except:
            return False

    def __str__(self):
        return self.alg + "," + self.output


class ModelerAlgorithm(GeoAlgorithm):

    CANVAS_SIZE = 4000

    def getCopy(self):
        newone = ModelerAlgorithm()
        newone.provider = self.provider
        newone.algs = copy.deepcopy(self.algs)
        newone.inputs = copy.deepcopy(self.inputs)
        newone.defineCharacteristics()
        newone.name = self.name
        newone.group = self.group
        newone.descriptionFile = self.descriptionFile
        newone.helpContent = copy.deepcopy(self.helpContent)
        return newone

    def __init__(self):
        self.name = self.tr('Model', 'ModelerAlgorithm')
        # The dialog where this model is being edited
        self.modelerdialog = None
        self.descriptionFile = None
        self.helpContent = {}

        # Geoalgorithms in this model. A dict of Algorithm objects, with names as keys
        self.algs = {}

        #Input parameters. A dict of Input objects, with names as keys
        self.inputs = {}
        GeoAlgorithm.__init__(self)

    def getIcon(self):
        return QtGui.QIcon(os.path.dirname(__file__) + '/../images/model.png')

    def defineCharacteristics(self):
        classes = [ParameterRaster, ParameterVector, ParameterTable, ParameterTableField,
                   ParameterBoolean, ParameterString, ParameterNumber]
        self.parameters = []
        for c in classes:
            for inp in self.inputs.values():
                if isinstance(inp.param, c):
                    self.parameters.append(inp.param)
        for inp in self.inputs.values():
            if inp.param not in self.parameters:
                self.parameters.append(inp.param)
        self.outputs = []
        for alg in self.algs.values():
            if alg.active:
                for out in alg.outputs:
                    modelOutput = copy.deepcopy(alg.algorithm.getOutputFromName(out))
                    modelOutput.name = self.getSafeNameForOutput(alg.name, out)
                    modelOutput.description = alg.outputs[out].description
                    self.outputs.append(modelOutput)

    def addParameter(self, param):
        self.inputs[param.param.name] = param

    def updateParameter(self, param):
        self.inputs[param.name].param = param

    def addAlgorithm(self, alg):
        name = self.getNameForAlgorithm(alg)
        alg.name = name
        self.algs[name] = alg

    def getNameForAlgorithm(self, alg):
        i = 1
        while alg.consoleName.upper().replace(":", "") + "_" + str(i) in self.algs.keys():
            i += 1
        return alg.consoleName.upper().replace(":", "") + "_" + str(i)

    def updateAlgorithm(self, alg):
        alg.pos = self.algs[alg.name].pos
        self.algs[alg.name] = alg

        from processing.modeler.ModelerGraphicItem import ModelerGraphicItem
        for i, out in enumerate(alg.outputs):
            alg.outputs[out].pos = (alg.outputs[out].pos or
                    alg.pos + QtCore.QPointF(ModelerGraphicItem.BOX_WIDTH,
                                             (i + 1.5) * ModelerGraphicItem.BOX_HEIGHT))

    def removeAlgorithm(self, name):
        """Returns True if the algorithm could be removed, False if
        others depend on it and could not be removed.
        """
        if self.hasDependencies(name):
            return False
        del self.algs[name]
        self.modelerdialog.hasChanged = True
        return True

    def removeParameter(self, name):
        """Returns True if the parameter could be removed, False if
        others depend on it and could not be removed.
        """
        if self.hasDependencies(name):
            return False
        del self.inputs[name]
        self.modelerdialog.hasChanged = True
        return True

    def hasDependencies(self, name):
        """This method returns True if some other element depends on
        the passed one.
        """
        for alg in self.algs.values():
            for value in alg.params.values():
                if value is None:
                    continue
                if isinstance(value, list):
                    for v in value:
                        if isinstance(v, ValueFromInput):
                            if v.name == name:
                                return True
                        elif isinstance(v, ValueFromOutput):
                            if v.alg == name:
                                return True
                if isinstance(value, ValueFromInput):
                    if value.name == name:
                        return True
                elif isinstance(value, ValueFromOutput):
                    if value.alg == name:
                        return True
        return False


    def getDependsOnAlgorithms(self, name):
        """This method returns a list with names of algorithms
        a given one depends on.
        """
        alg = self.algs[name]
        algs = set()
        algs.update(set(alg.dependencies))
        for value in alg.params.values():
            if value is None:
                continue
            if isinstance(value, list):
                for v in value:
                    if isinstance(v, ValueFromOutput):
                        algs.add(v.alg)
                        algs.update(self.getDependsOnAlgorithms(v.alg))
            elif isinstance(value, ValueFromOutput):
                algs.add(value.alg)
                algs.update(self.getDependsOnAlgorithms(value.alg))


        return algs

    def getDependentAlgorithms(self, name):
        """This method returns a list with the names of algorithms
        depending on a given one. It includes the algorithm itself
        """
        algs = set()
        algs.add(name)
        for alg in self.algs.values():
            for value in alg.params.values():
                if value is None:
                    continue
                if isinstance(value, list):
                    for v in value:
                        if isinstance(v, ValueFromOutput) and v.alg == name:
                            algs.update(self.getDependentAlgorithms(alg.name))
                elif isinstance(value, ValueFromOutput) and value.alg == name:
                            algs.update(self.getDependentAlgorithms(alg.name))

        return algs

    def setPositions(self, paramPos, algPos, outputsPos):
        for param, pos in paramPos.iteritems():
            self.inputs[param].pos = pos
        for alg, pos in algPos.iteritems():
            self.algs[alg].pos = pos
        for alg, positions in outputsPos.iteritems():
            for output, pos in positions.iteritems():
                self.algs[alg].outputs[output].pos = pos

    def prepareAlgorithm(self, alg):
        algInstance = alg.algorithm
        for param in algInstance.parameters:
            if not param.hidden:
                value = self.resolveValue(alg.params[param.name])
                if value is None and isinstance(param, ParameterExtent):
                    value = self.getMinCoveringExtent()
                # We allow unexistent filepaths, since that allows
                # algorithms to skip some conversion routines
                if not param.setValue(value) and not isinstance(param,
                        ParameterDataObject):
                    raise GeoAlgorithmExecutionException(
                       self.tr('Wrong value: %s', 'ModelerAlgorithm') % value)
        for out in algInstance.outputs:
            if not out.hidden:
                if out.name in alg.outputs:
                    name = self.getSafeNameForOutput(alg.name, out.name)
                    modelOut = self.getOutputFromName(name)
                    if modelOut:
                        out.value = modelOut.value
                else:
                    out.value = None

        return algInstance

    def deactivateAlgorithm(self, algName):
        dependent = self.getDependentAlgorithms(algName)
        for alg in dependent:
            self.algs[alg].active = False

    def activateAlgorithm(self, algName):
        parents = self.getDependsOnAlgorithms(algName)
        for alg in parents:
            if not self.algs[alg].active:
                return False
        self.algs[algName].active = True
        return True

    def getSafeNameForOutput(self, algName, outName):
        return outName + '_ALG' + algName

    def resolveValue(self, value):
        if value is None:
            return None
        if isinstance(value, list):
            return ";".join([self.resolveValue(v) for v in value])
        if isinstance(value, ValueFromInput):
            return self.getParameterFromName(value.name).value
        elif isinstance(value, ValueFromOutput):
            return self.algs[value.alg].algorithm.getOutputFromName(value.output).value
        else:
            return value

    def getMinCoveringExtent(self):
        first = True
        found = False
        for param in self.parameters:
            if param.value:
                if isinstance(param, (ParameterRaster, ParameterVector)):
                    found = True
                    if isinstance(param.value, (QgsRasterLayer, QgsVectorLayer)):
                        layer = param.value
                    else:
                        layer = dataobjects.getObjectFromUri(param.value)
                    self.addToRegion(layer, first)
                    first = False
                elif isinstance(param, ParameterMultipleInput):
                    found = True
                    layers = param.value.split(';')
                    for layername in layers:
                        layer = dataobjects.getObjectFromUri(layername)
                        self.addToRegion(layer, first)
                        first = False
        if found:
            return ','.join([str(v) for v in [self.xmin, self.xmax, self.ymin, self.ymax]])
        else:
            return None

    def addToRegion(self, layer, first):
        if first:
            self.xmin = layer.extent().xMinimum()
            self.xmax = layer.extent().xMaximum()
            self.ymin = layer.extent().yMinimum()
            self.ymax = layer.extent().yMaximum()
        else:
            self.xmin = min(self.xmin, layer.extent().xMinimum())
            self.xmax = max(self.xmax, layer.extent().xMaximum())
            self.ymin = min(self.ymin, layer.extent().yMinimum())
            self.ymax = max(self.ymax, layer.extent().yMaximum())


    def processAlgorithm(self, progress):
        executed = []
        toExecute = [alg for alg in self.algs.values() if alg.active]
        while len(executed) < len(toExecute):
            for alg in toExecute:
                if alg.name not in executed:
                    canExecute = True
                    required = self.getDependsOnAlgorithms(alg.name)
                    for requiredAlg in required:
                        if requiredAlg != alg.name and requiredAlg not in executed:
                            canExecute = False
                            break
                    if canExecute:
                        try:
                            progress.setDebugInfo(
                                self.tr('Prepare algorithm: %s', 'ModelerAlgorithm') % alg.name)
                            self.prepareAlgorithm(alg)
                            progress.setText(
                                self.tr('Running %s [%i/%i]', 'ModelerAlgorithm') % (alg.description, len(executed) + 1 ,len(toExecute)))
                            progress.setDebugInfo('Parameters: ' + ', '.join([unicode(p).strip()
                                                + '=' + unicode(p.value) for p in alg.algorithm.parameters]))
                            t0 = time.time()
                            alg.algorithm.execute(progress, self)
                            dt = time.time() - t0
                            executed.append(alg.name)
                            progress.setDebugInfo(
                                self.tr('OK. Execution took %0.3f ms (%i outputs).', 'ModelerAlgorithm') % (dt, len(alg.algorithm.outputs)))
                        except GeoAlgorithmExecutionException, e:
                            progress.setDebugInfo(self.tr('Failed', 'ModelerAlgorithm'))
                            raise GeoAlgorithmExecutionException(
                                self.tr('Error executing algorithm %s\n%s', 'ModelerAlgorithm') % (alg.description, e.msg))

        progress.setDebugInfo(
            self.tr('Model processed ok. Executed %i algorithms total', 'ModelerAlgorithm') % len(executed))


    def getAsCommand(self):
        if self.descriptionFile:
            return GeoAlgorithm.getAsCommand(self)
        else:
            return None

    def commandLineName(self):
        if self.descriptionFile is None:
            return ''
        else:
            return 'modeler:' + os.path.basename(self.descriptionFile)[:-6].lower()

    def setModelerView(self, dialog):
        self.modelerdialog = dialog

    def updateModelerView(self):
        if self.modelerdialog:
            self.modelerdialog.repaintModel()

    def help(self):
        print self.helpContent
        try:
            return True, getHtmlFromDescriptionsDict(self, self.helpContent)
        except:
            return False, None

    def todict(self):
        keys = ["inputs", "group", "name", "algs", "helpContent"]
        return {k:v for k,v in self.__dict__.iteritems() if k in keys}

    def toJson(self):
        def todict(o):
            if isinstance(o, QtCore.QPointF):
                return {"class": "point", "values": {"x": o.x(), "y": o.y()}}
            try:
                d = o.todict()
                return {"class": o.__class__.__module__ + "." + o.__class__.__name__, "values": d}
            except Exception, e:
                pass
        return json.dumps(self, default=todict, indent=4)


    @staticmethod
    def fromJson(s):
        def fromdict(d):
            try:
                fullClassName = d["class"]
                tokens = fullClassName.split(".")
                className = tokens[-1]
                moduleName = ".".join(tokens[:-1])
                values = d["values"]
                if className == "point":
                    return QtCore.QPointF(values["x"], values["y"])
                def _import(name):
                    __import__(name)
                    return sys.modules[name]
                if moduleName.startswith("processing.parameters"):
                    moduleName = "processing.core.parameters"
                module = _import(moduleName)
                clazz = getattr(module, className)
                instance = clazz()
                for k,v in values.iteritems():
                    instance.__dict__[k] = v
                return instance
            except KeyError:
                return d
            except Exception, e:
                raise e
        try:
            model = json.loads(s, object_hook = fromdict)
        except Exception, e:
            raise WrongModelException(e.args[0])
        return model


    @staticmethod
    def fromJsonFile(filename):
        with open(filename) as f:
            s = f.read()
        alg = ModelerAlgorithm.fromJson(s)
        alg.descriptionFile = filename
        return alg


    ############LEGACY METHOD TO SUPPORT OLD FORMAT###########

    LINE_BREAK_STRING = '%%%'

    @staticmethod
    def fromFile(filename):
        try:
            alg = ModelerAlgorithm.fromJsonFile(filename)
            return alg
        except WrongModelException, e:
            alg = ModelerAlgorithm.fromOldFormatFile(filename)
            return alg


    @staticmethod
    def fromOldFormatFile(filename):
        def _tr(s):
            return QtCore.QCoreApplication.translate('ModelerAlgorithm', s)
        hardcodedValues = {}
        modelParameters = []
        modelAlgs = []
        model = ModelerAlgorithm()
        model.descriptionFile = filename
        lines = codecs.open(filename, 'r', encoding='utf-8')
        line = lines.readline().strip('\n').strip('\r')
        try:
            while line != '':
                if line.startswith('PARAMETER:'):
                    paramLine = line[len('PARAMETER:'):]
                    param = getParameterFromString(paramLine)
                    if param:
                        pass
                    else:
                        raise WrongModelException(
                            _tr('Error in parameter line: %s', 'ModelerAlgorithm') % line)
                    line = lines.readline().strip('\n')
                    tokens = line.split(',')
                    model.addParameter(ModelerParameter(param, QtCore.QPointF(
                                            float(tokens[0]), float(tokens[1]))))
                    modelParameters.append(param.name)
                elif line.startswith('VALUE:'):
                    valueLine = line[len('VALUE:'):]
                    tokens = valueLine.split('===')
                    name = tokens[0]
                    value = tokens[1].replace(ModelerAlgorithm.LINE_BREAK_STRING, '\n')
                    hardcodedValues[name] = value
                elif line.startswith('NAME:'):
                    model.name = line[len('NAME:'):]
                elif line.startswith('GROUP:'):
                    model.group = line[len('GROUP:'):]
                elif line.startswith('ALGORITHM:'):
                    algLine = line[len('ALGORITHM:'):]
                    alg = ModelerUtils.getAlgorithm(algLine)
                    if alg is not None:
                        modelAlg = Algorithm(alg.commandLineName())
                        modelAlg.description = alg.name
                        posline = lines.readline().strip('\n').strip('\r')
                        tokens = posline.split(',')
                        modelAlg.pos = QtCore.QPointF(float(tokens[0]), float(tokens[1]))
                        dependenceline = lines.readline().strip('\n').strip('\r') #unused
                        for param in alg.parameters:
                            if not param.hidden:
                                line = lines.readline().strip('\n').strip('\r')
                                if line == str(None):
                                    modelAlg.params[param.name]  = None
                                else:
                                    tokens = line.split('|')
                                    algIdx = int(tokens[0])
                                    if algIdx == -1:
                                        if tokens[1] in modelParameters:
                                            modelAlg.params[param.name] = ValueFromInput(tokens[1])
                                        else:
                                            modelAlg.params[param.name] = hardcodedValues[tokens[1]]
                                    else:
                                        modelAlg.params[param.name] = ValueFromOutput(algIdx, tokens[1])

                        for out in alg.outputs:
                            if not out.hidden:
                                line = lines.readline().strip('\n').strip('\r')
                                if str(None) != line:
                                    if '|' in line:
                                        tokens = line.split('|')
                                        name = tokens[0]
                                        tokens = tokens[1].split(',')
                                        pos = QtCore.QPointF(
                                                float(tokens[0]), float(tokens[1]))
                                    else:
                                        name = line
                                        pos = None
                                    modelerOutput = ModelerOutput(name)
                                    modelerOutput.pos = pos
                                    modelAlg.outputs[out.name] = modelerOutput

                        model.addAlgorithm(modelAlg)
                        modelAlgs.append(modelAlg.name)
                    else:
                        raise WrongModelException(
                            _tr('Error in algorithm name: %s', ) % algLine)
                line = lines.readline().strip('\n').strip('\r')
            for modelAlg in model.algs.values():
                for name, value in modelAlg.params.iteritems():
                    if isinstance(value, ValueFromOutput):
                        value.alg = modelAlgs[value.alg]
            return model
        except Exception, e:
            if isinstance(e, WrongModelException):
                raise e
            else:
                raise WrongModelException(_tr('Error in model definition line: ') + '%s\n%s' % (line.strip(), traceback.format_exc()))