diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cf436299..d0c96f90d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,38 @@ Correção de Bug: - Correção de erro ao criar banco EDGV 3.0; +## 4.11.16 - dev + + + +Novas Funcionalidades: + +- Novo processo de corrigir erros de segmentação em linhas (barragem versus rodovias); +- Novo processo para validar a estrutura do banco de dados em relação ao masterfile; +- Novo processo de detectar as mudanças entre dois banco de dados, realizados em dias distintos; + +Melhorias: + +- O estado do Menu de classificação agora é salvo no projeto; +- Adiciona parâmetros opcionais na rotina de extração de pontos cotados; +- Melhoria de desempenho na extração de pontos cotados; +- O seletor genérico agora ignora camadas somente leitura; +- Melhoria de desempenho no processo Unir Linhas com Mesmo Conjunto de Atributos (Merge Lines With Same Attribute Set): Código refatorado para usar busca utilizando grafo. Além disso, o processo pega casos que não eram unidos anteriormente (linhas com mesmo conjunto de atributos e encadeadas); +- Melhoria de desempenho no processo Identify Unmerged Lines With Same Attribute Set: Código refatorado para usar busca utilizando grafo; + +Correção de bug: + +- Corrige bug de quando o usuário tenta reclassificar primitivas não compatíveis utilizando o menu de classificação; +- Corrige bug na rotina de extração de pontos cotados; +- Corrige bug de flags incorretas na rotina de identificar erros de segmentação em linhas (Identify Segment Errors Between Lines); +- Corrige bug de número de conexões ativas na ferramenta de setar estilo do banco (os métodos nativos do QGIS abrem conexão e não fecham, dessa forma derrubando todo mundo da produção por exceder o número máximo de conexões defindo no postgres.conf); +- Corrige bug no processo de verificar atributo unicode; +- Corrige bug no processo de identificar pontas soltas (retira distância mínima na busca, agora pega erro até no "mundo quântico"); +- Corrige bug na ferramenta de inverter o sentido de linhas (flip lines) quando é uma camada não salva do banco de dados; +- Corrige bug na geração de moldura relativo a camada (Generate Systematic Grid Related to Layer) quando se usa memory layer (o memory layer não atualiza seu extent automaticamente, logo foi necessário colocar um layer.updateExtents() no código antes de calcular o extent); +- Corrige bug no processo de calcular azimute (estava sendo calculado com +90 graus); + + ## 4.10.0 - 2023-09-08 Novas Funcionalidades: @@ -28,6 +60,7 @@ Novas Funcionalidades: - Novo processo de ajustar parâmetros da ferramenta de aquisição com ângulos retos (integração com o FP/SAP); - Novo processo de converter entre bancos de mesma modelagem, clipando com um polígono feito para integração com FP/SAP (ClipAndCopyFeaturesBetweenDatabasesAlgorithm); - Novo processo de verificar ligação na moldura; +- Novo processo de cálculo do azimute; Melhorias: diff --git a/DsgTools/Modules/acquisitionMenu/controllers/acquisitionMenuCtrl.py b/DsgTools/Modules/acquisitionMenu/controllers/acquisitionMenuCtrl.py index 2e5a5597d..51bb41524 100644 --- a/DsgTools/Modules/acquisitionMenu/controllers/acquisitionMenuCtrl.py +++ b/DsgTools/Modules/acquisitionMenu/controllers/acquisitionMenuCtrl.py @@ -2,7 +2,7 @@ from PyQt5 import QtCore, uic, QtWidgets, QtGui from DsgTools.Modules.qgis.controllers.qgisCtrl import QgisCtrl import json -from qgis.core import QgsWkbTypes +from qgis.core import QgsWkbTypes, QgsProject, QgsExpressionContextUtils from qgis.utils import iface class AcquisitionMenuCtrl: @@ -14,6 +14,7 @@ def __init__(self, qgis=None, widgetFactory=None): self.addMenuTab = None self.addMenuButton = None self.reclassifyDialog = None + self.menuConfigs = None self.ignoreSignal = False self.connectQgisSignals() @@ -25,12 +26,43 @@ def connectQgisSignals(self): self.qgis.connectSignal("ClickLayerTreeView", self.deactiveMenu) self.qgis.connectSignal("AddLayerTreeView", self.deactiveMenu) self.qgis.connectSignal("StartEditing", self.deactiveMenu) + self.qgis.connectSignal("ProjectSaved", self.saveStateOnProject) + self.qgis.connectSignal("ProjectRead", self.loadStateOnProject) def disconnectQgisSignals(self): self.qgis.disconnectSignal("StartAddFeature", self.deactiveMenu) self.qgis.disconnectSignal("ClickLayerTreeView", self.deactiveMenu) self.qgis.disconnectSignal("AddLayerTreeView", self.deactiveMenu) self.qgis.disconnectSignal("StartEditing", self.deactiveMenu) + self.qgis.disconnectSignal("ProjectSaved", self.saveStateOnProject) + self.qgis.disconnectSignal("ProjectRead", self.loadStateOnProject) + + def loadStateOnProject(self): + state = json.loads( + QgsExpressionContextUtils.projectScope(QgsProject.instance()).variable( + "dsgtools_menu_state" + ) + or "{}" + ) + if state == {}: + return + self.createMenuDock(state) + + def saveStateOnProject(self): + if self.menuConfigs is None: + return + currentProject = QgsProject.instance() + currentProject.projectSaved.disconnect(self.saveStateOnProject) + QgsExpressionContextUtils.setProjectVariable( + currentProject, + "dsgtools_menu_state", + json.dumps(self.menuConfigs), + ) + currentProject.blockSignals(True) + QgsProject.instance().write() + QgsProject.instance().projectSaved.connect(self.saveStateOnProject) + currentProject.blockSignals(False) + def openMenuEditor(self): if not self.menuEditor: @@ -159,6 +191,8 @@ def createMenuDock(self, menuConfigs): self.menuDock.setMenuWidget(self.getMenuWidget()) self.menuDock.loadMenus(menuConfigs) self.qgis.addDockWidget(self.menuDock) + self.menuConfigs = menuConfigs + self.saveStateOnProject() def removeMenuDock(self): self.qgis.removeDockWidget(self.menuDock) if self.menuDock else "" @@ -209,6 +243,7 @@ def validLayersToReclassification(self, buttonConfig): noActive = l.id() != iface.activeLayer().id() if noActive: raise Exception("Selecione somente feições da camada que está em uso!") + def reclassify(self, buttonConfig, reclassifyData): destinatonLayerName = buttonConfig["buttonLayer"] @@ -227,14 +262,14 @@ def reclassify(self, buttonConfig, reclassifyData): def getLayersForReclassification(self, layerName, geometryType): layers = self.qgis.getLoadedVectorLayers() geometryFilterDict = { - QgsWkbTypes.PointGeometry: (QgsWkbTypes.PointGeometry,), - QgsWkbTypes.LineGeometry: (QgsWkbTypes.LineGeometry,), - QgsWkbTypes.PolygonGeometry: (QgsWkbTypes.PointGeometry, QgsWkbTypes.PolygonGeometry) + QgsWkbTypes.PointGeometry: (QgsWkbTypes.PointGeometry, QgsWkbTypes.PolygonGeometry), + QgsWkbTypes.LineGeometry: (QgsWkbTypes.LineGeometry, ), + QgsWkbTypes.PolygonGeometry: (QgsWkbTypes.PointGeometry, ), } return [ l for l in layers - if l.selectedFeatureCount() > 0 and l.geometryType() in geometryFilterDict[l.geometryType()] + if l.selectedFeatureCount() > 0 and l.geometryType() in geometryFilterDict[geometryType] ] def activeMenuButton(self, buttonConfig): diff --git a/DsgTools/Modules/qgis/controllers/qgisCtrl.py b/DsgTools/Modules/qgis/controllers/qgisCtrl.py index 9decd9b0d..cdb23b222 100644 --- a/DsgTools/Modules/qgis/controllers/qgisCtrl.py +++ b/DsgTools/Modules/qgis/controllers/qgisCtrl.py @@ -122,7 +122,8 @@ def getAcquisitionToolNames(self): "Mão Livre": "FreeHand", } - def addDockWidget(self, dockWidget, side=QtCore.Qt.LeftDockWidgetArea): + def addDockWidget(self, dockWidget, side=None): + side = side if side is not None else QtCore.Qt.LeftDockWidgetArea iface.addDockWidget(side, dockWidget) def removeDockWidget(self, dockWidget): @@ -267,6 +268,8 @@ def getSignals(self): "ClickLayerTreeView": iface.layerTreeView().clicked, "AddLayerTreeView": core.QgsProject.instance().legendLayersAdded, "StartEditing": iface.actionToggleEditing().triggered, + "ProjectSaved": core.QgsProject.instance().projectSaved, + "ProjectRead": iface.projectRead, } def suppressLayerForm(self, layer, suppress): diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/DataManagementAlgs/clipAndCopyFeaturesBetweenDatabasesAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/DataManagementAlgs/clipAndCopyFeaturesBetweenDatabasesAlgorithm.py index 0ba482860..a0b6361f2 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/DataManagementAlgs/clipAndCopyFeaturesBetweenDatabasesAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/DataManagementAlgs/clipAndCopyFeaturesBetweenDatabasesAlgorithm.py @@ -204,6 +204,14 @@ def processAlgorithm(self, parameters, context, feedback): return {} if multiStepFeedback is not None: multiStepFeedback.pushInfo(self.tr("Commiting changes")) + if len(outputLayerDict) > 0: + self.commitChanges(multiStepFeedback, outputLayerDict) + if not loadDestinationLayers: + for lyrName, lyr in outputLayerDict.items(): + QgsProject.instance().removeMapLayer(lyr.id()) + return {} + + def commitChanges(self, multiStepFeedback, outputLayerDict): stepSize = 100 / len(outputLayerDict) for current, (lyrName, lyr) in enumerate(outputLayerDict.items()): if multiStepFeedback is not None: @@ -213,10 +221,6 @@ def processAlgorithm(self, parameters, context, feedback): lyr.commitChanges() if multiStepFeedback is not None: multiStepFeedback.setProgress(current * stepSize) - if not loadDestinationLayers: - for lyrName, lyr in outputLayerDict.items(): - QgsProject.instance().removeMapLayer(lyr.id()) - return {} def getLayersFromDbConnectionName( self, @@ -269,20 +273,28 @@ def clipInputLayerList(self, inputLayerList, geom, context, feedback): if feedback is not None else None ) - clipLayer = self.layerHandler.createMemoryLayerFromGeometry( - geom=geom, crs=QgsProject.instance().crs() - ) if geom is not None else None + clipLayer = ( + self.layerHandler.createMemoryLayerFromGeometry( + geom=geom, crs=QgsProject.instance().crs() + ) + if geom is not None + else None + ) for currentIdx, lyr in enumerate(inputLayerList): if multiStepFeedback is not None and multiStepFeedback.isCanceled(): return outputDict if multiStepFeedback is not None: multiStepFeedback.setCurrentStep(2 * currentIdx) - clippedLyr = self.algRunner.runClip( - inputLayer=lyr, - overlayLayer=clipLayer, - context=context, - feedback=multiStepFeedback, - ) if clippedLyr is not None else lyr + clippedLyr = ( + self.algRunner.runClip( + inputLayer=lyr, + overlayLayer=clipLayer, + context=context, + feedback=multiStepFeedback, + ) + if clipLayer is not None + else lyr + ) if multiStepFeedback is not None: multiStepFeedback.setCurrentStep(2 * currentIdx + 1) outputDict[lyr.name()] = self.algRunner.runCreateFieldWithExpression( diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/GeometricAlgs/extractElevationPoints.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/GeometricAlgs/extractElevationPoints.py index 981e20236..44ce212c9 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/GeometricAlgs/extractElevationPoints.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/GeometricAlgs/extractElevationPoints.py @@ -54,6 +54,7 @@ QgsProcessingParameterField, QgsVectorLayerUtils, QgsProcessingParameterVectorLayer, + QgsSpatialIndex, ) @@ -168,6 +169,7 @@ def initAlgorithm(self, config=None): self.WATER_BODIES, self.tr("Water Bodies"), [QgsProcessing.TypeVectorPolygon], + optional=True, ) ) @@ -176,6 +178,7 @@ def initAlgorithm(self, config=None): self.AREA_WITHOUT_INFORMATION_POLYGONS, self.tr("Area without information layer"), [QgsProcessing.TypeVectorPolygon], + optional=True, ) ) @@ -192,6 +195,7 @@ def initAlgorithm(self, config=None): self.DRAINAGE_LINES_WITHOUT_NAME, self.tr("Drainage lines without name"), [QgsProcessing.TypeVectorLine], + optional=True, ) ) @@ -208,6 +212,7 @@ def initAlgorithm(self, config=None): self.OTHER_ROADS, self.tr("Other Roads"), [QgsProcessing.TypeVectorLine], + optional=True, ) ) @@ -286,7 +291,8 @@ def processAlgorithm(self, parameters, context, feedback): if multiStepFeedback.isCanceled(): break multiStepFeedback.setCurrentStep(currentStep) - multiStepFeedback.pushInfo(self.tr(f"Evaluating region {currentStep+1}/{nFeats}")) + self.currentStepText = self.tr(f"Evaluating region {currentStep+1}/{nFeats}") + multiStepFeedback.pushInfo(self.currentStepText) localBoundsLyr = layerHandler.createMemoryLayerWithFeature( geographicBoundaryLyr, feat, context ) @@ -342,7 +348,7 @@ def computePoints( ): algRunner = AlgRunner() layerHandler = LayerHandler() - nSteps = 16 + ( + nSteps = 18 + ( naturalPointFeaturesLyr is not None ) + 3 * (waterBodiesLyr is not None) # handle this count after alg is done multiStepFeedback = ( @@ -353,7 +359,7 @@ def computePoints( if multiStepFeedback is not None: currentStep = 0 multiStepFeedback.setCurrentStep(currentStep) - multiStepFeedback.setProgressText(self.tr("Clipping raster")) + multiStepFeedback.setProgressText(self.tr(f"{self.currentStepText}: Clipping raster")) frameCentroid = ( [i for i in geographicBoundsLyr.getFeatures()][0].geometry().centroid() ) @@ -375,7 +381,7 @@ def computePoints( inputRaster, mask=geographicBoundsLyr, context=context, - feedback=feedback, + feedback=multiStepFeedback, noData=-9999, outputRaster=QgsProcessingUtils.generateTempFilename( f"clip_{str(uuid4().hex)}.tif" @@ -390,7 +396,7 @@ def computePoints( if multiStepFeedback is not None: currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) - multiStepFeedback.setProgressText(self.tr("Reading raster with numpy...")) + multiStepFeedback.setProgressText(self.tr(f"{self.currentStepText}: Extracting buffer from contours...")) localContourBufferLength = geometryHandler.convertDistance( self.contourBufferLength, originEpsg=originEpsg, @@ -406,6 +412,8 @@ def computePoints( currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) if waterBodiesLyr is not None: + if multiStepFeedback is not None: + multiStepFeedback.setProgressText(self.tr(f"{self.currentStepText}: Extracting local water bodies...")) localWaterBodiesLyr = algRunner.runExtractByLocation( inputLyr=waterBodiesLyr, intersectLyr=geographicBoundsLyr, @@ -416,12 +424,15 @@ def computePoints( if multiStepFeedback is not None: currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr(f"{self.currentStepText}:Evaluating buffer on water bodies...")) localBufferedWaterBodiesLyr = algRunner.runBuffer( inputLayer=localWaterBodiesLyr, distance=localContourBufferLength, context=context, feedback=multiStepFeedback, is_child_algorithm=True ) if multiStepFeedback is not None: currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr(f"{self.currentStepText}: Running clip on water bodies...")) + algRunner.runCreateSpatialIndex(localBufferedWaterBodiesLyr, context, is_child_algorithm=True) waterBodiesLyr = algRunner.runClip( localBufferedWaterBodiesLyr, overlayLayer=geographicBoundsLyr, @@ -431,6 +442,7 @@ def computePoints( if multiStepFeedback is not None: currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr(f"{self.currentStepText}: Building masked raster...")) npRaster, transform = self.readAndMaskRaster( clippedRasterLyr, geographicBoundsLyr, @@ -446,10 +458,10 @@ def computePoints( currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( - self.tr("Getting max min feats and building exclusion polygons...") + self.tr("Getting max min feats...") ) minMaxFeats = self.getMinMaxFeatures( - fields, npRaster, transform, distance=localBufferDistance + fields, npRaster, transform, distance=localBufferDistance, feedback=multiStepFeedback, ) elevationPointsLayer = layerHandler.createMemoryLayerWithFeatures( featList=minMaxFeats, @@ -458,6 +470,12 @@ def computePoints( wkbType=QgsWkbTypes.Point, context=context, ) + if multiStepFeedback is not None: + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText( + self.tr("Getting building exclusion polygons...") + ) # compute number of points minNPoints, maxNPoints = self.getRangeOfNumberOfPoints(minMaxFeats) maxPointsPerGridUnit = maxNPoints // 9 @@ -472,7 +490,7 @@ def computePoints( if multiStepFeedback is not None: currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) - multiStepFeedback.setProgressText(self.tr("Preparing contours...")) + multiStepFeedback.setProgressText(self.tr(f"{self.currentStepText}: Preparing contours...")) contourAreaDict, polygonLyr = self.prepareContours( contourLyr=contourLyr, geographicBoundsLyr=geographicBoundsLyr, @@ -495,7 +513,7 @@ def computePoints( if naturalPointFeaturesLyr is not None: if multiStepFeedback is not None: multiStepFeedback.setProgressText( - self.tr("Getting elevation points from natural points...") + self.tr(f"{self.currentStepText}: Getting elevation points from natural points...") ) elevationPointsFromNaturalPointFeatures = ( self.getElevationPointsFromNaturalPoints( @@ -532,7 +550,7 @@ def computePoints( if multiStepFeedback is not None: multiStepFeedback.setProgressText( - self.tr("Getting elevation points from hilltops...") + self.tr(f"{self.currentStepText}: Getting elevation points from hilltops...") ) # create points from hilltops @@ -569,7 +587,7 @@ def computePoints( currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( - self.tr("Getting elevation points from main road intersections...") + self.tr(f"{self.currentStepText}: Getting elevation points from main road intersections...") ) elevationPointsFromRoadIntersections = ( self.getElevationPointsFromLineIntersections( @@ -607,7 +625,7 @@ def computePoints( currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( - self.tr("Getting elevation points from other road intersections...") + self.tr(f"{self.currentStepText}: Getting elevation points from other road intersections...") ) elevationPointsFromOtherRoadIntersections = ( self.getElevationPointsFromLineIntersections( @@ -646,7 +664,7 @@ def computePoints( multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( self.tr( - "Getting elevation points from intersections of road and rivers with names outside polygons..." + f"{self.currentStepText}: Getting elevation points from intersections of road and rivers with names outside polygons..." ) ) elevationPointsFromIntersectionsOfRiverWithNamesAndRoads = ( @@ -688,7 +706,7 @@ def computePoints( multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( self.tr( - "Getting elevation points from intersections of road and rivers without names..." + f"{self.currentStepText}: Getting elevation points from intersections of road and rivers without names..." ) ) @@ -731,7 +749,7 @@ def computePoints( multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( self.tr( - "Getting elevation points from intersections of rivers with names..." + f"{self.currentStepText}: Getting elevation points from intersections of rivers with names..." ) ) @@ -774,7 +792,7 @@ def computePoints( multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( self.tr( - "Getting elevation points from intersections of rivers with names..." + f"{self.currentStepText}: Getting elevation points from intersections of rivers with names..." ) ) @@ -817,7 +835,7 @@ def computePoints( multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( self.tr( - "Getting elevation points from intersections of rivers with and without names..." + f"{self.currentStepText}: Getting elevation points from intersections of rivers with and without names..." ) ) @@ -860,7 +878,7 @@ def computePoints( multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( self.tr( - "Getting elevation points from intersections of rivers without names..." + f"{self.currentStepText}: Getting elevation points from intersections of rivers without names..." ) ) @@ -902,7 +920,7 @@ def computePoints( currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) multiStepFeedback.setProgressText( - self.tr("Getting elevation points from plane areas...") + self.tr(f"{self.currentStepText}: Getting elevation points from plane areas...") ) planeAreasElevationPoints = self.getElevationPointsFromPlaneAreas( @@ -1032,11 +1050,12 @@ def buildExclusionLyr( ) layerList = [buffer] for lyr in [areaWithoutInformationLyr, waterBodiesLyr]: - if lyr is not None: - auxLyr = algRunner.runMultipartToSingleParts( - lyr, context, is_child_algorithm=True - ) - layerList.append(auxLyr) + if lyr is None: + continue + auxLyr = algRunner.runMultipartToSingleParts( + lyr, context, is_child_algorithm=True + ) + layerList.append(auxLyr) outputLyr = ( buffer if len(layerList) == 1 @@ -1439,9 +1458,16 @@ def getElevationPointsFromLineIntersections( feedback=multiStepFeedback, ) - def getMinMaxFeatures(self, fields, npRaster, transform, distance): + def getMinMaxFeatures(self, fields, npRaster, transform, distance, feedback=None): featSet = set() + multiStepFeedback = QgsProcessingMultiStepFeedback(4, feedback) if feedback is not None else None + if multiStepFeedback is not None: + multiStepFeedback.setCurrentStep(0) + multiStepFeedback.pushInfo(self.tr("Getting max coordinates from numpy array...")) maxCoordinatesArray = rasterHandler.getMaxCoordinatesFromNpArray(npRaster) + if multiStepFeedback is not None: + multiStepFeedback.setCurrentStep(1) + multiStepFeedback.pushInfo(self.tr("Creating max feature list from pixel coordinates array...")) maxFeatList = ( rasterHandler.createFeatureListWithPixelValuesFromPixelCoordinatesArray( maxCoordinatesArray, @@ -1452,9 +1478,16 @@ def getMinMaxFeatures(self, fields, npRaster, transform, distance): defaultAtributeMap=dict(self.defaultAttrMap), ) ) - featSet |= self.filterFeaturesByBuffer(maxFeatList, distance, cotaMaisAlta=True) - + featSet |= self.filterFeaturesByBuffer(maxFeatList, distance, cotaMaisAlta=True, feedback=multiStepFeedback) + if multiStepFeedback is not None: + multiStepFeedback.setCurrentStep(2) + multiStepFeedback.pushInfo(self.tr("Getting min coordinates from numpy array...")) minCoordinatesArray = rasterHandler.getMinCoordinatesFromNpArray(npRaster) + if minCoordinatesArray == []: + return list(featSet) + if multiStepFeedback is not None: + multiStepFeedback.setCurrentStep(3) + multiStepFeedback.pushInfo(self.tr("Creating min feature list from pixel coordinates array...")) minFeatList = ( rasterHandler.createFeatureListWithPixelValuesFromPixelCoordinatesArray( minCoordinatesArray, @@ -1465,7 +1498,7 @@ def getMinMaxFeatures(self, fields, npRaster, transform, distance): defaultAtributeMap=dict(self.defaultAttrMap), ) ) - featSet |= self.filterFeaturesByBuffer(minFeatList, distance) + featSet |= self.filterFeaturesByBuffer(minFeatList, distance, feedback=multiStepFeedback) return list(featSet) def filterWithAllCriteria( @@ -1538,20 +1571,33 @@ def filterWithAllCriteria( ) def filterFeaturesByBuffer( - self, filterFeatList: List, distance, cotaMaisAlta=False + self, filterFeatList: List, distance, cotaMaisAlta=False, feedback=None, ): outputSet = set() - exclusionGeom = None - for feat in filterFeatList: + featDict = dict() + spatialIdx = None + nFeats = len(filterFeatList) + if nFeats == 0: + return outputSet + stepSize = 100/nFeats + for current, feat in enumerate(filterFeatList): + if feedback is not None and feedback.isCanceled(): + return outputSet geom = feat.geometry() buffer = geom.buffer(distance, -1) - if exclusionGeom is not None and exclusionGeom.intersects(geom): + bbox = buffer.boundingBox() + if spatialIdx is not None and any(featDict[idx].geometry().intersects(buffer) for idx in spatialIdx.intersects(bbox)): continue + if spatialIdx is None: + spatialIdx = QgsSpatialIndex() feat["cota_mais_alta"] = 1 if cotaMaisAlta else 2 - exclusionGeom = ( - buffer if exclusionGeom is None else exclusionGeom.combine(buffer) - ) + featCopy = QgsFeature(feat) + featCopy.setId(current) + spatialIdx.addFeature(featCopy) + featDict[current] = featCopy outputSet.add(feat) + if feedback is not None: + feedback.setProgress(current * stepSize) return outputSet def filterFeaturesByDistanceAndExclusionLayer( @@ -1804,7 +1850,7 @@ def extractElevationPointsFromHilltops( originEpsg=originEpsg, destinationEpsg=geographicBoundsLyr.crs(), ) - minusBufferLength = geometryHandler.convertDistance( + minusBufferLength = - geometryHandler.convertDistance( self.contourBufferLength, originEpsg=originEpsg, destinationEpsg=geographicBoundsLyr.crs(), @@ -1966,9 +2012,9 @@ def getElevationPointsFromPlaneAreas( currentStep += 1 multiStepFeedback.setCurrentStep(currentStep) candidateGridLyr = algRunner.runExtractByLocation( - inputLyr=contourLyr, - intersectLyr=planeGrid, - predicate=[2], + inputLyr=planeGrid, + intersectLyr=contourLyr, + predicate=[AlgRunner.Disjoint], context=context, feedback=multiStepFeedback, ) diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/applyStylesFromDatabaseToLayersAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/applyStylesFromDatabaseToLayersAlgorithm.py index 7a48da7ec..3dedce5a6 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/applyStylesFromDatabaseToLayersAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/applyStylesFromDatabaseToLayersAlgorithm.py @@ -20,42 +20,16 @@ * * ***************************************************************************/ """ -import os +from collections import defaultdict from PyQt5.QtCore import QCoreApplication -from qgis.PyQt.Qt import QVariant +from DsgTools.core.dsgEnums import DsgEnums +from DsgTools.core.Factories.DbFactory.dbFactory import DbFactory from qgis.PyQt.QtXml import QDomDocument from qgis.core import ( QgsProcessing, - QgsFeatureSink, QgsProcessingAlgorithm, - QgsProcessingParameterFeatureSource, - QgsProcessingParameterFeatureSink, - QgsFeature, - QgsDataSourceUri, - QgsProcessingOutputVectorLayer, - QgsProcessingParameterVectorLayer, - QgsWkbTypes, - QgsProcessingParameterBoolean, - QgsProcessingParameterEnum, - QgsProcessingParameterNumber, QgsProcessingParameterMultipleLayers, - QgsProcessingUtils, - QgsSpatialIndex, - QgsGeometry, - QgsProcessingParameterField, - QgsProcessingMultiStepFeedback, - QgsProcessingParameterFile, - QgsProcessingParameterExpression, - QgsProcessingException, QgsProcessingParameterString, - QgsProcessingParameterDefinition, - QgsProcessingParameterType, - QgsProcessingParameterCrs, - QgsCoordinateTransform, - QgsProject, - QgsCoordinateReferenceSystem, - QgsField, - QgsFields, QgsProcessingOutputMultipleLayers, QgsProcessingParameterString, ) @@ -98,28 +72,26 @@ def processAlgorithm(self, parameters, context, feedback): styleName = self.parameterAsString(parameters, self.STYLE_NAME, context) listSize = len(inputLyrList) progressStep = 100 / listSize if listSize else 0 + styleDict = self.getStyleDict(inputLyrList, feedback) + for current, lyr in enumerate(inputLyrList): if feedback.isCanceled(): break - count, idList, styleList, time, _ = lyr.listStylesInDatabase() - styleDict = dict(zip(styleList, idList)) - if styleName in styleDict: - styleQml, _ = lyr.getStyleFromDatabase(styleDict[styleName]) - self.applyStyle(lyr, styleQml) - elif ( - "{style_name}/{layer_name}".format( - style_name=styleName, layer_name=lyr.name() - ) - in styleDict - ): - styleQml, _ = lyr.getStyleFromDatabase( - styleDict[ - "{style_name}/{layer_name}".format( - style_name=styleName, layer_name=lyr.name() - ) - ] - ) - self.applyStyle(lyr, styleQml) + if lyr.providerType() != "postgres": + continue + uri = lyr.dataProvider().uri() + dbName = uri.database() + if dbName not in styleDict: + continue + if styleName not in styleDict[dbName]: + continue + schema = uri.schema() + tableName = uri.table() + geometryColumn = uri.geometryColumn() + key = f"{schema}.{tableName}({geometryColumn})" + if key not in styleDict[dbName][styleName]: + continue + self.applyStyle(lyr, styleDict[dbName][styleName][key]) feedback.setProgress(current * progressStep) return {self.OUTPUT: [i.id() for i in inputLyrList]} @@ -129,6 +101,39 @@ def applyStyle(self, lyr, styleQml): styleDoc.setContent(styleQml) lyr.importNamedStyle(styleDoc) lyr.triggerRepaint() + + def getAbstractDb(self, host, port, database, user, password): + abstractDb = DbFactory().createDbFactory(DsgEnums.DriverPostGIS) + abstractDb.connectDatabaseWithParameters(host, port, database, user, password) + return abstractDb + + def getDbDict(self, lyrList, feedback): + dbDict = dict() + for lyr in lyrList: + if feedback.isCanceled(): + return dbDict + if lyr.providerType() != "postgres": + continue + uri = lyr.dataProvider().uri() + dbName = uri.database() + if dbName in dbDict: + continue + host = uri.host() + port = uri.port() + user = uri.username() + password = uri.password() + dbDict[dbName] = self.getAbstractDb(host, port, dbName, user, password) + return dbDict + + def getStyleDict(self, lyrList, feedback): + dbDict = self.getDbDict(lyrList, feedback) + dbStyleDict = dict() + for dbName, abstractDb in dbDict.items(): + if feedback.isCanceled(): + break + dbStyleDict[dbName] = abstractDb.getStyleDict() + del dbDict + return dbStyleDict def name(self): """ diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/loadLayersFromPostgisAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/loadLayersFromPostgisAlgorithm.py index 53b3bc657..db2539b19 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/loadLayersFromPostgisAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/loadLayersFromPostgisAlgorithm.py @@ -28,38 +28,11 @@ LayerLoaderFactory, ) from qgis.core import ( - QgsCoordinateReferenceSystem, - QgsCoordinateTransform, - QgsDataSourceUri, - QgsFeature, - QgsFeatureSink, - QgsField, - QgsFields, - QgsGeometry, - QgsProcessing, QgsProcessingAlgorithm, - QgsProcessingException, - QgsProcessingMultiStepFeedback, QgsProcessingOutputMultipleLayers, - QgsProcessingOutputVectorLayer, QgsProcessingParameterBoolean, - QgsProcessingParameterCrs, - QgsProcessingParameterDefinition, - QgsProcessingParameterEnum, - QgsProcessingParameterExpression, - QgsProcessingParameterFeatureSink, - QgsProcessingParameterFeatureSource, - QgsProcessingParameterField, - QgsProcessingParameterFile, - QgsProcessingParameterMultipleLayers, - QgsProcessingParameterNumber, QgsProcessingParameterString, - QgsProcessingParameterType, - QgsProcessingParameterVectorLayer, - QgsProcessingUtils, QgsProject, - QgsSpatialIndex, - QgsWkbTypes, QgsMapLayer, ) from qgis.utils import iface diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/azimuthCalculationAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/azimuthCalculationAlgorithm.py new file mode 100644 index 000000000..57390ee11 --- /dev/null +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/azimuthCalculationAlgorithm.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + DsgTools + A QGIS plugin + Brazilian Army Cartographic Production Tools + ------------------- + begin : 2023-08-22 + git sha : $Format:%H$ + copyright : (C) 2023 by Matheus Alves Silva - Cartographic Engineer @ Brazilian Army + email : matheus.silva@ime.eb.br + ***************************************************************************/ +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +from qgis import core +from qgis.core import ( + QgsProcessing, + QgsProcessingAlgorithm, + QgsProcessingMultiStepFeedback, + QgsProcessingParameterVectorLayer, + QgsProcessingParameterBoolean, + QgsProcessingException, +) +from qgis.PyQt.QtCore import QCoreApplication +from qgis.PyQt.Qt import QVariant + + +class AzimuthCalculationAlgorithm(QgsProcessingAlgorithm): + + INPUT_LAYER = "INPUT_LAYER" + ATTRIBUTE = "ATTRIBUTE" + FEATURES_SELECTED = "FEATURES_SELECTED" + OUTPUT = "OUTPUT" + + def initAlgorithm(self, config=None): + self.addParameter( + QgsProcessingParameterVectorLayer( + self.INPUT_LAYER, + self.tr("Input layer"), + [QgsProcessing.TypeVectorLine, QgsProcessing.TypeVectorPolygon], + ) + ) + + self.addParameter( + QgsProcessingParameterBoolean( + self.FEATURES_SELECTED, self.tr("Change filled values") + ) + ) + + self.addParameter( + core.QgsProcessingParameterField( + self.ATTRIBUTE, + self.tr("Select the attribute that will receive the azimuth"), + type=core.QgsProcessingParameterField.Any, + parentLayerParameterName=self.INPUT_LAYER, + allowMultiple=False, + defaultValue=None, + optional=False, + ) + ) + + def processAlgorithm(self, parameters, context, feedback): + inputLyr = self.parameterAsVectorLayer(parameters, self.INPUT_LAYER, context) + + filledFeatures = self.parameterAsBool( + parameters, self.FEATURES_SELECTED, context + ) + + if inputLyr is None: + raise QgsProcessingException("Choose a layer for azimuth calculation") + + attributeAzim = self.parameterAsFields(parameters, self.ATTRIBUTE, context)[0] + + inputLyr.startEditing() + inputLyr.beginEditCommand(f'Updating the attribute "{attributeAzim}"') + + nSteps = inputLyr.featureCount() + if nSteps == 0: + return {self.OUTPUT: ""} + + multiStepFeedback = QgsProcessingMultiStepFeedback(nSteps, feedback) + + for current, feat in enumerate(inputLyr.getFeatures()): + if multiStepFeedback.isCanceled(): + break + multiStepFeedback.setCurrentStep(current) + orientMiniBBox = self.orientedMinimunBBoxFeat(feat) + + angAzim = orientMiniBBox[2] + + if feat[f"{attributeAzim}"] != QVariant(None): + if not filledFeatures: + continue + feat[f"{attributeAzim}"] = round(angAzim) + else: + feat[f"{attributeAzim}"] = round(angAzim) + + inputLyr.updateFeature(feat) + + inputLyr.endEditCommand() + + return {self.OUTPUT: ""} + + def orientedMinimunBBoxFeat(self, feat): + geom = feat.geometry() + orientMiniBBox = geom.orientedMinimumBoundingBox() + return orientMiniBBox + + def name(self): + """ + Here is where the processing itself takes place. + """ + return "azimuthcalculation" + + def displayName(self): + """ + Returns the translated algorithm name, which should be used for any + user-visible display of the algorithm name. + """ + return self.tr("Azimuth Calculation") + + def group(self): + """ + Returns the name of the group this algorithm belongs to. This string + should be localised. + """ + return self.tr("Other Algorithms") + + def groupId(self): + """ + Returns the unique ID of the group this algorithm belongs to. This + string should be fixed for the algorithm, and must not be localised. + The group id should be unique within each provider. Group id should + contain lowercase alphanumeric characters only and no spaces or other + formatting characters. + """ + return "DSGTools: Other Algorithms" + + def tr(self, string): + return QCoreApplication.translate( + "The algorithm calculates the azimuth of each feature", string + ) + + def createInstance(self): + return AzimuthCalculationAlgorithm() diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/identifyDifferencesBetweenDatabaseModelsAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/identifyDifferencesBetweenDatabaseModelsAlgorithm.py new file mode 100644 index 000000000..4d2629f1e --- /dev/null +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/identifyDifferencesBetweenDatabaseModelsAlgorithm.py @@ -0,0 +1,894 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + DsgTools + A QGIS plugin + Brazilian Army Cartographic Production Tools + ------------------- + begin : 2023-09-19 + git sha : $Format:%H$ + copyright : (C) 2023 by Matheus Alves Silva - Cartographic Engineer @ Brazilian Army + email : matheus.silva@ime.eb.br + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +from PyQt5.QtCore import QCoreApplication +from qgis.core import ( + QgsProcessingParameterFile, + QgsProcessingParameterString, + QgsProcessingAlgorithm, + QgsProcessingMultiStepFeedback, + QgsProcessingParameterNumber, + QgsProcessingParameterFileDestination, +) +from processing.gui.wrappers import WidgetWrapper +from qgis.PyQt.QtWidgets import QLineEdit +from DsgTools.core.dsgEnums import DsgEnums +from DsgTools.core.Factories.DbFactory.dbFactory import DbFactory +from collections import defaultdict +import json + + +class IdentifyDifferencesBetweenDatabaseModelsAlgorithm(QgsProcessingAlgorithm): + MASTERFILE = "MASTERFILE" + SERVERIP = "SERVERIP" + PORT = "PORT" + DBNAME = "DBNAME" + USER = "USER" + PASSWORD = "PASSWORD" + OUTPUT = "OUTPUT" + + def initAlgorithm(self, config): + """ + Parameter setting. + """ + self.addParameter( + QgsProcessingParameterFile( + self.MASTERFILE, self.tr("Path of Masterfile"), extension="json" + ) + ) + + self.addParameter(QgsProcessingParameterString(self.SERVERIP, self.tr("IP"))) + + self.addParameter( + QgsProcessingParameterNumber( + self.PORT, + self.tr("Port"), + minValue=0, + maxValue=9999, + defaultValue=5432, + ) + ) + + self.addParameter( + QgsProcessingParameterString(self.DBNAME, self.tr("Database Name")) + ) + + self.addParameter( + QgsProcessingParameterString(self.USER, self.tr("Database User")) + ) + + password = QgsProcessingParameterString( + self.PASSWORD, + self.tr("Database Password"), + ) + password.setMetadata( + { + "widget_wrapper": "DsgTools.core.DSGToolsProcessingAlgs.Algs.OtherAlgs.identifyDifferencesBetweenDatabaseModelsAlgorithm.MyWidgetWrapper" + } + ) + + self.addParameter(password) + + self.addParameter( + QgsProcessingParameterFileDestination( + self.OUTPUT, + self.tr("Path to save .txt File"), + fileFilter=".txt", + createByDefault=True, + ) + ) + + def processAlgorithm(self, parameters, context, feedback): + """ + Here is where the processing itself takes place. + """ + masterFile = self.parameterAsFile(parameters, self.MASTERFILE, context) + serverIp = self.parameterAsString(parameters, self.SERVERIP, context) + port = self.parameterAsInt(parameters, self.PORT, context) + dbName = self.parameterAsString(parameters, self.DBNAME, context) + user = self.parameterAsString(parameters, self.USER, context) + password = self.parameterAsString(parameters, self.PASSWORD, context) + fileTxt = self.parameterAsFileOutput(parameters, self.OUTPUT, context) + abstractDb = self.getAbstractDb( + host=serverIp, + port=port, + database=dbName, + user=user, + password=password, + ) + masterDict = self.getMasterDict(masterFile) + msg = "" + multiStepFeedback = QgsProcessingMultiStepFeedback(4, feedback) + currentStep = 0 + multiStepFeedback.setCurrentStep(currentStep) + msg += self.validateDatabaseVersionAndImplementation(masterDict, abstractDb) + if multiStepFeedback.isCanceled(): + self.pushOutputMessage(feedback, msg) + return {} + + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + msg += self.validateDomainTables(masterDict, abstractDb) + if multiStepFeedback.isCanceled(): + self.pushOutputMessage(feedback, msg) + return {} + + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + outputMsg, nameTableMsgDict = self.validateEDGVTables(masterDict, abstractDb) + msg += outputMsg + if multiStepFeedback.isCanceled(): + self.pushOutputMessage(feedback, msg) + return {} + + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + nameTableMsgDictTwo = self.validateCheckConstraint(masterDict, abstractDb) + if multiStepFeedback.isCanceled(): + self.pushOutputMessage(feedback, msg) + return {} + + nameTableMsgDict.update(nameTableMsgDictTwo) + + for table in nameTableMsgDict: + msg += f"Erro na tabela = {table}: \n" + for typeMsg in nameTableMsgDict[table]: + if nameTableMsgDict[table][typeMsg] == []: + continue + msg += f" {typeMsg}\n" + msg += f" " + for element in nameTableMsgDict[table][typeMsg]: + msg += f"\t- {element}\n" + msg = msg[: len(msg) - 2] + "\n" + msg += "\n" + + self.pushOutputMessage(feedback, msg, fileTxt) + + return {self.OUTPUT: fileTxt} + + def pushOutputMessage(self, feedback, msg, fileTxt): + if msg == "": + feedback.pushInfo( + "A estrutura do banco de entrada corresponde à estrutura definida pelo masterfile de entrada." + ) + with open(f"{fileTxt}", "w") as file: + file.write( + "A estrutura do banco de entrada corresponde à estrutura definida pelo masterfile de entrada." + ) + else: + feedback.pushInfo( + "A estrutura do banco de entrada não corresponde à estrutura definida pelo masterfile de entrada:" + ) + feedback.pushInfo(msg) + with open(f"{fileTxt}", "w") as file: + file.write( + "A estrutura do banco de entrada não corresponde à estrutura definida pelo masterfile de entrada:\n" + ) + file.write(f"{msg}") + file.close() + + def getAbstractDb(self, host, port, database, user, password): + abstractDb = DbFactory().createDbFactory(DsgEnums.DriverPostGIS) + abstractDb.connectDatabaseWithParameters(host, port, database, user, password) + return abstractDb + + def getMasterDict(self, masterFile): + """ + Leitura do masterfile + """ + masterFile = open(masterFile) + masterDict = json.load(masterFile) + return masterDict + + def validateDatabaseVersionAndImplementation(self, masterDict, abstractDb): + msg = "" + ( + edgvVersion, + implementationVersion, + ) = abstractDb.getDatabaseAndImplementationVersions() + if ( + edgvVersion != masterDict["modelo"] + or implementationVersion != masterDict["versao"] + ): + msg += "Erro de versão:\n" + if edgvVersion != masterDict["modelo"]: + msg += f" A versão do banco ({edgvVersion}) não corresponde à versão do masterfile ({masterDict['modelo']})\n" + if implementationVersion != masterDict["versao"]: + msg += f" A versão de implementação do banco ({implementationVersion}) não corresponde à versão de implementação do masterfile ({masterDict['versao']})\n" + return msg + + def validateDomainTables(self, masterDict, abstractDb): + """ + 1. Validar schema do domínio + 2. Pegar conjunto de nome de domínios do masterDict; + 3. Pegar conjunto de nome de domínios do banco; + 4. Identificar domínios que existem no masterDict, mas não existem no banco (tabelas que faltam no banco); + 5. Identificar domínios que existem no banco, mas não estão previstas no masterDict (tabelas excedentes no banco); + """ + msg = "" + if not abstractDb.checkIfSchemaExistsInDatabase(masterDict["schema_dominios"]): + msg += "Erro no Schema de domínios:\n" + msg += f" A o esquema de domínios {masterDict['schema_dominios']} não está implementado no banco." + return msg + masterDictDomainNameSet = set(i["nome"] for i in masterDict["dominios"]) + dbDomainNameSet = abstractDb.getTableListFromSchema( + masterDict["schema_dominios"] + ) + + inMasterDictNotInDbSet = masterDictDomainNameSet.difference(dbDomainNameSet) + inDbNotInMasterDictSet = dbDomainNameSet.difference(masterDictDomainNameSet) + + if len(inMasterDictNotInDbSet) > 0 or len(inDbNotInMasterDictSet) > 0: + msg += "Erro, há disparidade entre as tabelas do banco de dados e do Masterfile no Schema dominios:\n" + + if len(inMasterDictNotInDbSet) > 0: + msg += " Os domínios que existem no masterDict, mas não exitem no banco (tabelas que faltam no banco) são: " + for e in inMasterDictNotInDbSet: + msg += f"{e}, " + msg = msg[: len(msg) - 2] + "\n\n" + + if len(inDbNotInMasterDictSet) > 0: + msg += " Os domínios que existem no banco, mas não estão previstas no masterDict (tabelas excedentes no banco) são: " + for e in inDbNotInMasterDictSet: + msg += f"{e}, " + msg = msg[: len(msg) - 2] + "\n\n" + domainDict = {i["nome"]: i["valores"] for i in masterDict["dominios"]} + nameIdxDict = { + i["nome"]: masterDict["dominios"].index(i) for i in masterDict["dominios"] + } + attributeNameDict = { + "code": "code", + "value": "code_name", + "valor_filtro": "filter", + } + tableColumnsSetDict = abstractDb.getColumnsDictFromSchema( + masterDict["schema_dominios"] + ) + tablePrimaryKeySetDict = abstractDb.getPrimaryKeyDictFromSchema( + masterDict["schema_dominios"] + ) + for domainName in masterDictDomainNameSet.intersection(dbDomainNameSet): + # 1. verificar se existem as colunas code e value + columnsMasterFileSet = set() + for column in domainDict[domainName][0]: + if column in attributeNameDict: + columnsMasterFileSet.add(attributeNameDict[column]) + if tableColumnsSetDict[domainName] != columnsMasterFileSet: + inMasterDictNotInDbSet = columnsMasterFileSet.difference( + tableColumnsSetDict[domainName] + ) + inDbNotInMasterDictSet = tableColumnsSetDict[domainName].difference( + columnsMasterFileSet + ) + msg += "Erro no Schema dominios colunas 'code' e 'value':\n" + msg += f" A tabela {domainName} " + if len(inMasterDictNotInDbSet) > 0: + msg += "possui as seguintes colunas no MasterFile, mas não no banco de dados:\n" + msg += ", ".join(list(inMasterDictNotInDbSet)) + "\n\n" + if len(inDbNotInMasterDictSet) > 0: + msg += "possui as seguintes colunas no banco de dados, mas não no MasterFile:\n" + msg += ", ".join(list(inDbNotInMasterDictSet)) + "\n\n" + continue + + # 2. verificar se a chave primária é a coluna code + setPrimaryKey = tablePrimaryKeySetDict[domainName] + if len(setPrimaryKey) > 1: + msg += "Erro Primary Key:\n" + msg += " A coluna 'code' deve ser a chave primária, mas foram passadas como chaves primárias:\n" + msg += ", ".join(list(setPrimaryKey)) + "\n\n" + elif len(setPrimaryKey) == 1: + for pk in setPrimaryKey: + break + if pk != "code": + msg += "Erro Primary Key:\n" + msg += f" A coluna 'code' deve ser a chave primária da tabela {domainName}, mas a chava primária passada foi: {pk}\n\n" + else: + msg += "Erro Primary Key:\n" + msg += f" A tabela {domainName} não possui chave primária.\n\n" + # 3. comparar os valores do masterfile, incluindo o valor a ser preenchido, com os valores populados no banco + # Início da parte 3 + # monta o conjunto de tuplas do masterfile + if not masterDict["dominios"][nameIdxDict[domainName]].get("filtro", False): + masterFileDomainTupleSet = { + ( + masterDict["a_ser_preenchido"]["code"], + masterDict["a_ser_preenchido"]["value"] + + f' ({masterDict["a_ser_preenchido"]["code"]})', + ) + } + masterFileDomainTupleSet |= set( + (i["code"], i["value"] + f' ({i["code"]})') + for i in domainDict[domainName] + ) + columnNameList = ["code", "code_name"] + else: + masterFileDomainTupleSet = { + ( + masterDict["a_ser_preenchido"]["code"], + masterDict["a_ser_preenchido"]["value"] + + f' ({masterDict["a_ser_preenchido"]["code"]})', + masterDict["a_ser_preenchido"]["value"] + + f' ({masterDict["a_ser_preenchido"]["code"]})', + ) + } + masterFileDomainTupleSet |= set( + (i["code"], i["value"] + f' ({i["code"]})', i["valor_filtro"]) + for i in domainDict[domainName] + ) + columnNameList = ["code", "code_name", "filter"] + # monta o conjunto de tuplas do banco + + dbDomainTupleSet = abstractDb.getTupleSetFromTable( + schemaName=masterDict["schema_dominios"], + tableName=domainName, + columnNameList=columnNameList, + ) + + if masterFileDomainTupleSet == dbDomainTupleSet: + continue + else: + inMasterFileDomainNotInDbDomainSet = ( + masterFileDomainTupleSet.difference(dbDomainTupleSet) + ) + inDbDomainNotInMasterFileDomainSet = dbDomainTupleSet.difference( + masterFileDomainTupleSet + ) + msg += "Erro valores no Schema dominios:\n" + msg += f" A tabela {domainName} apresenta os seguintes erros:\n" + if len(inMasterFileDomainNotInDbDomainSet) > 0: + if not masterDict["dominios"][nameIdxDict[domainName]].get( + "filtro", False + ): + msg += f"- Valores no MasterFile sem correspondência no banco de dados: " + for valor in inMasterFileDomainNotInDbDomainSet: + msg += f"'code': {valor[0]}, 'code_name': {valor[1]}" + msg += "\n" + else: + msg += f"- Valores no MasterFile sem correspondência no banco de dados: " + for valor in inMasterFileDomainNotInDbDomainSet: + msg += f"'code': {valor[0]}, 'code_name': {valor[1]}, 'filter': {valor[2]}" + msg += "\n" + if len(inDbDomainNotInMasterFileDomainSet) > 0: + if not masterDict["dominios"][nameIdxDict[domainName]].get( + "filtro", False + ): + msg += f"- Valores no banco de dados sem correspondência no MasterFile: " + for valor in inDbDomainNotInMasterFileDomainSet: + msg += f"'code': {valor[0]}, 'code_name': {valor[1]}" + msg += "\n" + else: + msg += f"- Valores no banco de dados sem correspondência no MasterFile: " + for valor in inDbDomainNotInMasterFileDomainSet: + msg += f"'code': {valor[0]}, 'code_name': {valor[1]}, 'filter': {valor[2]}" + msg += "\n" + msg += "\n" + return msg + + def validateEDGVTables(self, masterDict, abstractDb): + """ + 1 - verificar se o schema edgv faz parte do database + 2 - validar schema edgv, verificando a presença de todas as tabelas que são necessárias + 3 - validar a chave primária da tabela como sendo o 'id' + 4 - validas as colunas de cada tabela do schema edgv juntamente com os seus valores + """ + msg = "" + if not abstractDb.checkIfSchemaExistsInDatabase(masterDict["schema_dados"]): + msg += f"""Erro Schema {masterDict["schema_dados"]}: \n""" + msg += f""" O schema '{masterDict["schema_dados"]}' não está presente no banco de dados.\n""" + return msg + dbEDGVNameSet = abstractDb.getTableListFromSchema(masterDict["schema_dados"]) + masterDictEDGVSet = set( + f'{i["categoria"]}_{i["nome"]}{masterDict["geom_suffix"][j]}' + for i in masterDict["classes"] + for j in i["primitivas"] + ) + masterDictEDGVSet |= set( + f'{i["categoria"]}_{i["nome"]}{masterDict["geom_suffix"][j]}' + for i in masterDict["extension_classes"] + for j in i["primitivas"] + ) + inMasterDictNotInDbSet = masterDictEDGVSet.difference(dbEDGVNameSet) + inDbNotInMasterDictSet = dbEDGVNameSet.difference(masterDictEDGVSet) + if len(inMasterDictNotInDbSet) > 0 or len(inDbNotInMasterDictSet) > 0: + msg += "Erro, há divergência de tabelas no banco de dados e Masterfile:\n" + if len(inMasterDictNotInDbSet) > 0: + msg += f""" As tabelas do {masterDict["schema_dados"]} que estão no MasterFile, mas não estão do banco de dados são: """ + for table in inMasterDictNotInDbSet: + msg += f"{table}, " + msg = msg[: len(msg) - 2] + "\n\n" + if len(inDbNotInMasterDictSet) > 0: + msg += f""" As tabelas do {masterDict["schema_dados"]} que estão no banco de dados, mas não estão no MasterFile são: """ + for table in inDbNotInMasterDictSet: + msg += f"{table}, " + msg = msg[: len(msg) - 2] + "\n\n" + + tableNamePrimaryKeyDict = abstractDb.getPrimaryKeyDictFromSchema( + masterDict["schema_dados"] + ) + nameTableMsgDict = defaultdict(dict) + for tableName in tableNamePrimaryKeyDict: + if tableNamePrimaryKeyDict[tableName] == {"id"}: + continue + if not nameTableMsgDict[tableName].get( + "chave primária diferente do 'id' a chava primária é/são da tabela: ", + False, + ): + nameTableMsgDict[tableName][ + "chave primária diferente do 'id' a chava primária é/são da tabela: " + ] = [] + for pk in tableNamePrimaryKeyDict[tableName]: + nameTableMsgDict[tableName][ + "chave primária diferente do 'id' a chava primária é/são da tabela: " + ].append(f"{pk}, ") + + edgvDict = { + f'{i["categoria"]}_{i["nome"]}{masterDict["geom_suffix"][j]}': i[ + "atributos" + ] + for i in masterDict["classes"] + for j in i["primitivas"] + } + edgvDictExtensionClasses = { + f'{i["categoria"]}_{i["nome"]}{masterDict["geom_suffix"][j]}': i[ + "atributos" + ] + for i in masterDict["extension_classes"] + for j in i["primitivas"] + } + edgvDict.update(edgvDictExtensionClasses) + geomSuffixDict = { + "MultiPoint": "_p", + "MultiLinestring": "_l", + "MultiPolygon": "_a", + } + nameTableColumnIsNullableOrNoForeignTypeTableDict = ( + abstractDb.getTypeColumnFromSchema(masterDict["schema_dados"]) + ) + nameTableColumnMaxVarcharDict = abstractDb.getMaxLengthVarcharFromSchema( + masterDict["schema_dados"] + ) + for edgvName in masterDictEDGVSet.intersection(dbEDGVNameSet): + masterNameTypeCardinalityMapValueSet = set() + masterNameTypeCardinalityMapValueSet |= { + ("geom", "USER-DEFINED", "YES"), + ("observacao", "varchar(255)", "YES"), + } + for attribute in edgvDict[edgvName]: + if attribute.get("primitivas", False): + geomList = attribute["primitivas"] + geomSuffixSet = set(geomSuffixDict[i] for i in geomList) + if ("_" + edgvName[-1]) in geomSuffixSet: + if attribute["cardinalidade"] == "0..1": + masterNameTypeCardinalityMapValueSet |= { + (attribute["nome"], attribute["tipo"], "YES") + } + elif attribute["cardinalidade"] == "1..1": + if not attribute.get("mapa_valor", False): + masterNameTypeCardinalityMapValueSet |= { + (attribute["nome"], attribute["tipo"], "NO") + } + else: + masterNameTypeCardinalityMapValueSet |= { + ( + attribute["nome"], + attribute["tipo"], + "NO", + attribute["mapa_valor"], + ) + } + else: + if attribute["cardinalidade"] == "0..1": + masterNameTypeCardinalityMapValueSet |= { + (attribute["nome"], attribute["tipo"], "YES") + } + elif attribute["cardinalidade"] == "1..1": + if not attribute.get("mapa_valor", False): + masterNameTypeCardinalityMapValueSet |= { + (attribute["nome"], attribute["tipo"], "NO") + } + else: + masterNameTypeCardinalityMapValueSet |= { + ( + attribute["nome"], + attribute["tipo"], + "NO", + attribute["mapa_valor"], + ) + } + dbNameTypeCardinalityMapValueSet = set() + for column in nameTableColumnIsNullableOrNoForeignTypeTableDict[edgvName]: + if column == "id": + continue + typeColumn = nameTableColumnIsNullableOrNoForeignTypeTableDict[ + edgvName + ][column]["type"] + if typeColumn == "character varying": + maxVarchar = nameTableColumnMaxVarcharDict[edgvName][column] + typeColumn = f"varchar({maxVarchar})" + if ( + nameTableColumnIsNullableOrNoForeignTypeTableDict[edgvName][column][ + "nullable" + ] + == "YES" + ): + dbNameTypeCardinalityMapValueSet |= {(column, typeColumn, "YES")} + else: + if not nameTableColumnIsNullableOrNoForeignTypeTableDict[edgvName][ + column + ].get("foreignTable", False): + dbNameTypeCardinalityMapValueSet |= {(column, typeColumn, "NO")} + else: + mapValue = nameTableColumnIsNullableOrNoForeignTypeTableDict[ + edgvName + ][column]["foreignTable"] + dbNameTypeCardinalityMapValueSet |= { + (column, typeColumn, "NO", mapValue) + } + if masterNameTypeCardinalityMapValueSet == dbNameTypeCardinalityMapValueSet: + continue + else: + inMasterNotInDbSet = masterNameTypeCardinalityMapValueSet.difference( + dbNameTypeCardinalityMapValueSet + ) + inDbNotInMasterSet = dbNameTypeCardinalityMapValueSet.difference( + masterNameTypeCardinalityMapValueSet + ) + if len(inMasterNotInDbSet) > 0 or len(inDbNotInMasterSet) > 0: + if not nameTableMsgDict[edgvName].get( + "Os seguintes valores estão presentes no banco de dados, mas não estão no Masterfile: ", + False, + ): + nameTableMsgDict[edgvName][ + "Os seguintes valores estão presentes no banco de dados, mas não estão no Masterfile: " + ] = [] + if not nameTableMsgDict[edgvName].get( + "Os seguintes valores estão presentes no Masterfile, mas não estão no banco de dados: ", + False, + ): + nameTableMsgDict[edgvName][ + "Os seguintes valores estão presentes no Masterfile, mas não estão no banco de dados: " + ] = [] + if len(inMasterNotInDbSet) > 0: + for valor in inMasterNotInDbSet: + if len(valor) == 3: + nameTableMsgDict[edgvName][ + "Os seguintes valores estão presentes no Masterfile, mas não estão no banco de dados: " + ].append( + f"coluna: {valor[0]}, tipo: {valor[1]}, opcional: {valor[2]}, " + ) + else: + nameTableMsgDict[edgvName][ + "Os seguintes valores estão presentes no Masterfile, mas não estão no banco de dados: " + ].append( + f"coluna: {valor[0]}, tipo: {valor[1]}, opcional: {valor[2]}, mapa_valor = {valor[3]}, " + ) + if len(inDbNotInMasterSet) > 0: + for valor in inDbNotInMasterSet: + if len(valor) == 3: + nameTableMsgDict[edgvName][ + "Os seguintes valores estão presentes no banco de dados, mas não estão no Masterfile: " + ].append( + f"coluna: {valor[0]}, tipo: {valor[1]}, opcional: {valor[2]}, " + ) + else: + nameTableMsgDict[edgvName][ + "Os seguintes valores estão presentes no banco de dados, mas não estão no Masterfile: " + ].append( + f"coluna: {valor[0]}, tipo: {valor[1]}, opcional: {valor[2]}, mapa_valor = {valor[3]}, " + ) + return msg, nameTableMsgDict + + def validateCheckConstraint(self, masterDict, abstractDb): + """ """ + dbNameTableColumnCheckConstraintDict = abstractDb.getCheckConstraintDict() + for table in dbNameTableColumnCheckConstraintDict: + for column in dbNameTableColumnCheckConstraintDict[table]: + dbNameTableColumnCheckConstraintDict[table][column] = set( + dbNameTableColumnCheckConstraintDict[table][column] + ) + + masterNameTableColumnCheckConstraintDict = defaultdict(dict) + for table in masterDict["classes"]: + self.nameTableColumnCheckConstraint( + masterDict, table, masterNameTableColumnCheckConstraintDict + ) + + for table in masterDict["extension_classes"]: + self.nameTableColumnCheckConstraint( + masterDict, table, masterNameTableColumnCheckConstraintDict + ) + + correspNameTableMapValueColumnDict = defaultdict(dict) + for table in masterDict["classes"]: + self.nameTableMapValueColumn( + masterDict, table, correspNameTableMapValueColumnDict + ) + + for table in masterDict["extension_classes"]: + self.nameTableMapValueColumn( + masterDict, table, correspNameTableMapValueColumnDict + ) + + nameColumnCodeDict = defaultdict(set) + for dominio in masterDict["dominios"]: + for value in dominio["valores"]: + nameColumnCodeDict[dominio["nome"]].add(value["code"]) + + nameTableColumnForeignSetDict = defaultdict(dict) + for table in correspNameTableMapValueColumnDict: + for mapValue in correspNameTableMapValueColumnDict[table]: + if mapValue not in nameColumnCodeDict: + continue + nameTableColumnForeignSetDict[table][ + correspNameTableMapValueColumnDict[table][mapValue] + ] = nameColumnCodeDict[mapValue] + + tableToRemoveSet = self.tableToRemove( + masterNameTableColumnCheckConstraintDict, nameTableColumnForeignSetDict + ) + + for table in tableToRemoveSet: + masterNameTableColumnCheckConstraintDict.pop(table) + + for table in masterNameTableColumnCheckConstraintDict: + for column in masterNameTableColumnCheckConstraintDict[table]: + masterNameTableColumnCheckConstraintDict[table][column].add(9999) + + for table in dbNameTableColumnCheckConstraintDict: + for column in dbNameTableColumnCheckConstraintDict[table]: + nameTableMsgDict = self.validateCheckConstraintSet( + dbNameTableColumnCheckConstraintDict, + table, + column, + masterNameTableColumnCheckConstraintDict, + ) + + for table in masterNameTableColumnCheckConstraintDict: + for column in masterNameTableColumnCheckConstraintDict[table]: + nameTableMsgDictTwo = self.validateCheckConstraintSet( + dbNameTableColumnCheckConstraintDict, + table, + column, + masterNameTableColumnCheckConstraintDict, + ) + nameTableMsgDict.update(nameTableMsgDictTwo) + return nameTableMsgDict + + def tableToRemove( + self, masterNameTableColumnCheckConstraintDict, nameTableColumnForeignSetDict + ): + tableForRemove = set() + for table in masterNameTableColumnCheckConstraintDict: + for column in masterNameTableColumnCheckConstraintDict[table]: + if not nameTableColumnForeignSetDict.get(table, False): + continue + if not nameTableColumnForeignSetDict[table].get(column, False): + continue + foreignSet = nameTableColumnForeignSetDict[table][column] + checkSet = masterNameTableColumnCheckConstraintDict[table][column] + if foreignSet != checkSet: + continue + tableForRemove.add(table) + return tableForRemove + + def nameTableMapValueColumn( + self, masterDict, table, correspNameTableMapValueColumnDict + ): + for attribute in table["atributos"]: + if not attribute.get("valores", False): + continue + nameWithoutGeomSuffix = f'{table["categoria"]}_{table["nome"]}' + if type(attribute["valores"]) == list: + if type(attribute["valores"][0]) == int: + nameWithGeomSuffixSet = set( + nameWithoutGeomSuffix + masterDict["geom_suffix"][i] + for i in table["primitivas"] + ) + for name in nameWithGeomSuffixSet: + if correspNameTableMapValueColumnDict[name].get( + attribute["mapa_valor"], False + ): + continue + correspNameTableMapValueColumnDict[name][ + attribute["mapa_valor"] + ] = attribute["nome"] + else: + for value in attribute["valores"]: + if not value.get("primitivas", False): + nameWithGeomSuffixSet = set( + nameWithoutGeomSuffix + i for i in ["_p", "_l", "_a"] + ) + else: + nameWithGeomSuffixSet = set( + nameWithoutGeomSuffix + masterDict["geom_suffix"][i] + for i in value["primitivas"] + ) + for name in nameWithGeomSuffixSet: + if correspNameTableMapValueColumnDict[name].get( + attribute["mapa_valor"], False + ): + continue + correspNameTableMapValueColumnDict[name][ + attribute["mapa_valor"] + ] = attribute["nome"] + else: + for value in attribute["valores"]: + nameWithSuffixGeom = ( + nameWithoutGeomSuffix + masterDict["geom_suffix"][value] + ) + if correspNameTableMapValueColumnDict[nameWithSuffixGeom].get( + attribute["mapa_valor"], False + ): + continue + correspNameTableMapValueColumnDict[nameWithSuffixGeom][ + attribute["mapa_valor"] + ] = attribute["nome"] + + def validateCheckConstraintSet( + self, + dbNameTableColumnCheckConstraintDict, + table, + column, + masterNameTableColumnCheckConstraintDict, + ): + dbCheckConstraintSet = dbNameTableColumnCheckConstraintDict[table][column] + masterCheckConstraintSet = masterNameTableColumnCheckConstraintDict[table][ + column + ] + inDbNotInMasterSet = dbCheckConstraintSet.difference(masterCheckConstraintSet) + inMasterNotInMasterSet = masterCheckConstraintSet.difference( + dbCheckConstraintSet + ) + nameTableMsgDict = defaultdict(dict) + if len(inDbNotInMasterSet) > 0 or len(inMasterNotInMasterSet) > 0: + if not nameTableMsgDict[table].get( + "As chaves check que estão presentes no banco de dados, mas não estão no MasterFile: ", + False, + ): + nameTableMsgDict[table][ + "As chaves check que estão presentes no banco de dados, mas não estão no MasterFile: " + ] = [] + if not nameTableMsgDict[table].get( + "As chaves check que estão presentes no Materfile, mas não estão no banco de dados: ", + False, + ): + nameTableMsgDict[table][ + "As chaves check que estão presentes no Materfile, mas não estão no banco de dados: " + ] = [] + if len(inDbNotInMasterSet) > 0: + for check in inDbNotInMasterSet: + nameTableMsgDict[table][ + "As chaves check que estão presentes no banco de dados, mas não estão no MasterFile: " + ].append(f"{check}, ") + if len(inMasterNotInMasterSet) > 0: + for check in inMasterNotInMasterSet: + nameTableMsgDict[table][ + "As chaves check que estão presentes no Materfile, mas não estão no banco de dados: " + ].append(f"{check}, ") + return nameTableMsgDict + + def nameTableColumnCheckConstraint( + self, masterDict, table, masterNameTableColumnCheckConstraintDict + ): + for attribute in table["atributos"]: + if not attribute.get("valores", False): + continue + nameWithoutGeomSuffix = f'{table["categoria"]}_{table["nome"]}' + if type(attribute["valores"]) == list: + if type(attribute["valores"][0]) == int: + nameWithGeomSuffixSet = set( + nameWithoutGeomSuffix + masterDict["geom_suffix"][i] + for i in table["primitivas"] + ) + for name in nameWithGeomSuffixSet: + masterNameTableColumnCheckConstraintDict[name][ + attribute["nome"] + ] = set(attribute["valores"]) + else: + for value in attribute["valores"]: + if not value.get("primitivas", False): + nameWithGeomSuffixSet = set( + nameWithoutGeomSuffix + i for i in ["_p", "_l", "_a"] + ) + else: + nameWithGeomSuffixSet = set( + nameWithoutGeomSuffix + masterDict["geom_suffix"][i] + for i in value["primitivas"] + ) + for name in nameWithGeomSuffixSet: + if not masterNameTableColumnCheckConstraintDict[name].get( + attribute["nome"], False + ): + masterNameTableColumnCheckConstraintDict[name][ + attribute["nome"] + ] = {value["code"]} + else: + masterNameTableColumnCheckConstraintDict[name][ + attribute["nome"] + ].add(value["code"]) + else: + for value in attribute["valores"]: + nameWithSuffixGeom = ( + nameWithoutGeomSuffix + masterDict["geom_suffix"][value] + ) + masterNameTableColumnCheckConstraintDict[nameWithSuffixGeom][ + attribute["nome"] + ] = set(attribute["valores"][value]) + + def name(self): + """ + Returns the algorithm name, used for identifying the algorithm. This + string should be fixed for the algorithm, and must not be localised. + The name should be unique within each provider. Names should contain + lowercase alphanumeric characters only and no spaces or other + formatting characters. + """ + return "identifydifferencesbetweendatabasemodelsalgorithm" + + def displayName(self): + """ + Returns the translated algorithm name, which should be used for any + user-visible display of the algorithm name. + """ + return self.tr("Identify Differences Between banco de dados Models") + + def group(self): + """ + Returns the name of the group this algorithm belongs to. This string + should be localised. + """ + return self.tr("Other Algorithms") + + def groupId(self): + """ + Returns the unique ID of the group this algorithm belongs to. This + string should be fixed for the algorithm, and must not be localised. + The group id should be unique within each provider. Group id should + contain lowercase alphanumeric characters only and no spaces or other + formatting characters. + """ + return "DSGTools: Other Algorithms" + + def tr(self, string): + return QCoreApplication.translate( + "IdentifyDifferencesBetweenDatabaseModelsAlgorithm", string + ) + + def createInstance(self): + return IdentifyDifferencesBetweenDatabaseModelsAlgorithm() + + +class MyWidgetWrapper(WidgetWrapper): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.placeholder = args[0] + + def createWidget(self): + self._lineedit = QLineEdit() + self._lineedit.setEchoMode(QLineEdit.Password) + # if self.placeholder: + # self._lineedit.setPlaceholderText(self.placeholder) + return self._lineedit + + def value(self): + return self._lineedit.text() diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/detectChangesGroupAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/detectChangesGroupAlgorithm.py new file mode 100644 index 000000000..779458331 --- /dev/null +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/detectChangesGroupAlgorithm.py @@ -0,0 +1,465 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + DsgTools + A QGIS plugin + Brazilian Army Cartographic Production Tools + ------------------- + begin : 2023-08-18 + git sha : $Format:%H$ + copyright : (C) 2018 by Matheus Alves Silva - Cartographic Engineer @ Brazilian Army + email : matheus.silva@ime.eb.br + ***************************************************************************/ +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +from DsgTools.core.DSGToolsProcessingAlgs.algRunner import AlgRunner +from PyQt5 import QtWidgets +from PyQt5.QtCore import QCoreApplication, QVariant +from processing.gui.wrappers import WidgetWrapper +from qgis.core import ( + QgsFeature, + QgsFeatureSink, + QgsProcessingParameterDefinition, + QgsProcessingParameterString, + QgsProcessingException, + QgsProcessingMultiStepFeedback, + QgsFields, + QgsProcessingParameterFeatureSink, + QgsField, + QgsProject, + QgsWkbTypes, +) + +from .validationAlgorithm import ValidationAlgorithm + + +class DetectChangesGroupAlgorithm(ValidationAlgorithm): + ORIGINAL = "ORIGINAL" + REVIEWED = "REVIEWED" + BLACK_ATTRIBUTES = "BLACK_ATTRIBUTES" + GROUP = "GROUP" + POINT_FLAG = "POINT_FLAG" + LINE_FLAG = "LINE_FLAG" + POLYGON_FLAG = "POLYGON_FLAG" + + def initAlgorithm(self, config): + """ + Parameter setting. + """ + self.addParameter( + ParameterGroup( + self.ORIGINAL, + self.tr("Input group original"), + ) + ) + + self.addParameter( + ParameterGroup( + self.REVIEWED, + self.tr("Input group reviewed"), + ) + ) + + self.addParameter( + QgsProcessingParameterString( + self.BLACK_ATTRIBUTES, + self.tr("Select the attributes to ignore"), + optional=True, + ) + ) + + self.addParameter( + QgsProcessingParameterString(self.GROUP, self.tr("Attribute to group by")) + ) + + self.addParameter( + QgsProcessingParameterFeatureSink( + self.POINT_FLAG, self.tr("{0} Point Flags").format(self.displayName()) + ) + ) + + self.addParameter( + QgsProcessingParameterFeatureSink( + self.LINE_FLAG, self.tr("{0} Line Flags").format(self.displayName()) + ) + ) + + self.addParameter( + QgsProcessingParameterFeatureSink( + self.POLYGON_FLAG, + self.tr("{0} Polygon Flags").format(self.displayName()), + ) + ) + + def parameterAsGroup(self, parameters, name, context): + return parameters[name] + + def processAlgorithm(self, parameters, context, feedback): + """ + Here is where the processing itself takes place. + """ + algRunner = AlgRunner() + groupOriginal = self.parameterAsGroup(parameters, self.ORIGINAL, context) + groupReviewed = self.parameterAsGroup(parameters, self.REVIEWED, context) + strBlackAttributes = self.parameterAsString( + parameters, self.BLACK_ATTRIBUTES, context + ) + attributeGroup = self.parameterAsString(parameters, self.GROUP, context) + + if not groupOriginal or not groupReviewed: + raise QgsProcessingException( + "Must have a input group original and input review" + ) + + project = context.project() + groupO = project.layerTreeRoot().findGroup(groupOriginal) + groupR = project.layerTreeRoot().findGroup(groupReviewed) + + fields = self.fieldsFlag(attributeGroup) + + if not groupO or not groupR: + raise QgsProcessingException( + "Input group original and input review group not found" + ) + + dictLyrsOriginals, crs = self.dictNameLyrCrs(groupO) + dictLyrsReviewed, crs = self.dictNameLyrCrs(groupR) + + point_flag_sink, point_flag_id = self.sinkLyr( + parameters, self.POINT_FLAG, context, fields, QgsWkbTypes.Point, crs + ) + line_flag_sink, line_flag_id = self.sinkLyr( + parameters, self.LINE_FLAG, context, fields, QgsWkbTypes.LineString, crs + ) + poly_flag_sink, poly_flag_id = self.sinkLyr( + parameters, self.POLYGON_FLAG, context, fields, QgsWkbTypes.Polygon, crs + ) + + multiStepFeedback = QgsProcessingMultiStepFeedback( + len(dictLyrsOriginals), feedback + ) + nSteps = len(dictLyrsOriginals) + if nSteps == 0: + return { + self.POINT_FLAG: "", + self.LINE_FLAG: "", + self.POLYGON_FLAG: "", + } + + for current, nameLyrOriginal in enumerate(dictLyrsOriginals): + if multiStepFeedback.isCanceled(): + break + multiStepFeedback.setCurrentStep(current) + + if nameLyrOriginal not in dictLyrsReviewed: + raise QgsProcessingException( + "There is no correspondence of layers between the groups" + ) + + lyrOriginal = dictLyrsOriginals[nameLyrOriginal] + lyrReviewed = dictLyrsReviewed[nameLyrOriginal] + + listWhiteAttributes = self.compareAttributes( + strBlackAttributes, lyrOriginal + ) + + unchangedLyr, addedLyr, deletedLyr = algRunner.runDetectDatasetChanges( + inputLayer=lyrOriginal, + reviewedLayer=lyrReviewed, + attributesList=listWhiteAttributes, + matchComparation=0, + context=context, + ) + + lyrPoint, lyrLine, lyrPolygon = self.typeOfLayer(addedLyr) + + # set of id feature in deleted layer + setDelAddFeat = set() + + nStepsAddedLyr = addedLyr.featureCount() + multiStep = QgsProcessingMultiStepFeedback( + nStepsAddedLyr, multiStepFeedback + ) + + for current1, featureAdd in enumerate(addedLyr.getFeatures()): + if multiStep.isCanceled(): + break + multiStep.setCurrentStep(current1) + bothLyr = False + differentAttribute = False + differentGeometry = False + nStepsDel = deletedLyr.featureCount() + if nStepsDel == 0: + continue + stepSize = 100 / nStepsDel + for current2, featureDel in enumerate(deletedLyr.getFeatures()): + if multiStep.isCanceled(): + break + if ( + featureAdd[f"{attributeGroup}"] + != featureDel[f"{attributeGroup}"] + ): + continue + setDelAddFeat.add(featureDel[f"{attributeGroup}"]) + flagMsg = "" + wktFeatAdd = self.geomWkt(featureAdd) + wktFeatDel = self.geomWkt(featureDel) + if wktFeatAdd != wktFeatDel: + flagMsg += "Different geometry, " + differentGeometry = True + for attribute in listWhiteAttributes: + if featureAdd[attribute] == featureDel[attribute]: + continue + flagMsg += f"{attribute}, " + differentAttribute = True + if not (differentAttribute or differentGeometry): + continue + flagMsg = flagMsg[: len(flagMsg) - 2] + " distinct attributes" + self.flagFeature( + nameLyrOriginal, + featureAdd, + flagMsg, + attributeGroup, + fields, + "Update", + point_flag_sink, + line_flag_sink, + poly_flag_sink, + lyrPoint, + lyrLine, + lyrPolygon, + ) + bothLyr = True + multiStep.setProgress(current2 * stepSize) + + if not bothLyr: + self.flagFeature( + nameLyrOriginal, + featureAdd, + None, + attributeGroup, + fields, + "Added", + point_flag_sink, + line_flag_sink, + poly_flag_sink, + lyrPoint, + lyrLine, + lyrPolygon, + ) + + for featureDel in deletedLyr.getFeatures(): + if featureDel[f"{attributeGroup}"] in setDelAddFeat: + continue + self.flagFeature( + nameLyrOriginal, + featureDel, + None, + attributeGroup, + fields, + "Deleted", + point_flag_sink, + line_flag_sink, + poly_flag_sink, + lyrPoint, + lyrLine, + lyrPolygon, + ) + + return { + self.POINT_FLAG: point_flag_id, + self.LINE_FLAG: line_flag_id, + self.POLYGON_FLAG: poly_flag_id, + } + + def sinkLyr(self, parameters, flag, context, fields, wkbType, crs): + (flag_sink, flag_id) = self.parameterAsSink( + parameters, + flag, + context, + fields, + wkbType, + crs, + ) + return flag_sink, flag_id + + def geomWkt(self, feature): + geomFeat = feature.geometry() + wkt = geomFeat.asWkt() + return wkt + + def flagFeature( + self, + nameLyr, + feature, + flagMsg, + attributeGroup, + fields, + type_change, + point_flag_sink, + line_flag_sink, + poly_flag_sink, + lyrPoint, + lyrLine, + lyrPolygon, + ): + newFeat = QgsFeature(fields) + geomFeatAdd = feature.geometry() + newFeat.setGeometry(geomFeatAdd) + newFeat[f"{attributeGroup}"] = feature[f"{attributeGroup}"] + newFeat["name_layer"] = nameLyr + newFeat["type_change"] = type_change + newFeat["update"] = flagMsg + if lyrPoint: + point_flag_sink.addFeature(newFeat, QgsFeatureSink.FastInsert) + elif lyrLine: + line_flag_sink.addFeature(newFeat, QgsFeatureSink.FastInsert) + elif lyrPolygon: + poly_flag_sink.addFeature(newFeat, QgsFeatureSink.FastInsert) + + def fieldsFlag(self, attributeGroup): + fields = QgsFields() + fields.append(QgsField(f"{attributeGroup}", QVariant.String)) + fields.append(QgsField("name_layer", QVariant.String)) + fields.append(QgsField("type_change", QVariant.String)) + fields.append(QgsField("update", QVariant.String)) + return fields + + def compareAttributes(self, strBlackAttributes, lyrOriginal): + listAttributes = list(lyrOriginal.attributeAliases()) + listBlackAttributes = strBlackAttributes.split(", ") + listWhiteAttributes = [ + attribute + for attribute in listAttributes + if attribute not in listBlackAttributes + ] + return listWhiteAttributes + + def typeOfLayer(self, addedLyr): + lyrPoint = addedLyr.geometryType() == QgsWkbTypes.PointGeometry + lyrLine = addedLyr.geometryType() == QgsWkbTypes.LineGeometry + lyrPolygon = addedLyr.geometryType() == QgsWkbTypes.PolygonGeometry + return lyrPoint, lyrLine, lyrPolygon + + def dictNameLyrCrs(self, group): + dictLyrs = dict() + for layer in group.findLayers(): + lyr = layer.layer() + dictLyrs[lyr.name()] = lyr + crs = lyr.sourceCrs() + return dictLyrs, crs + + def name(self): + """ + Returns the algorithm name, used for identifying the algorithm. This + string should be fixed for the algorithm, and must not be localised. + The name should be unique within each provider. Names should contain + lowercase alphanumeric characters only and no spaces or other + formatting characters. + """ + return "detectchangesingroup" + + def displayName(self): + """ + Returns the translated algorithm name, which should be used for any + user-visible display of the algorithm name. + """ + return self.tr("Detect Changes In Group") + + def group(self): + """ + Returns the name of the group this algorithm belongs to. This string + should be localised. + """ + return self.tr("Quality Assurance Tools (Identification Processes)") + + def groupId(self): + """ + Returns the unique ID of the group this algorithm belongs to. This + string should be fixed for the algorithm, and must not be localised. + The group id should be unique within each provider. Group id should + contain lowercase alphanumeric characters only and no spaces or other + formatting characters. + """ + return "DSGTools: Quality Assurance Tools (Identification Processes)" + + def tr(self, string): + return QCoreApplication.translate("DetectChangesInGroup", string) + + def createInstance(self): + return DetectChangesGroupAlgorithm() + + +class GroupsWidgetWrapper(WidgetWrapper): + def __init__(self, *args, **kwargs): + super(GroupsWidgetWrapper, self).__init__(*args, **kwargs) + + def getGroupNames(self): + groupsList = [ + g.name() for g in QgsProject.instance().layerTreeRoot().findGroups() + ] + groupsList.insert(0, "") + return groupsList + + def createWidget(self): + self.widget = QtWidgets.QComboBox() + self.widget.addItems(self.getGroupNames()) + self.widget.dialogType = self.dialogType + return self.widget + + def parentLayerChanged(self, layer=None): + pass + + def setLayer(self, layer): + pass + + def setValue(self, value): + pass + + def value(self): + return self.widget.currentText() + + def postInitialize(self, wrappers): + pass + + +class ParameterGroup(QgsProcessingParameterDefinition): + def __init__(self, name, description=""): + super().__init__(name, description) + + def clone(self): + copy = ParameterGroup(self.name(), self.description()) + return copy + + def type(self): + return self.typeName() + + @staticmethod + def typeName(): + return "group" + + def checkValueIsAcceptable(self, value, context=None): + return True + + def metadata(self): + return { + "widget_wrapper": "DsgTools.core.DSGToolsProcessingAlgs.Algs.ValidationAlgs.detectChangesGroupAlgorithm.GroupsWidgetWrapper" + } + + def valueAsPythonString(self, value, context): + return str(value) + + def asScriptCode(self): + raise NotImplementedError() + + @classmethod + def fromScriptCode(cls, name, description, isOptional, definition): + raise NotImplementedError() diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/fixDrainageFlowAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/fixDrainageFlowAlgorithm.py index 2c76cd57d..7441980f2 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/fixDrainageFlowAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/fixDrainageFlowAlgorithm.py @@ -503,7 +503,7 @@ def displayName(self): Returns the translated algorithm name, which should be used for any user-visible display of the algorithm name. """ - return self.tr("Fix Drainage Flow Algoritm") + return self.tr("Fix Drainage Flow Algorithm") def group(self): """ diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/fixSegmentErrorsBetweenLinesAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/fixSegmentErrorsBetweenLinesAlgorithm.py new file mode 100644 index 000000000..ecb582791 --- /dev/null +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/fixSegmentErrorsBetweenLinesAlgorithm.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + DsgTools + A QGIS plugin + Brazilian Army Cartographic Production Tools + ------------------- + begin : 2023-06-08 + git sha : $Format:%H$ + copyright : (C) 2023 by Philipe Borba - Cartographic Engineer @ Brazilian Army + email : borba.philipe@eb.mil.br + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +from collections import defaultdict +from typing import Dict +from DsgTools.core.DSGToolsProcessingAlgs.algRunner import AlgRunner +from DsgTools.core.GeometricTools.layerHandler import LayerHandler +from PyQt5.QtCore import QCoreApplication +from qgis.core import ( + QgsProcessing, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterFeatureSink, + QgsWkbTypes, + QgsGeometry, + QgsProcessingParameterNumber, + QgsProcessingMultiStepFeedback, + QgsFeedback, + QgsVectorLayer, + QgsProcessingContext, +) + +from .validationAlgorithm import ValidationAlgorithm + + +class FixSegmentErrorsBetweenLinesAlgorithm(ValidationAlgorithm): + INPUT = "INPUT" + REFERENCE_LINE = "REFERENCE_LINE" + SEARCH_RADIUS = "SEARCH_RADIUS" + FLAGS = "FLAGS" + + def initAlgorithm(self, config): + """ + Parameter setting. + """ + self.addParameter( + QgsProcessingParameterFeatureSource( + self.INPUT, self.tr("Input lines"), [QgsProcessing.TypeVectorLine] + ) + ) + self.addParameter( + QgsProcessingParameterFeatureSource( + self.REFERENCE_LINE, + self.tr("Reference lines"), + [QgsProcessing.TypeVectorLine], + ) + ) + self.addParameter( + QgsProcessingParameterNumber( + self.SEARCH_RADIUS, + self.tr("Search Radius"), + type=QgsProcessingParameterNumber.Double, + ) + ) + + def processAlgorithm(self, parameters, context, feedback): + """ + Here is where the processing itself takes place. + """ + layerHandler = LayerHandler() + self.algRunner = AlgRunner() + inputSource = self.parameterAsSource(parameters, self.INPUT, context) + referenceSource = self.parameterAsSource( + parameters, self.REFERENCE_LINE, context + ) + searchRadius = self.parameterAsDouble(parameters, self.SEARCH_RADIUS, context) + currentStep = 0 + multiStepFeedback = QgsProcessingMultiStepFeedback(5, feedback) + multiStepFeedback.setCurrentStep(currentStep) + flagLyr = self.algRunner.runIdentifySegmentErrorBetweenLines( + inputLayer=parameters[self.INPUT], + referenceLineLayer=parameters[self.REFERENCE_LINE], + searchRadius=searchRadius, + context=context, + feedback=multiStepFeedback + ) + if flagLyr.featureCount() == 0: + return + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + singlePartFlags = self.algRunner.runMultipartToSingleParts( + flagLyr, context, is_child_algorithm=True + ) + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + self.algRunner.runCreateSpatialIndex(singlePartFlags, context, feedback=multiStepFeedback, is_child_algorithm=True) + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + self.algRunner.runSnapLayerOnLayer( + inputLayer=inputSource, + referenceLayer=singlePartFlags, + tol=searchRadius, + ) + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + self.algRunner.runSnapLayerOnLayer( + inputLayer=referenceSource, + referenceLayer=singlePartFlags, + tol=searchRadius, + ) + + + + def name(self): + """ + Returns the algorithm name, used for identifying the algorithm. This + string should be fixed for the algorithm, and must not be localised. + The name should be unique within each provider. Names should contain + lowercase alphanumeric characters only and no spaces or other + formatting characters. + """ + return "fixsegmenterrorsbetweenlines" + + def displayName(self): + """ + Returns the translated algorithm name, which should be used for any + user-visible display of the algorithm name. + """ + return self.tr("Fix Segment Errors Between Lines") + + def group(self): + """ + Returns the name of the group this algorithm belongs to. This string + should be localised. + """ + return self.tr("Quality Assurance Tools (Manipulation Processes)") + + def groupId(self): + """ + Returns the unique ID of the group this algorithm belongs to. This + string should be fixed for the algorithm, and must not be localised. + The group id should be unique within each provider. Group id should + contain lowercase alphanumeric characters only and no spaces or other + formatting characters. + """ + return "DSGTools: Quality Assurance Tools (Manipulation Processes)" + + def tr(self, string): + return QCoreApplication.translate( + "FixSegmentErrorsBetweenLinesAlgorithm", string + ) + + def createInstance(self): + return FixSegmentErrorsBetweenLinesAlgorithm() diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyCrossingLinesAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyCrossingLinesAlgorithm.py index 48d082f08..3955d7b93 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyCrossingLinesAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyCrossingLinesAlgorithm.py @@ -43,7 +43,8 @@ class IdentifyCrossingLinesAlgorithm(ValidationAlgorithm): INPUT = "INPUT" - COMPARE_INPUT = "COMPARE_INPUT" + COMPARE_INPUT_LINES = "COMPARE_INPUT_LINES" + COMPARE_INPUT_POLYGONS = "COMPARE_INPUT_POLYGONS" FLAGS = "FLAGS" def initAlgorithm(self, config): @@ -57,9 +58,18 @@ def initAlgorithm(self, config): ) self.addParameter( QgsProcessingParameterMultipleLayers( - self.COMPARE_INPUT, - self.tr("Compare lines"), - QgsProcessing.TypeVectorAnyGeometry, + self.COMPARE_INPUT_LINES, + self.tr("Compare layers (lines)"), + QgsProcessing.TypeVectorLine, + optional=True, + ) + ) + self.addParameter( + QgsProcessingParameterMultipleLayers( + self.COMPARE_INPUT_POLYGONS, + self.tr("Compare layers (polygons)"), + QgsProcessing.TypeVectorPolygon, + optional=True, ) ) self.addParameter( @@ -74,10 +84,18 @@ def processAlgorithm(self, parameters, context, feedback): """ self.algRunner = AlgRunner() input = self.parameterAsVectorLayer(parameters, self.INPUT, context) - compareList = self.parameterAsLayerList(parameters, self.COMPARE_INPUT, context) + compareList = [] + compareListLines = self.parameterAsLayerList( + parameters, self.COMPARE_INPUT_LINES, context + ) + compareListPoygons = self.parameterAsLayerList( + parameters, self.COMPARE_INPUT_POLYGONS, context + ) + compareList.extend(compareListLines) + compareList.extend(compareListPoygons) self.prepareFlagSink(parameters, input, QgsWkbTypes.MultiPoint, context) nFeats = input.featureCount() - if input is None or nFeats == 0 or compareList == 0: + if input is None or nFeats == 0 or not compareList: return {self.FLAGS: self.flag_id} multiStepFeedback = QgsProcessingMultiStepFeedback(3, feedback) currentStep = 0 @@ -252,5 +270,12 @@ def groupId(self): def tr(self, string): return QCoreApplication.translate("IdentifyCrossingLinesAlgorithm", string) + def shortHelpString(self): + return self.tr( + """ + Return intersections between 'Input Lines' and the layers selected in 'Compare layers' + """ + ) + def createInstance(self): return IdentifyCrossingLinesAlgorithm() diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyDanglesAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyDanglesAlgorithm.py index bcbfc43cd..e895653b1 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyDanglesAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyDanglesAlgorithm.py @@ -390,7 +390,6 @@ def evaluate(point) -> Union[QgsPointXY, None]: bufferCount, intersectCount = 0, 0 point_relationship_lambda = ( lambda x: geomEngine.intersects(x.constGet()) - or qgisPoint.distance(x) < 1e-8 if ignoreDanglesOnUnsegmentedLines else geomEngine.touches(x.constGet()) ) diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifySegmentErrorsBetweenLinesAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifySegmentErrorsBetweenLinesAlgorithm.py index 6b48975c6..2161310df 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifySegmentErrorsBetweenLinesAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifySegmentErrorsBetweenLinesAlgorithm.py @@ -35,7 +35,7 @@ QgsProcessingParameterNumber, QgsProcessingMultiStepFeedback, QgsFeedback, - QgsProcessingFeatureSource, + QgsVectorLayer, QgsProcessingContext, ) @@ -93,11 +93,59 @@ def processAlgorithm(self, parameters, context, feedback): nReferenceFeats = referenceSource.featureCount() if inputSource is None or nFeats == 0 or nReferenceFeats == 0: return {self.FLAGS: self.flag_id} - multiStepFeedback = QgsProcessingMultiStepFeedback(4, feedback) + multiStepFeedback = QgsProcessingMultiStepFeedback(10, feedback) currentStep = 0 + multiStepFeedback.setCurrentStep(currentStep) + cacheLyr = self.algRunner.runCreateFieldWithExpression( + inputLyr=parameters[self.INPUT], + expression="$id", + fieldType=1, + fieldName="featid", + feedback=multiStepFeedback, + context=context, + ) + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + bufferedCache = self.algRunner.runBuffer( + inputLayer=cacheLyr, + distance=searchRadius, + context=context, + feedback=multiStepFeedback, + ) + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + self.algRunner.runCreateSpatialIndex(bufferedCache, context, feedback=multiStepFeedback, is_child_algorithm=True) + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + disjointBuffer = self.algRunner.runExtractByLocation( + inputLyr=bufferedCache, + intersectLyr=parameters[self.REFERENCE_LINE], + context=context, + predicate=[AlgRunner.Disjoint], + feedback=multiStepFeedback + ) + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + idsToIgnore = set(feat["featid"] for feat in disjointBuffer.getFeatures()) + validInput = self.algRunner.runFilterExpression( + inputLyr=cacheLyr, + context=context, + expression=f"featid not in {tuple(idsToIgnore)}".replace(",)", ")"), + feedback=multiStepFeedback, + ) if len(idsToIgnore) > 0 else cacheLyr + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + self.algRunner.runCreateSpatialIndex(validInput, context, feedback=multiStepFeedback, is_child_algorithm=True) + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) vertexNearEdgeFlagDict = layerHandler.getUnsharedVertexOnSharedEdgesDict( - [parameters[self.INPUT], parameters[self.REFERENCE_LINE]], + [validInput, parameters[self.REFERENCE_LINE]], [], searchRadius, feedback=multiStepFeedback, @@ -112,8 +160,8 @@ def processAlgorithm(self, parameters, context, feedback): multiStepFeedback.setCurrentStep(currentStep) intersectedFeats = self.algRunner.runDifference( - inputLyr=parameters[self.INPUT], - overlayLyr=parameters[self.REFERENCE_LINE], + inputLyr=parameters[self.REFERENCE_LINE], + overlayLyr=validInput, context=context, feedback=multiStepFeedback, ) @@ -132,7 +180,7 @@ def processAlgorithm(self, parameters, context, feedback): multiStepFeedback.setCurrentStep(currentStep) self.raiseFlagsFromVertexFlagSet( - parameters[self.INPUT], vertexFlagSet, context, feedback=multiStepFeedback + validInput, vertexFlagSet, context, feedback=multiStepFeedback ) return {self.FLAGS: self.flag_id} @@ -150,42 +198,36 @@ def getFlagVertexesFromGeomDict( def raiseFlagsFromVertexFlagSet( self, - inputSource: QgsProcessingFeatureSource, + localCache: QgsVectorLayer, vertexFlagSet: set, context: QgsProcessingContext, feedback: QgsFeedback, ): - multiStepFeedback = QgsProcessingMultiStepFeedback(5, feedback) + multiStepFeedback = QgsProcessingMultiStepFeedback(4, feedback) multiStepFeedback.setCurrentStep(0) - localCache = self.algRunner.runCreateFieldWithExpression( - inputLyr=inputSource, - expression="$id", - fieldName="featid", - fieldType=1, - context=context, - feedback=multiStepFeedback, - is_child_algorithm=False, - ) - multiStepFeedback.setCurrentStep(1) vertexLyr = self.algRunner.runExtractVertices( inputLyr=localCache, context=context, feedback=multiStepFeedback ) - multiStepFeedback.setCurrentStep(2) + multiStepFeedback.setCurrentStep(1) vertexDict = defaultdict(set) + vertexCount = defaultdict(int) stepSize = 100 / vertexLyr.featureCount() for current, vertexFeat in enumerate(vertexLyr.getFeatures()): if multiStepFeedback.isCanceled(): return geom = vertexFeat.geometry() + vertexCount[vertexFeat["featid"]] += 1 if geom.asWkt() not in vertexFlagSet: continue vertexDict[vertexFeat["featid"]].add(geom) multiStepFeedback.setProgress(current * stepSize) - multiStepFeedback.setCurrentStep(3) + multiStepFeedback.setCurrentStep(2) stepSize = 100 / len(vertexDict) for current, (featid, vertexSet) in enumerate(vertexDict.items()): if multiStepFeedback.isCanceled(): return + if len(vertexSet) == vertexCount[featid] - 1: + continue flagText = f"Line with id={featid} from input has construction errors with reference layer." baseGeom, *geomList = list(vertexSet) if len(geomList) > 0: diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyUnmergedLinesWithSameAttributeSetAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyUnmergedLinesWithSameAttributeSetAlgorithm.py index c70c28bf1..29192bcab 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyUnmergedLinesWithSameAttributeSetAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyUnmergedLinesWithSameAttributeSetAlgorithm.py @@ -26,6 +26,7 @@ import os from DsgTools.core.DSGToolsProcessingAlgs.algRunner import AlgRunner +from DsgTools.core.GeometricTools import graphHandler from DsgTools.core.GeometricTools.layerHandler import LayerHandler from PyQt5.QtCore import QCoreApplication from qgis.core import ( @@ -40,6 +41,7 @@ QgsProcessingParameterMultipleLayers, QgsProcessingParameterVectorLayer, QgsWkbTypes, + QgsProcessingException, ) from .validationAlgorithm import ValidationAlgorithm @@ -125,8 +127,16 @@ def processAlgorithm(self, parameters, context, feedback): """ Here is where the processing itself takes place. """ + try: + import networkx as nx + except ImportError: + raise QgsProcessingException( + self.tr( + "This algorithm requires the Python networkx library. Please install this library and try again." + ) + ) self.layerHandler = LayerHandler() - algRunner = AlgRunner() + self.algRunner = AlgRunner() inputLyr = self.parameterAsVectorLayer(parameters, self.INPUT, context) onlySelected = self.parameterAsBoolean(parameters, self.SELECTED, context) pointFilterLyrList = self.parameterAsLayerList( @@ -141,7 +151,7 @@ def processAlgorithm(self, parameters, context, feedback): attributeBlackList = self.parameterAsFields( parameters, self.ATTRIBUTE_BLACK_LIST, context ) - fieldList = self.layerHandler.getAttributesFromBlackList( + attributeNameList = self.layerHandler.getAttributesFromBlackList( inputLyr, attributeBlackList, ignoreVirtualFields=self.parameterAsBoolean( @@ -151,169 +161,149 @@ def processAlgorithm(self, parameters, context, feedback): parameters, self.IGNORE_PK_FIELDS, context ), ) - fieldIdList = [ - i for i, field in enumerate(inputLyr.fields()) if field.name() in fieldList - ] - multiStepFeedback = QgsProcessingMultiStepFeedback(6, feedback) - multiStepFeedback.setCurrentStep(0) - multiStepFeedback.setProgressText(self.tr("Building local cache...")) - localLyr = algRunner.runAddAutoIncrementalField( + multiStepFeedback = QgsProcessingMultiStepFeedback(11, feedback) + currentStep = 0 + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText( + self.tr("Building local cache on input layer...") + ) + localCache = self.algRunner.runCreateFieldWithExpression( inputLyr=inputLyr if not onlySelected else QgsProcessingFeatureSourceDefinition(inputLyr.id(), True), - fieldName="AUTO", + expression="$id", + fieldName="featid", + fieldType=1, context=context, feedback=multiStepFeedback, ) - multiStepFeedback.setCurrentStep(1) - multiStepFeedback.setProgressText( - self.tr("Building initial and end point dict...") + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + nodesLayer = self.algRunner.runExtractSpecificVertices( + inputLyr=localCache, + vertices="0,-1", + context=context, + feedback=multiStepFeedback, ) - initialAndEndPointDict = self.buildInitialAndEndPointDict( - localLyr, algRunner=algRunner, context=context, feedback=multiStepFeedback + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + nodesLayer = self.algRunner.runCreateFieldWithExpression( + inputLyr=nodesLayer, + expression="$id", + fieldName="nfeatid", + fieldType=1, + context=context, + feedback=multiStepFeedback, ) - multiStepFeedback.setProgressText(self.tr("Building aux structure...")) - multiStepFeedback.setCurrentStep(2) - mergedPointLyr = ( - algRunner.runMergeVectorLayers( + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr("Building graph aux structures")) + ( + nodeDict, + nodeIdDict, + edgeDict, + hashDict, + networkBidirectionalGraph, + ) = graphHandler.buildAuxStructures( + nx, + nodesLayer=nodesLayer, + edgesLayer=localCache, + feedback=multiStepFeedback, + useWkt=False, + computeNodeLayerIdDict=False, + addEdgeLength=False, + ) + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr("Finding mergeable edges")) + outputGraphDict = graphHandler.find_mergeable_edges_on_graph( + nx=nx, G=networkBidirectionalGraph, feedback=multiStepFeedback + ) + nSteps = len(outputGraphDict) + if nSteps == 0: + return {"FLAGS": self.flag_id} + + multiStepFeedback.setProgressText( + self.tr("Building aux structure on input point list...") + ) + multiStepFeedback.setCurrentStep(currentStep) + mergedPointConstraintLyr = ( + self.algRunner.runMergeVectorLayers( pointFilterLyrList, context, multiStepFeedback ) if pointFilterLyrList else None ) - multiStepFeedback.setCurrentStep(3) + currentStep += 1 + + multiStepFeedback.setProgressText( + self.tr("Building aux structure on input line list...") + ) + multiStepFeedback.setCurrentStep(currentStep) mergedLineLyr = ( - algRunner.runMergeVectorLayers( + self.algRunner.runMergeVectorLayers( lineFilterLyrList, context, multiStepFeedback ) if lineFilterLyrList else None ) - multiStepFeedback.setCurrentStep(4) + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) if mergedLineLyr is not None: - algRunner.runCreateSpatialIndex(mergedLineLyr, context, multiStepFeedback) - dictSize = len(initialAndEndPointDict) - if dictSize == 0: - return {"FLAGS": self.flag_id} - filterPointSet = ( - set(i.geometry().asWkb() for i in mergedPointLyr.getFeatures()) - if mergedPointLyr is not None + self.algRunner.runCreateSpatialIndex( + mergedLineLyr, context, multiStepFeedback + ) + filterPointIdSet = ( + set(nodeDict[i.geometry().asWkb()] for i in mergedPointConstraintLyr.getFeatures() if i.geometry().asWkb() in nodeDict) + if mergedPointConstraintLyr is not None else set() ) - multiStepFeedback.setCurrentStep(5) - multiStepFeedback.setProgressText(self.tr("Evaluating candidates")) - self.evaluateFlagCandidates( - fieldList, - fieldIdList, - multiStepFeedback, - localLyr, - initialAndEndPointDict, - mergedLineLyr, - dictSize, - filterPointSet, - ) - return {"FLAGS": self.flag_id} - - def evaluateFlagCandidates( - self, - fieldList, - fieldIdList, - multiStepFeedback, - localLyr, - initialAndEndPointDict, - mergedLineLyr, - dictSize, - filterPointSet, - ): - stepSize = 100 / dictSize - multiStepFeedback = QgsProcessingMultiStepFeedback(2, multiStepFeedback) - multiStepFeedback.setCurrentStep(0) - - def evaluate(pointXY, idSet): - if multiStepFeedback.isCanceled(): - return None - geom = QgsGeometry.fromPointXY(pointXY) - geomWkb = geom.asWkb() - if geomWkb in filterPointSet: - return None - if len(idSet) != 2: - return None - if mergedLineLyr is not None: - bbox = geom.boundingBox() - nIntersects = len( - [ - i - for i in mergedLineLyr.getFeatures(bbox) - if i.geometry().intersects(geom) - ] - ) - if nIntersects > 0: - return None - request = ( - QgsFeatureRequest() - .setFilterExpression(f"AUTO in {tuple(idSet)}") - .setFlags(QgsFeatureRequest.NoGeometry) - .setSubsetOfAttributes(fieldIdList) - ) - f1, f2 = [i for i in localLyr.getFeatures(request)] - differentFeats = any(f1[k] != f2[k] for k in fieldList) - return geomWkb if not differentFeats else None - pool = concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count() - 1) + computeLambda = lambda x: graphHandler.identify_unmerged_edges_on_graph( + nx=nx, + G=x, + featDict=edgeDict, + nodeIdDict=nodeIdDict, + filterPointSet=filterPointIdSet, + filterLineLayer=mergedLineLyr, + attributeNameList=attributeNameList, + ) futures = set() - - for current, (pointXY, idSet) in enumerate(initialAndEndPointDict.items()): + currentStep += 1 + stepSize = 100 / nSteps + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText( + self.tr("Submitting identification tasks to thread") + ) + for current, G in enumerate(outputGraphDict.values()): if multiStepFeedback.isCanceled(): break - futures.add(pool.submit(evaluate, pointXY, idSet)) + futures.add(pool.submit(computeLambda, G)) multiStepFeedback.setProgress(current * stepSize) + currentStep += 1 - multiStepFeedback.setCurrentStep(1) + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr("Evaluating results")) + flagIdSet = set() for current, future in enumerate(concurrent.futures.as_completed(futures)): if multiStepFeedback.isCanceled(): break - geomWkb = future.result() - if geomWkb is not None: - self.flagFeature( - flagGeom=geomWkb, - flagText=self.tr("Not merged lines with same attribute set"), - fromWkb=True, - ) + flagIdSet |= future.result() multiStepFeedback.setProgress(current * stepSize) - - def buildInitialAndEndPointDict(self, lyr, algRunner, context, feedback): - pointDict = defaultdict(set) - nSteps = 3 - currentStep = 0 - multiStepFeedback = QgsProcessingMultiStepFeedback(nSteps, feedback) - multiStepFeedback.setCurrentStep(currentStep) - boundaryLyr = algRunner.runBoundary( - inputLayer=lyr, context=context, feedback=multiStepFeedback - ) - currentStep += 1 - multiStepFeedback.setCurrentStep(currentStep) - boundaryLyr = algRunner.runMultipartToSingleParts( - inputLayer=boundaryLyr, context=context, feedback=multiStepFeedback - ) currentStep += 1 + if len(flagIdSet) == 0: + return {"FLAGS": self.flag_id} + flagLambda = lambda x: self.flagFeature( + flagGeom=nodeIdDict[x], + flagText=self.tr("Lines with same attribute set that are not merged."), + fromWkb=True, + ) multiStepFeedback.setCurrentStep(currentStep) - featCount = boundaryLyr.featureCount() - if featCount == 0: - return pointDict - step = 100 / featCount - for current, feat in enumerate(boundaryLyr.getFeatures()): - if multiStepFeedback.isCanceled(): - break - geom = feat.geometry() - if geom is None or not geom.isGeosValid(): - continue - id = feat["AUTO"] - pointList = geom.asMultiPoint() if geom.isMultipart() else [geom.asPoint()] - for point in pointList: - pointDict[point].add(id) - multiStepFeedback.setProgress(current * step) - return pointDict + multiStepFeedback.setProgressText(self.tr("Raising flags")) + list(map(flagLambda, flagIdSet)) + return {"FLAGS": self.flag_id} def name(self): """ @@ -358,7 +348,7 @@ def shortHelpString(self): return help().shortHelpString(self.name()) def helpUrl(self): - return help().helpUrl(self.name()) + return help().helpUrl(self.name()) def createInstance(self): return IdentifyUnmergedLinesWithSameAttributeSetAlgorithm() diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/mergeLinesAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/mergeLinesAlgorithm.py index 7ddd37b46..925b0dc7e 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/mergeLinesAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/mergeLinesAlgorithm.py @@ -20,28 +20,23 @@ * * ***************************************************************************/ """ +import os +import concurrent.futures from PyQt5.QtCore import QCoreApplication +from DsgTools.core.DSGToolsProcessingAlgs.algRunner import AlgRunner +from DsgTools.core.GeometricTools import graphHandler -import processing from DsgTools.core.GeometricTools.layerHandler import LayerHandler from qgis.core import ( - QgsDataSourceUri, - QgsFeature, - QgsFeatureSink, - QgsGeometry, + QgsProcessingFeatureSourceDefinition, QgsProcessing, - QgsProcessingAlgorithm, QgsProcessingOutputVectorLayer, QgsProcessingParameterBoolean, - QgsProcessingParameterEnum, - QgsProcessingParameterFeatureSink, - QgsProcessingParameterFeatureSource, QgsProcessingParameterField, - QgsProcessingParameterMultipleLayers, - QgsProcessingParameterNumber, QgsProcessingParameterVectorLayer, - QgsProcessingUtils, - QgsSpatialIndex, + QgsWkbTypes, + QgsProcessingException, + QgsProcessingMultiStepFeedback, QgsWkbTypes, ) @@ -105,7 +100,16 @@ def processAlgorithm(self, parameters, context, feedback): """ Here is where the processing itself takes place. """ + try: + import networkx as nx + except ImportError: + raise QgsProcessingException( + self.tr( + "This algorithm requires the Python networkx library. Please install this library and try again." + ) + ) layerHandler = LayerHandler() + self.algRunner = AlgRunner() inputLyr = self.parameterAsVectorLayer(parameters, self.INPUT, context) onlySelected = self.parameterAsBool(parameters, self.SELECTED, context) attributeBlackList = self.parameterAsFields( @@ -115,15 +119,117 @@ def processAlgorithm(self, parameters, context, feedback): parameters, self.IGNORE_VIRTUAL_FIELDS, context ) ignorePK = self.parameterAsBool(parameters, self.IGNORE_PK_FIELDS, context) + nSteps = 8 + multiStepFeedback = QgsProcessingMultiStepFeedback(nSteps, feedback) + currentStep = 0 + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr("Building aux structures")) + localCache = self.algRunner.runCreateFieldWithExpression( + inputLyr=inputLyr + if not onlySelected + else QgsProcessingFeatureSourceDefinition(inputLyr.id(), True), + expression="$id", + fieldName="featid", + fieldType=1, + context=context, + feedback=multiStepFeedback, + ) + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + nodesLayer = self.algRunner.runExtractSpecificVertices( + inputLyr=localCache, + vertices="0,-1", + context=context, + feedback=multiStepFeedback, + ) + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + nodesLayer = self.algRunner.runCreateFieldWithExpression( + inputLyr=nodesLayer, + expression="$id", + fieldName="nfeatid", + fieldType=1, + context=context, + feedback=multiStepFeedback, + ) + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr("Building graph aux structures")) + ( + nodeDict, + nodeIdDict, + edgeDict, + hashDict, + networkBidirectionalGraph, + ) = graphHandler.buildAuxStructures( + nx, + nodesLayer=nodesLayer, + edgesLayer=localCache, + feedback=multiStepFeedback, + useWkt=False, + computeNodeLayerIdDict=False, + addEdgeLength=False, + ) + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr("Finding mergeable edges")) + outputGraphDict = graphHandler.find_mergeable_edges_on_graph( + nx=nx, G=networkBidirectionalGraph, feedback=multiStepFeedback + ) + nSteps = len(outputGraphDict) + if nSteps == 0: + return {self.OUTPUT: inputLyr} + stepSize = 100 / nSteps + + pool = concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count() - 1) + attributeNameList = [ + f.name() + for f in layerHandler.getFieldsFromAttributeBlackList( + originalLayer=inputLyr, + attributeBlackList=attributeBlackList, + ignoreVirtualFields=ignoreVirtual, + ) + ] + computeLambda = lambda x: graphHandler.filter_mergeable_graphs_using_attibutes( + nx=nx, + G=x, + featDict=edgeDict, + attributeNameList=attributeNameList, + isMulti=QgsWkbTypes.isMultiType(inputLyr.wkbType()), + ) + futures = set() + currentStep += 1 + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr("Submitting merge task to thread")) + for current, G in enumerate(outputGraphDict.values()): + if multiStepFeedback.isCanceled(): + break + futures.add(pool.submit(computeLambda, G)) + multiStepFeedback.setProgress(current * stepSize) + currentStep += 1 + + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr("Evaluating results")) + outputFeatSet, idsToDeleteSet = set(), set() + for current, future in enumerate(concurrent.futures.as_completed(futures)): + if multiStepFeedback.isCanceled(): + break + featSet, idsToDelete = future.result() + outputFeatSet |= featSet + idsToDeleteSet |= idsToDelete + multiStepFeedback.setProgress(current * stepSize) + currentStep += 1 - layerHandler.mergeLinesOnLayer( - inputLyr, - feedback=feedback, - onlySelected=onlySelected, - ignoreVirtualFields=ignoreVirtual, - attributeBlackList=attributeBlackList, - excludePrimaryKeys=ignorePK, + multiStepFeedback.setCurrentStep(currentStep) + multiStepFeedback.setProgressText(self.tr("Saving changes on input layer")) + updateLambda = lambda x: inputLyr.changeGeometry(x["featid"], x.geometry()) + inputLyr.startEditing() + inputLyr.beginEditCommand( + f"Merging lines with same attribute set from {inputLyr.name()}" ) + list(map(updateLambda, outputFeatSet)) + inputLyr.deleteFeatures(list(idsToDeleteSet)) + inputLyr.endEditCommand() return {self.OUTPUT: inputLyr} diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/unicodeFilterAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/unicodeFilterAlgorithm.py index fe1500d6e..1a3c6da02 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/unicodeFilterAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/unicodeFilterAlgorithm.py @@ -131,20 +131,20 @@ def getOutputDict(self, parameters, context, fields): point_sink, point_sink_id = self.createOutput( parameters, context, self.OUTPUT1, QgsWkbTypes.MultiPoint, fields ) - returnDict[self.OUTPUT1] = point_sink - sinkDict[QgsWkbTypes.PointGeometry] = point_sink_id + returnDict[self.OUTPUT1] = point_sink_id + sinkDict[QgsWkbTypes.PointGeometry] = point_sink line_sink, line_sink_id = self.createOutput( parameters, context, self.OUTPUT2, QgsWkbTypes.MultiLineString, fields ) - returnDict[self.OUTPUT2] = line_sink - sinkDict[QgsWkbTypes.LineGeometry] = line_sink_id + returnDict[self.OUTPUT2] = line_sink_id + sinkDict[QgsWkbTypes.LineGeometry] = line_sink polygon_sink, polygon_sink_id = self.createOutput( parameters, context, self.OUTPUT3, QgsWkbTypes.MultiPolygon, fields ) - returnDict[self.OUTPUT3] = polygon_sink - sinkDict[QgsWkbTypes.MultiPolygon] = polygon_sink_id + returnDict[self.OUTPUT3] = polygon_sink_id + sinkDict[QgsWkbTypes.MultiPolygon] = polygon_sink return returnDict, sinkDict diff --git a/DsgTools/core/DSGToolsProcessingAlgs/algRunner.py b/DsgTools/core/DSGToolsProcessingAlgs/algRunner.py index 1040b8301..ce2d3dce7 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/algRunner.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/algRunner.py @@ -1770,3 +1770,59 @@ def runAddUnsharedVertexOnSharedEdges( feedback=feedback, is_child_algorithm=is_child_algorithm, ) + + def runIdentifySegmentErrorBetweenLines( + self, + inputLayer, + referenceLineLayer, + searchRadius, + context, + flagLyr=None, + feedback=None, + is_child_algorithm=False, + ): + flagLyr = "memory:" if flagLyr is None else flagLyr + output = processing.run( + "dsgtools:identifysegmenterrorsbetweenlines", + { + "INPUT": inputLayer, + "REFERENCE_LINE": referenceLineLayer, + "SEARCH_RADIUS": searchRadius, + "FLAGS": flagLyr, + }, + context=context, + feedback=feedback, + is_child_algorithm=is_child_algorithm, + ) + return output["FLAGS"] + + def runDetectDatasetChanges( + self, + inputLayer, + reviewedLayer, + attributesList, + matchComparation, + context, + unchangedLayer=None, + addedLayer=None, + deletedLayer=None, + feedback=None, + ): + unchangedLayer = "memory:" if unchangedLayer is None else unchangedLayer + addedLayer = "memory:" if addedLayer is None else addedLayer + deletedLayer = "memory:" if deletedLayer is None else deletedLayer + output = processing.run( + "native:detectvectorchanges", + { + "ORIGINAL": inputLayer, + "REVISED": reviewedLayer, + "COMPARE_ATTRIBUTES": attributesList, + "MATCH_TYPE": matchComparation, + "UNCHANGED": unchangedLayer, + "ADDED": addedLayer, + "DELETED": deletedLayer, + }, + context=context, + feedback=feedback, + ) + return output["UNCHANGED"], output["ADDED"], output["DELETED"] diff --git a/DsgTools/core/DSGToolsProcessingAlgs/dsgtoolsProcessingAlgorithmProvider.py b/DsgTools/core/DSGToolsProcessingAlgs/dsgtoolsProcessingAlgorithmProvider.py index 831d8c6c6..5b10ead86 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/dsgtoolsProcessingAlgorithmProvider.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/dsgtoolsProcessingAlgorithmProvider.py @@ -33,6 +33,9 @@ from DsgTools.core.DSGToolsProcessingAlgs.Algs.DataManagementAlgs.appendFeaturesToLayerAlgorithm import ( AppendFeaturesToLayerAlgorithm, ) +from DsgTools.core.DSGToolsProcessingAlgs.Algs.ValidationAlgs.fixSegmentErrorsBetweenLinesAlgorithm import ( + FixSegmentErrorsBetweenLinesAlgorithm, +) from DsgTools.core.DSGToolsProcessingAlgs.Algs.ValidationAlgs.identifySmallObjectsOnLayersAlgorithm import ( IdentifySmallObjectsOnLayersAlgorithm, ) @@ -161,6 +164,9 @@ from DsgTools.core.DSGToolsProcessingAlgs.Algs.OtherAlgs.selectFeaturesOnCurrentCanvas import ( SelectFeaturesOnCurrentCanvas, ) +from DsgTools.core.DSGToolsProcessingAlgs.Algs.OtherAlgs.azimuthCalculationAlgorithm import ( + AzimuthCalculationAlgorithm, +) from DsgTools.core.DSGToolsProcessingAlgs.Algs.ValidationAlgs.addUnsharedVertexOnIntersectionsAlgorithm import ( AddUnsharedVertexOnIntersectionsAlgorithm, ) @@ -216,6 +222,9 @@ from DsgTools.core.DSGToolsProcessingAlgs.Algs.OtherAlgs.validateTrackerAlgorithm import ( ValidateTrackerAlgorithm, ) +from DsgTools.core.DSGToolsProcessingAlgs.Algs.OtherAlgs.identifyDifferencesBetweenDatabaseModelsAlgorithm import ( + IdentifyDifferencesBetweenDatabaseModelsAlgorithm, +) from DsgTools.core.DSGToolsProcessingAlgs.Algs.ValidationAlgs.addUnsharedVertexOnIntersectionsAlgorithm import ( AddUnsharedVertexOnIntersectionsAlgorithm, ) @@ -234,6 +243,9 @@ from DsgTools.core.DSGToolsProcessingAlgs.Algs.ValidationAlgs.deaggregateGeometriesAlgorithm import ( DeaggregatorAlgorithm, ) +from DsgTools.core.DSGToolsProcessingAlgs.Algs.ValidationAlgs.detectChangesGroupAlgorithm import ( + DetectChangesGroupAlgorithm, +) from DsgTools.core.DSGToolsProcessingAlgs.Algs.ValidationAlgs.detectNullGeometriesAlgorithm import ( DetectNullGeometriesAlgorithm, ) @@ -600,6 +612,10 @@ def getAlgList(self): GenericSelectionToolParametersAlgorithm(), ClipAndCopyFeaturesBetweenDatabasesAlgorithm(), VerifyAdjacentGeographicBoundaryDataAlgorithm(), + FixSegmentErrorsBetweenLinesAlgorithm(), + IdentifyDifferencesBetweenDatabaseModelsAlgorithm(), + AzimuthCalculationAlgorithm(), + DetectChangesGroupAlgorithm(), ] return algList diff --git a/DsgTools/core/Factories/DbFactory/abstractDb.py b/DsgTools/core/Factories/DbFactory/abstractDb.py index 745a687c9..a84ed83a0 100644 --- a/DsgTools/core/Factories/DbFactory/abstractDb.py +++ b/DsgTools/core/Factories/DbFactory/abstractDb.py @@ -928,7 +928,7 @@ def getStyleDict(self, dbVersion): except: pass try: - dbStyles = self.getStylesFromDb(dbVersion) + dbStyles = self.listStylesFromDb(dbVersion) if dbStyles: for style in dbStyles: name = style.split("/")[-1] diff --git a/DsgTools/core/Factories/DbFactory/postgisDb.py b/DsgTools/core/Factories/DbFactory/postgisDb.py index 26ff7aa83..56508a481 100644 --- a/DsgTools/core/Factories/DbFactory/postgisDb.py +++ b/DsgTools/core/Factories/DbFactory/postgisDb.py @@ -190,9 +190,20 @@ def getDatabaseVersion(self): version = query.value(0) return version + def getDatabaseAndImplementationVersions(self): + self.checkAndOpenDb() + sql = self.gen.getEDGVVersionAndImplementationVersion() + query = QSqlQuery(sql, self.db) + if not query.isActive(): + return "Non_EDGV", "-1" + while query.next(): + edgvVersion = query.value(0) + implementationVersion = query.value(1) + return edgvVersion, implementationVersion + def listGeomClassesFromDatabase( self, - primitiveFilter=[], + primitiveFilter=None, withElements=False, excludeViews=True, getGeometryColumn=False, @@ -202,6 +213,7 @@ def listGeomClassesFromDatabase( returns dict if getGeometryColumn = True return list if getGeometryColumn = False """ + primitiveFilter = [] if primitiveFilter is None else primitiveFilter self.checkAndOpenDb() classList = [] schemaList = [ @@ -2326,9 +2338,10 @@ def checkAndCreateStyleTable(self, useTransaction=True): self.db.commit() return created - def getStylesFromDb(self, dbVersion): + def listStylesFromDb(self, dbVersion=None): self.checkAndOpenDb() - sql = self.gen.getStylesFromDb(dbVersion) + dbVersion = dbVersion if dbVersion is not None else self.getDatabaseVersion() + sql = self.gen.listStylesFromDb(dbVersion) if not sql: return [] query = QSqlQuery(sql, self.db) @@ -2353,9 +2366,8 @@ def getStyle(self, styleName, table_name, parsing=True): query.next() qml = query.value(0) # TODO: post parse qml to remove possible attribute value type - if parsing: - if qml: - qml = self.utils.parseStyle(qml) + if parsing and qml: + qml = self.utils.parseStyle(qml) tempPath = None if qml: tempPath = os.path.join(os.path.dirname(__file__), "temp.qml") @@ -2364,6 +2376,24 @@ def getStyle(self, styleName, table_name, parsing=True): f.close() return tempPath + def getStyleDict(self): + self.checkAndOpenDb() + sql = self.gen.getStoredStyles() + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception( + self.tr("Problem getting styles from db: ") + query.lastError().text() + ) + styleDict = defaultdict(dict) + while query.next(): + tableSchema = query.value(0) + tableName = query.value(1) + geom = query.value(2) + styleName = query.value(3) + qml = query.value(4) + styleDict[styleName][f"{tableSchema}.{tableName}({geom})"] = qml + return styleDict + def importStyle(self, styleName, table_name, qml, tableSchema, useTransaction=True): self.checkAndOpenDb() if useTransaction: @@ -5009,3 +5039,139 @@ def databaseInfo(self): rowDict["srid"] = str(query.value(4)) out.append(rowDict) return out + + def checkIfSchemaExistsInDatabase(self, schemaName): + """ + Método que retorna true se o esquema existir, e falso caso contrário. + """ + self.checkAndOpenDb() + sql = self.gen.getSchemasFromInformationSchema() + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception(self.tr("Problem getting schemas exists in database")) + schemaExistsInDatabaseSet = set() + while query.next(): + schemaExistsInDatabaseSet.add(query.value(0)) + if schemaName in schemaExistsInDatabaseSet: + return True + return False + + def getTableListFromSchema(self, schemaName): + """ + Método que recebe um nome de esquema e retorna a lista com os nomes das tabelas. + """ + self.checkAndOpenDb() + sql = self.gen.getTablesFromInformationSchema(schemaName) + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception(self.tr("Problem getting table name in schema")) + tableListFromSchemaSet = set() + while query.next(): + tableListFromSchemaSet.add(query.value(0)) + return tableListFromSchemaSet + + def getColumnsDictFromSchema(self, schemaName): + """ + Método que recebe um nome de esquema e retorna um dicionário com a chave sendo o nome da tabela + e a chave como sendo um conjunto com os nomes da coluna. + """ + self.checkAndOpenDb() + sql = self.gen.getColumnsFromInformationSchema(schemaName) + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception(self.tr("Problem getting columns name in schema")) + tableColumnsSetDict = defaultdict(set) + while query.next(): + tableColumnsSetDict[query.value(2)].add(query.value(3)) + return tableColumnsSetDict + + def getPrimaryKeyDictFromSchema(self, schemaName): + """ + Método que recebe um nome de esquema e retonra um dicionário com a chave sendo o nome da tabela + e a chave como sendo um conjunto com os nomes da coluna. + """ + self.checkAndOpenDb() + sql = self.gen.getPrimaryKeyFromInformationSchema(schemaName) + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception( + self.tr(f"Problem getting primary key name in {schemaName}") + ) + tablePrimaryKeySetDict = defaultdict(set) + while query.next(): + tablePrimaryKeySetDict[query.value(1)].add(query.value(4)) + return tablePrimaryKeySetDict + + def getTupleSetFromTable(self, schemaName, tableName, columnNameList): + self.checkAndOpenDb() + sql = self.gen.getAllValuesFromTable(schemaName, tableName, columnNameList) + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception( + self.tr(f"Problem getting values from {schemaName}.{tableName}") + ) + tupleSet = set() + nColumns = len(columnNameList) + while query.next(): + newTuple = tuple(query.value(i) for i in range(nColumns)) + tupleSet.add(newTuple) + return tupleSet + + def getNameColumnIsNullableOrNoFromSchema(self, schemaName): + self.checkAndOpenDb() + sql = self.gen.getIfCollumnIsNullable(schemaName) + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception(self.tr(f"Problem getting values from {schemaName}")) + nameTableColumnIsNullableOrNoDict = defaultdict(dict) + while query.next(): + nameTableColumnIsNullableOrNoDict[query.value(2)][query.value(3)] = { + "nullable": query.value(6) + } + return nameTableColumnIsNullableOrNoDict + + def getForeignTableFromSchema(self, schemaName): + nameTableColumnIsNullableOrNoForeignTableDict = ( + self.getNameColumnIsNullableOrNoFromSchema(schemaName) + ) + sql = self.gen.getConstraintMapValueFromSchema() + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception(self.tr(f"Problem getting foreing valus from {schemaName}")) + while query.next(): + nameTableColumnIsNullableOrNoForeignTableDict[query.value(2)][ + query.value(3) + ]["foreignTable"] = query.value(5) + return nameTableColumnIsNullableOrNoForeignTableDict + + def getTypeColumnFromSchema(self, schemaName): + nameTableColumnIsNullableOrNoForeignTypeTableDict = ( + self.getForeignTableFromSchema(schemaName) + ) + sql = self.gen.getNameColumnTableDataTypeFromSchema(schemaName) + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception( + self.tr(f"Problem getting type of column from {schemaName}") + ) + while query.next(): + nameTableColumnIsNullableOrNoForeignTypeTableDict[query.value(1)][ + query.value(0) + ]["type"] = query.value(2) + return nameTableColumnIsNullableOrNoForeignTypeTableDict + + def getMaxLengthVarcharFromSchema(self, schemaName): + self.checkAndOpenDb() + sql = self.gen.getMaxLengthVarcharFromSchema(schemaName) + query = QSqlQuery(sql, self.db) + if not query.isActive(): + raise Exception( + self.tr(f"Problem getting maximum length from {schemaName}") + ) + nameTableColumnMaxVarcharDict = defaultdict(dict) + while query.next(): + if query.value(7) == "character varying": + nameTableColumnMaxVarcharDict[query.value(2)][ + query.value(3) + ] = query.value(8) + return nameTableColumnMaxVarcharDict diff --git a/DsgTools/core/Factories/DbFactory/shapefileDb.py b/DsgTools/core/Factories/DbFactory/shapefileDb.py index 08a194951..2ae7c10b8 100644 --- a/DsgTools/core/Factories/DbFactory/shapefileDb.py +++ b/DsgTools/core/Factories/DbFactory/shapefileDb.py @@ -406,7 +406,7 @@ def getOrphanGeomTables(self): def checkAndCreateStyleTable(self): return None - def getStylesFromDb(self, dbVersion): + def listStylesFromDb(self, dbVersion): return None def getGeomTypeDict(self, loadCentroids=False): diff --git a/DsgTools/core/Factories/DbFactory/spatialiteDb.py b/DsgTools/core/Factories/DbFactory/spatialiteDb.py index 91f1f49b0..f2eba12b0 100644 --- a/DsgTools/core/Factories/DbFactory/spatialiteDb.py +++ b/DsgTools/core/Factories/DbFactory/spatialiteDb.py @@ -591,7 +591,7 @@ def getOrphanGeomTables(self): def checkAndCreateStyleTable(self): return None - def getStylesFromDb(self, dbVersion): + def listStylesFromDb(self, dbVersion): return None def getGeomTypeDict(self, loadCentroids=False): diff --git a/DsgTools/core/Factories/SqlFactory/postgisSqlGenerator.py b/DsgTools/core/Factories/SqlFactory/postgisSqlGenerator.py index 220cc5ef5..b368fdc08 100644 --- a/DsgTools/core/Factories/SqlFactory/postgisSqlGenerator.py +++ b/DsgTools/core/Factories/SqlFactory/postgisSqlGenerator.py @@ -1178,6 +1178,10 @@ def createSpatialIndex(self, tableName, geomColumnName="geom"): ) return sql + def getStoredStyles(self): + sql = "select f_table_schema, f_table_name, f_geometry_column, stylename, styleqml from public.layer_styles where f_table_catalog = current_database()" + return sql + def getStyles(self): sql = "select description, f_table_schema, f_table_name, stylename from public.layer_styles where f_table_catalog = current_database()" return sql @@ -1208,7 +1212,7 @@ def createStyleTable(self): """ return sql - def getStylesFromDb(self, dbVersion): + def listStylesFromDb(self, dbVersion): """ Returns the stylenames of the database. The replace(stylename,'/' || f_table_name, '') is done due to compatibility issues @@ -2302,3 +2306,67 @@ def implementationVersion(self): :return: (str) query to database's implementation version (e.g. '5.2'). """ return """SELECT dbimplversion FROM public.db_metadata;""" + + def getSchemasFromInformationSchema(self): + return f"""SELECT schema_name FROM information_schema.schemata""" + + def getTablesFromInformationSchema(self, schemaName): + return f"""SELECT DISTINCT table_name from information_schema.tables where table_schema = '{schemaName}' """ + + def getColumnsFromInformationSchema(self, schemaName): + return f"""SELECT * FROM information_schema.columns WHERE table_schema = '{schemaName}'""" + + def getPrimaryKeyFromInformationSchema(self, schemaName): + return f""" + SELECT kcu.table_schema, + kcu.table_name, + tco.constraint_name, + kcu.ordinal_position AS POSITION, + kcu.column_name AS key_column + FROM information_schema.table_constraints tco + JOIN information_schema.key_column_usage kcu + ON kcu.constraint_name = tco.constraint_name + AND kcu.constraint_schema = tco.constraint_schema + AND kcu.constraint_name = tco.constraint_name + WHERE tco.constraint_type = 'PRIMARY KEY' AND kcu.table_schema = '{schemaName}' + ORDER BY kcu.table_schema, + kcu.table_name, + position + """ + + def getAllValuesFromTable(self, schemaName, tableName, columnNameList): + return f"SELECT {', '.join(columnNameList)} from {schemaName}.{tableName}" + + def getIfCollumnIsNullable(self, schemaName): + return f"SELECT * FROM information_schema.columns WHERE table_schema='{schemaName}'" + + def getConstraintMapValueFromSchema(self): + return f""" + SELECT + tc.table_schema, + tc.constraint_name, + tc.table_name, + kcu.column_name, + ccu.table_schema AS foreign_table_schema, + ccu.table_name AS foreign_table_name, + ccu.column_name AS foreign_column_name + FROM information_schema.table_constraints AS tc + JOIN information_schema.key_column_usage AS kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = tc.constraint_name + WHERE tc.constraint_type = 'FOREIGN KEY' + """ + + def getNameColumnTableDataTypeFromSchema(self, schemaName): + return f""" + SELECT column_name, table_name, data_type + FROM information_schema.columns + WHERE table_schema = '{schemaName}' + """ + + def getMaxLengthVarcharFromSchema(self, schemaName): + return f""" + SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = '{schemaName}' + """ diff --git a/DsgTools/core/Factories/SqlFactory/spatialiteSqlGenerator.py b/DsgTools/core/Factories/SqlFactory/spatialiteSqlGenerator.py index f4e6ec7c2..c87746c97 100644 --- a/DsgTools/core/Factories/SqlFactory/spatialiteSqlGenerator.py +++ b/DsgTools/core/Factories/SqlFactory/spatialiteSqlGenerator.py @@ -242,7 +242,7 @@ def getEDGVVersionAndImplementationVersion(self): sql = "SELECT edgvversion, dbimplversion FROM public_db_metadata LIMIT 1" return sql - def getStylesFromDb(self, dbVersion): + def listStylesFromDb(self, dbVersion): return None def getGeomTablesFromGeometryColumns(self, edgvVersion): diff --git a/DsgTools/core/GeometricTools/featureHandler.py b/DsgTools/core/GeometricTools/featureHandler.py index 99fd9de0a..2d2d056bd 100644 --- a/DsgTools/core/GeometricTools/featureHandler.py +++ b/DsgTools/core/GeometricTools/featureHandler.py @@ -515,14 +515,15 @@ def addFeatureToSpatialIndex( def getLyrUnprojectedGeographicBounds(self, inputLyr): crs = inputLyr.crs() + inputLyr.updateExtents() coordinateTransformer = QgsCoordinateTransform( crs, QgsCoordinateReferenceSystem(crs.geographicCrsAuthId()), QgsProject.instance(), - ) + ) if QgsCoordinateReferenceSystem(crs.geographicCrsAuthId()) != crs else None reprojectedGeographicBB = coordinateTransformer.transformBoundingBox( inputLyr.extent() - ) + ) if coordinateTransformer is not None else inputLyr.extent() xmin = reprojectedGeographicBB.xMinimum() ymin = reprojectedGeographicBB.yMinimum() xmax = reprojectedGeographicBB.xMaximum() diff --git a/DsgTools/core/GeometricTools/geometryHandler.py b/DsgTools/core/GeometricTools/geometryHandler.py index 834316c98..abf1bc0e1 100644 --- a/DsgTools/core/GeometricTools/geometryHandler.py +++ b/DsgTools/core/GeometricTools/geometryHandler.py @@ -161,7 +161,7 @@ def flipFeature(self, layer, feature, geomType=None, refreshCanvas=False): isMulti = QgsWkbTypes.isMultiType(layer.wkbType()) geom = feature.geometry() if geomType == 0: - if isMulti: + if geom.isMultipart(): nodes = geom.asMultiPoint() # inverting the point list by parts for idx, part in enumerate(nodes): @@ -174,7 +174,7 @@ def flipFeature(self, layer, feature, geomType=None, refreshCanvas=False): nodes = nodes[::-1] flippedFeatureGeom = QgsGeometry.fromPoint(nodes) elif geomType == 1: - if isMulti: + if geom.isMultipart(): nodes = geom.asMultiPolyline() for idx, part in enumerate(nodes): nodes[idx] = part[::-1] @@ -184,7 +184,7 @@ def flipFeature(self, layer, feature, geomType=None, refreshCanvas=False): nodes = nodes[::-1] flippedFeatureGeom = QgsGeometry.fromPolylineXY(nodes) elif geomType == 2: - if isMulti: + if geom.isMultipart(): nodes = geom.asMultiPolygon() for idx, part in enumerate(nodes): nodes[idx] = part[::-1] @@ -194,6 +194,8 @@ def flipFeature(self, layer, feature, geomType=None, refreshCanvas=False): nodes = nodes[::-1] flippedFeatureGeom = QgsGeometry.fromPolygonXY(nodes) # setting feature geometry to the flipped one + if isMulti and not geom.isMultipart(): + flippedFeatureGeom.convertToMultiType() layer.beginEditCommand("Flipping feature") feature.setGeometry(flippedFeatureGeom) layer.updateFeature(feature) diff --git a/DsgTools/core/GeometricTools/graphHandler.py b/DsgTools/core/GeometricTools/graphHandler.py index 44a938713..7698630cd 100644 --- a/DsgTools/core/GeometricTools/graphHandler.py +++ b/DsgTools/core/GeometricTools/graphHandler.py @@ -121,7 +121,8 @@ def buildGraph( hashDict: Dict[int, List[QByteArray]], nodeDict: Dict[QByteArray, int], feedback: Optional[QgsFeedback] = None, - directed: bool = False + directed: bool = False, + add_inside_river_attribute: bool = True, ) -> Any: """ Build a graph from hash dictionary and node dictionary. @@ -150,7 +151,8 @@ def buildGraph( break G.add_edge(nodeDict[wkb_1], nodeDict[wkb_2]) G[nodeDict[wkb_1]][nodeDict[wkb_2]]["featid"] = edgeId - G[nodeDict[wkb_1]][nodeDict[wkb_2]]["inside_river"] = False + if add_inside_river_attribute: + G[nodeDict[wkb_1]][nodeDict[wkb_2]]["inside_river"] = False if feedback is not None: feedback.setProgress(current * progressStep) return G @@ -165,7 +167,13 @@ def buildAuxStructures( useWkt: Optional[bool] = False, computeNodeLayerIdDict: Optional[bool] = False, addEdgeLength: Optional[bool] = False, -) -> Tuple[Dict[QByteArray, int], Dict[int, QByteArray], Dict[int, QgsFeature], Dict[int, Dict[int, QByteArray]], Any]: +) -> Tuple[ + Dict[QByteArray, int], + Dict[int, QByteArray], + Dict[int, QgsFeature], + Dict[int, Dict[int, QByteArray]], + Any, +]: """ Build auxiliary data structures for network analysis. @@ -257,13 +265,21 @@ def buildAuxStructures( ) ) -def buildDirectionalGraphFromIdList(nx: Any, G: Any, nodeDict: Dict[QByteArray, int], hashDict: Dict[int, Dict[int, List[int]]], idSet: Set[int], feedback: Optional[QgsFeedback]=None) -> Any: + +def buildDirectionalGraphFromIdList( + nx: Any, + G: Any, + nodeDict: Dict[QByteArray, int], + hashDict: Dict[int, Dict[int, List[int]]], + idSet: Set[int], + feedback: Optional[QgsFeedback] = None, +) -> Any: DiG = nx.DiGraph() nFeats = len(idSet) if nFeats == 0: return DiG if feedback is not None: - stepSize = 100/nFeats + stepSize = 100 / nFeats for current, featid in enumerate(idSet): if feedback is not None and feedback.isCanceled(): return DiG @@ -275,7 +291,7 @@ def buildDirectionalGraphFromIdList(nx: Any, G: Any, nodeDict: Dict[QByteArray, return DiG -def evaluateStreamOrder(G: Any, feedback: Optional[QgsFeedback]=None) -> Any: +def evaluateStreamOrder(G: Any, feedback: Optional[QgsFeedback] = None) -> Any: """ Evaluate stream order for the given graph. @@ -556,6 +572,7 @@ def is_flow_invalid(DiG, node: int) -> bool: succs = len(list(DiG.successors(node))) return (preds > 0 and succs == 0) or (preds == 0 and succs > 0) + def flip_edge(DiG, edge: Tuple): start, end = edge attrDict = DiG[start][end] @@ -563,15 +580,14 @@ def flip_edge(DiG, edge: Tuple): DiG.add_edge(end, start, **attrDict) - def buildAuxFlowGraph( nx, G, fixedInNodeSet: Set[int], fixedOutNodeSet: Set[int], nodeIdDict: Dict[int, QByteArray], - constantSinkPointSet: Optional[Set[int]]=None, - DiG: Optional[Any]=None, + constantSinkPointSet: Optional[Set[int]] = None, + DiG: Optional[Any] = None, feedback: Optional[QgsFeedback] = None, ): """ @@ -597,7 +613,9 @@ def buildAuxFlowGraph( DiG = nx.DiGraph() if DiG is None else DiG visitedNodes = set() nEdges = len(list(G.edges)) - constantSinkPointSet = set() if constantSinkPointSet is None else constantSinkPointSet + constantSinkPointSet = ( + set() if constantSinkPointSet is None else constantSinkPointSet + ) if nEdges == 0: return DiG multiStepFeedback = ( @@ -638,18 +656,27 @@ def buildAuxFlowGraph( remainingEdges = nEdges - len(list(DiG.edges)) stepSize = 100 / remainingEdges currentEdge = 0 + def distance(start, end): startGeom, endGeom = QgsGeometry(), QgsGeometry() startGeom.fromWkb(nodeIdDict[start]) endGeom.fromWkb(nodeIdDict[end]) return startGeom.distance(endGeom) - + pairList = sorted( ( - (start, end) for start, end in product(fixedInNodeSet, fixedOutNodeSet) if nx.has_path(G, start, end) + (start, end) + for start, end in product(fixedInNodeSet, fixedOutNodeSet) + if nx.has_path(G, start, end) + ), + key=lambda x: ( + any( + DiG[a][b]["inside_river"] + for (a, b) in chain(DiG.in_edges(x[0]), DiG.out_edges(x[0])) + ), + distance(x[0], x[1]), ), - key=lambda x: (any(DiG[a][b]["inside_river"] for (a, b) in chain(DiG.in_edges(x[0]), DiG.out_edges(x[0]))), distance(x[0], x[1])), - reverse=True + reverse=True, ) for (start, end) in pairList: if multiStepFeedback is not None and feedback.isCanceled(): @@ -660,7 +687,7 @@ def distance(start, end): if (n0, n1) in DiG.edges or (n1, n0) in DiG.edges: continue add_edge_from_graph_to_digraph(G, DiG, n0, n1) - if (DiG.degree(n0) == G.degree(n0) and is_flow_invalid(DiG, n0)): + if DiG.degree(n0) == G.degree(n0) and is_flow_invalid(DiG, n0): flip_edge(DiG, (n0, n1)) if multiStepFeedback is not None: currentEdge += 1 @@ -704,9 +731,165 @@ def distance(start, end): stepSize = 100 / remainingEdges currentEdge = 0 for (a, b) in G.edges: - if ((a, b) in DiG.edges and DiG[a][b]["featid"] == G[a][b]["featid"] ) or ((b, a) in DiG.edges and DiG[b][a]["featid"] == G[b][a]["featid"]): + if ((a, b) in DiG.edges and DiG[a][b]["featid"] == G[a][b]["featid"]) or ( + (b, a) in DiG.edges and DiG[b][a]["featid"] == G[b][a]["featid"] + ): continue add_edge_from_graph_to_digraph(G, DiG, a, b) - if (is_flow_invalid(DiG, a) and set(nx.dfs_postorder_nodes(DiG, a)).intersection(fixedOutNodeSet) == set()) or (is_flow_invalid(DiG, b) and set(nx.dfs_postorder_nodes(DiG, b)).intersection(fixedOutNodeSet) == set()): + if ( + is_flow_invalid(DiG, a) + and set(nx.dfs_postorder_nodes(DiG, a)).intersection(fixedOutNodeSet) + == set() + ) or ( + is_flow_invalid(DiG, b) + and set(nx.dfs_postorder_nodes(DiG, b)).intersection(fixedOutNodeSet) + == set() + ): flip_edge(DiG, (a, b)) return DiG + + +def find_mergeable_edges_on_graph(nx, G, feedback: Optional[QgsFeedback] = None): + """ + Find mergeable edges in a graph. + + This function analyzes a graph to identify mergeable edges. Mergeable edges are pairs of edges + that share common nodes with degree 2. The function returns a dictionary where keys are sets of nodes that + can be merged, and values are sets of mergeable edge pairs. + + Parameters: + - G (networkx.Graph): The input graph to analyze. + - feedback (Optional[QgsFeedback]): A QgsFeedback object for providing user feedback during + processing. If provided and canceled, the function will terminate early. + + Returns: + - Dict[Set[Hashable], Set[Tuple[Hashable, Hashable]]]: A dictionary where keys are sets of nodes + that can be merged, and values are sets of frozenset pairs representing mergeable edges. + + Note: + - Mergeable edges are defined as edges that connect the same set of nodes, potentially forming + a multi-edge in the graph. + + Example: + ``` + G = nx.Graph() + G.add_edges_from([ + (1, 2), (3, 2), + (2, 4), (4, 5), (2, 18), (18, 6), + (7, 6), (7, 17), (17, 8), + (8, 9), (8, 13), + (9, 10), + (11, 10), + (12, 10), + (13, 14), + (15, 13), (15, 16), + ]) + mergeable_edges = find_mergeable_edges_on_graph(G) + ``` + + In the example above, `mergeable_edges` may contain: + ``` + { + frozenset({4, 5}): {frozenset({4, 5}), frozenset({2, 4})}, + frozenset({17, 18, 6, 7}): {frozenset({8, 17}), frozenset({18, 2}), frozenset({6, 7}), frozenset({17, 7}), frozenset({18, 6})}, + frozenset({16, 15}): {frozenset({13, 15}), frozenset({16, 15})}, + frozenset({9}): {frozenset({8, 9}), frozenset({9, 10})} + } + ``` + """ + outputGraphDict = defaultdict(lambda: nx.Graph()) + degree2nodes = (i for i in G.nodes if G.degree(i) == 2) + if feedback is not None and feedback.isCanceled(): + return outputGraphDict + candidatesSetofFrozenSets = set( + frozenset(fetch_connected_nodes(G, n, 2)) for n in degree2nodes + ) + if feedback is not None and feedback.isCanceled(): + return outputGraphDict + nSteps = len(candidatesSetofFrozenSets) + if nSteps == 0: + return outputGraphDict + if feedback is not None: + stepSize = 100 / nSteps + for current, candidateSet in enumerate(candidatesSetofFrozenSets): + if feedback is not None and feedback.isCanceled(): + break + for node in candidateSet: + for n0, n1 in G.edges(node): + outputGraphDict[candidateSet].add_edge(n0, n1) + outputGraphDict[candidateSet][n0][n1]["featid"] = G[n0][n1]["featid"] + if feedback is not None: + feedback.setProgress(current * stepSize) + return outputGraphDict + + +def filter_mergeable_graphs_using_attibutes( + nx, G, featDict: Dict[int, QgsFeature], attributeNameList: List[str], isMulti: bool +) -> Tuple[Set[int], Set[int]]: + auxDict = defaultdict(lambda: nx.Graph()) + featureSetToUpdate, deleteIdSet = set(), set() + for n0, n1 in G.edges: + featid = G[n0][n1]["featid"] + feat = featDict[featid] + attrTuple = tuple(feat[i] for i in attributeNameList) + auxDict[attrTuple].add_edge(n0, n1) + auxDict[attrTuple][n0][n1]["featid"] = featid + for auxGraph in auxDict.values(): + for mergeableG in find_mergeable_edges_on_graph(nx, auxGraph).values(): + if len(mergeableG.edges) < 2: + continue + idToKeep, *idsToDelete = [ + mergeableG[n0][n1]["featid"] for n0, n1 in mergeableG.edges + ] + outputFeat = featDict[idToKeep] + geom = outputFeat.geometry() + for id in idsToDelete: + geom_b = featDict[id].geometry() + geom = geom.combine(geom_b).mergeLines() + deleteIdSet.add(id) + if isMulti: + geom.convertToMultiType() + outputFeat.setGeometry(geom) + featureSetToUpdate.add(outputFeat) + return featureSetToUpdate, deleteIdSet + + +def identify_unmerged_edges_on_graph( + nx, + G, + featDict: Dict[int, QgsFeature], + nodeIdDict: Dict[int, QByteArray], + filterPointSet: Set[QByteArray], + filterLineLayer: QgsVectorLayer, + attributeNameList: List[str], +) -> Set[int]: + auxDict = defaultdict(lambda: nx.Graph()) + outputIdSet = set() + for n0, n1 in G.edges: + featid = G[n0][n1]["featid"] + feat = featDict[featid] + attrTuple = tuple(feat[i] for i in attributeNameList) + auxDict[attrTuple].add_edge(n0, n1) + auxDict[attrTuple][n0][n1]["featid"] = featid + for auxGraph in auxDict.values(): + for idSet, mergeableG in find_mergeable_edges_on_graph(nx, auxGraph).items(): + if len(mergeableG.edges) < 2: + continue + candidatePointSet = idSet - filterPointSet + if candidatePointSet == set(): + continue + for nodeId in candidatePointSet: + if nodeId in outputIdSet or mergeableG.degree(nodeId) != 2: + continue + if filterLineLayer is not None: + geom = QgsGeometry() + geom.fromWkb(nodeIdDict[nodeId]) + buffer = geom.buffer(1e-6, -1) + geomBB = buffer.boundingBox() + if any( + f.geometry().intersects(geom) + for f in filterLineLayer.getFeatures(geomBB) + ): + continue + outputIdSet.add(nodeId) + return outputIdSet diff --git a/DsgTools/core/GeometricTools/rasterHandler.py b/DsgTools/core/GeometricTools/rasterHandler.py index 04e9afcda..326f8f704 100644 --- a/DsgTools/core/GeometricTools/rasterHandler.py +++ b/DsgTools/core/GeometricTools/rasterHandler.py @@ -41,12 +41,12 @@ def readAsNumpy(inputRaster: Union[str, QgsRasterLayer]) -> Tuple[Dataset, np.array]: - inputRaster = ( + inputRasterPath = ( inputRaster.dataProvider().dataSourceUri() if isinstance(inputRaster, QgsRasterLayer) else inputRaster ) - ds = gdal.Open(inputRaster) + ds = gdal.Open(inputRasterPath) return ds, np.array(ds.GetRasterBand(1).ReadAsArray().transpose()) @@ -59,7 +59,13 @@ def getMaxCoordinatesFromNpArray(npArray: np.array) -> np.array: def getMinCoordinatesFromNpArray(npArray: np.array) -> np.array: - return np.argwhere(npArray == npArray[~np.isnan(npArray)].min()) + not_nan_mask = ~np.isnan(npArray) + try: + min_value = np.min(npArray[np.where((npArray > -500.) & not_nan_mask)]) + except: + return np.array([]) + return np.transpose(np.where((npArray == min_value) & not_nan_mask)) + # return np.argwhere(npArray == npArray[~np.isnan(npArray)].min()) def createFeatureWithPixelValueFromPixelCoordinates( diff --git a/DsgTools/gui/CustomWidgets/ConnectionWidgets/ServerConnectionWidgets/customServerConnectionWidget.py b/DsgTools/gui/CustomWidgets/ConnectionWidgets/ServerConnectionWidgets/customServerConnectionWidget.py index 38d134f1e..699dac8d6 100644 --- a/DsgTools/gui/CustomWidgets/ConnectionWidgets/ServerConnectionWidgets/customServerConnectionWidget.py +++ b/DsgTools/gui/CustomWidgets/ConnectionWidgets/ServerConnectionWidgets/customServerConnectionWidget.py @@ -104,7 +104,10 @@ def selectedDatabases(self, dbList, type): ) self.selectedDbsDict[dbNameAlias] = localDb # do get dicts - localDict = localDb.getStyleDict(localDb.getDatabaseVersion()) + try: + localDict = localDb.getStyleDict() + except: + localDict = dict() for key in list(localDict.keys()): self.stylesDict[key]["style"] = localDict[key] if dbNameAlias not in self.stylesDict[key]["dbList"]: @@ -274,5 +277,5 @@ def getStyles(self, type, abstractDb): """ dbVersion = abstractDb.getDatabaseVersion() abstractDb.checkAndCreateStyleTable() - styles = abstractDb.getStyleDict(dbVersion) + styles = abstractDb.listStylesFromDb(dbVersion) self.styleChanged.emit(type, styles) diff --git a/DsgTools/gui/ProductionTools/MapTools/GenericSelectionTool/genericSelectionTool.py b/DsgTools/gui/ProductionTools/MapTools/GenericSelectionTool/genericSelectionTool.py index 5ac8f4db8..1cebf6dc3 100644 --- a/DsgTools/gui/ProductionTools/MapTools/GenericSelectionTool/genericSelectionTool.py +++ b/DsgTools/gui/ProductionTools/MapTools/GenericSelectionTool/genericSelectionTool.py @@ -232,7 +232,7 @@ def getPrimitiveDict(self, e, hasControlModifier=False, hasAltModifier=False): if ( not isinstance(lyr, QgsVectorLayer) or (self.layerHasPartInBlackList(lyr.name())) - or lyr not in visibleLayers + or lyr not in visibleLayers or lyr.readOnly() ): continue if ( diff --git a/DsgTools/gui/ProductionTools/Toolbars/ReviewTools/reviewToolbar.py b/DsgTools/gui/ProductionTools/Toolbars/ReviewTools/reviewToolbar.py index 95ccc6531..4575c7b93 100644 --- a/DsgTools/gui/ProductionTools/Toolbars/ReviewTools/reviewToolbar.py +++ b/DsgTools/gui/ProductionTools/Toolbars/ReviewTools/reviewToolbar.py @@ -28,25 +28,19 @@ QgsCoordinateReferenceSystem, QgsCoordinateTransform, QgsFeatureRequest, - QgsMapLayer, QgsProject, QgsRectangle, QgsVectorLayer, - QgsWkbTypes, QgsFeature, QgsExpression, ) -from qgis.gui import QgsMapTool, QgsMessageBar, QgisInterface -from qgis.PyQt import QtCore, QtGui, uic -from qgis.PyQt.Qt import QObject, QVariant -from qgis.PyQt.QtCore import QObject, QSettings, Qt, pyqtSignal, pyqtSlot +from qgis.gui import QgsMapTool, QgisInterface +from qgis.PyQt.Qt import QVariant +from qgis.PyQt.QtCore import QSettings, pyqtSignal, pyqtSlot from qgis.PyQt.QtGui import QIcon -from qgis.PyQt.QtWidgets import QAction, QMessageBox, QSpinBox, QWidget -from qgis.PyQt.QtXml import QDomDocument -from qgis.core.additions.edit import edit +from qgis.PyQt.QtWidgets import QAction, QMessageBox, QWidget from .review_ui import Ui_ReviewToolbar -from enum import Enum class ReviewToolbar(QWidget, Ui_ReviewToolbar): @@ -72,7 +66,7 @@ def __init__(self, iface: QgisInterface, parent: Optional[QWidget] = None): self.visitedFieldComboBox.setAllowEmptyFieldName(True) self.rankFieldComboBox.setToolTip(self.tr("Set rank field")) self.rankFieldComboBox.setAllowEmptyFieldName(True) - self.zoomComboBox.setCurrentIndex(ReviewToolbar.ZoomToNext) + self.zoomComboBox.setCurrentIndex(ReviewToolbar.DoNothing) icon_path = ":/plugins/DsgTools/icons/attributeSelector.png" text = self.tr("DSGTools: Mark tile as done") self.applyPushButtonAction = self.add_action( @@ -601,7 +595,7 @@ def setState( layer: QgsVectorLayer, rankFieldName: str, visitedFieldName: str, - zoomType: int = 0, + zoomType: int = 2, currentTile: Optional[int] = None, ): self.mMapLayerComboBox.setLayer(layer) diff --git a/DsgTools/gui/ProductionTools/Toolboxes/toolBoxesGuiManager.py b/DsgTools/gui/ProductionTools/Toolboxes/toolBoxesGuiManager.py index 3f58f632b..5ff184db1 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/toolBoxesGuiManager.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/toolBoxesGuiManager.py @@ -46,7 +46,7 @@ def __init__( parentMenu=None, toolbar=None, stackButton=None, - acquisitionMenuCtrl=AcquisitionMenuCtrl(), + acquisitionMenuCtrl=None, ): """Constructor.""" super(ToolBoxesGuiManager, self).__init__() @@ -57,7 +57,7 @@ def __init__( self.stackButton = stackButton self.iconBasePath = ":/plugins/DsgTools/icons/" - self.acquisitionMenuCtrl = acquisitionMenuCtrl + self.acquisitionMenuCtrl = acquisitionMenuCtrl if acquisitionMenuCtrl is not None else AcquisitionMenuCtrl() def initGui(self): self.qaToolBox = None diff --git a/DsgTools/gui/ServerTools/batchDbManager.py b/DsgTools/gui/ServerTools/batchDbManager.py index 134277f11..beb61e19f 100644 --- a/DsgTools/gui/ServerTools/batchDbManager.py +++ b/DsgTools/gui/ServerTools/batchDbManager.py @@ -401,7 +401,7 @@ def getStyleDir(self, versionList): ) return "" - def getStylesFromDbs(self, perspective="style"): + def listStylesFromDbs(self, perspective="style"): """ Returns a dict of styles in a form acording to perspective: if perspective = 'style' : [styleName][dbName][tableName] = timestamp @@ -427,7 +427,7 @@ def createItem(self, parent, text, column): def populateStylesInterface(self): self.stylesTreeWidget.clear() - allStylesDict = self.getStylesFromDbs() + allStylesDict = self.listStylesFromDbs() rootNode = self.stylesTreeWidget.invisibleRootItem() for styleName in list(allStylesDict.keys()): parentStyleItem = self.createItem(rootNode, styleName, 0) @@ -451,7 +451,7 @@ def populateStylesInterface(self): @pyqtSlot(bool) def on_deleteStyles_clicked(self): dbsDict = self.instantiateAbstractDbs() - styleDict = self.getStylesFromDbs() + styleDict = self.listStylesFromDbs() styleList = list(styleDict.keys()) dlg = SelectStyles(styleList) execStatus = dlg.exec_() diff --git a/DsgTools/metadata.txt b/DsgTools/metadata.txt index 4b8ec60b3..8ffe05357 100644 --- a/DsgTools/metadata.txt +++ b/DsgTools/metadata.txt @@ -65,7 +65,7 @@ changelog= - Novo processo de ajustar parâmetros da ferramenta de aquisição com ângulos retos (integração com o FP/SAP); - Novo processo de converter entre bancos de mesma modelagem, clipando com um polígono feito para integração com FP/SAP (ClipAndCopyFeaturesBetweenDatabasesAlgorithm); - Novo processo de verificar ligação na moldura; - + Melhorias: - Melhoria de desempenho na construção de polígonos quando se utiliza polígonos na entrada que contém linhas que podem delimitar (caso área edificada versus via deslocamento); - Adicionado registro de alteração do tile na barra de ferramentas de revisão; diff --git a/tests/expected_outputs/identify_unmerged_lines_with_same_attribute_set/test_1/FLAGS.geojson b/tests/expected_outputs/identify_unmerged_lines_with_same_attribute_set/test_1/FLAGS.geojson new file mode 100644 index 000000000..d7c1200c8 --- /dev/null +++ b/tests/expected_outputs/identify_unmerged_lines_with_same_attribute_set/test_1/FLAGS.geojson @@ -0,0 +1,17 @@ +{ +"type": "FeatureCollection", +"name": "FLAGS", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ -0.580298931890692, 0.616020648792184 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.034201701235946, 0.059432622646842 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.474522088609893, -0.422016168719688 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ -0.339498582828625, 0.280874280611013 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.21160310475492, 0.139866764454223 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.323077417695473, 0.235243055555556 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.47052649033558, 0.36366644140339 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.814820843850651, -0.169379286197885 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.109687375547591, -0.348805256869773 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ -0.121574747395765, -0.294493531767752 ] } } +] +} diff --git a/tests/expected_outputs/identify_unmerged_lines_with_same_attribute_set/test_2/FLAGS.geojson b/tests/expected_outputs/identify_unmerged_lines_with_same_attribute_set/test_2/FLAGS.geojson new file mode 100644 index 000000000..8f38e0f51 --- /dev/null +++ b/tests/expected_outputs/identify_unmerged_lines_with_same_attribute_set/test_2/FLAGS.geojson @@ -0,0 +1,15 @@ +{ +"type": "FeatureCollection", +"name": "FLAGS", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.034201701235946, 0.059432622646842 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.474522088609893, -0.422016168719688 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ -0.339498582828625, 0.280874280611013 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.21160310475492, 0.139866764454223 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.323077417695473, 0.235243055555556 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.47052649033558, 0.36366644140339 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.814820843850651, -0.169379286197885 ] } }, +{ "type": "Feature", "properties": { "reason": "Lines with same attribute set that are not merged." }, "geometry": { "type": "Point", "coordinates": [ 0.109687375547591, -0.348805256869773 ] } } +] +} diff --git a/tests/testing_datasets/GeoJSON/identify_unmerged_lines_with_same_attribute_set/line_constraint.geojson b/tests/testing_datasets/GeoJSON/identify_unmerged_lines_with_same_attribute_set/line_constraint.geojson new file mode 100644 index 000000000..e5ba9ddf9 --- /dev/null +++ b/tests/testing_datasets/GeoJSON/identify_unmerged_lines_with_same_attribute_set/line_constraint.geojson @@ -0,0 +1,8 @@ +{ +"type": "FeatureCollection", +"name": "line_constraint", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ -0.248456790123457, -0.695061728395061 ], [ -0.121574747395765, -0.294493531767752 ], [ -0.114506172839507, -0.146296296296296 ] ] } } +] +} diff --git a/tests/testing_datasets/GeoJSON/identify_unmerged_lines_with_same_attribute_set/point_constraint.geojson b/tests/testing_datasets/GeoJSON/identify_unmerged_lines_with_same_attribute_set/point_constraint.geojson new file mode 100644 index 000000000..9862ca926 --- /dev/null +++ b/tests/testing_datasets/GeoJSON/identify_unmerged_lines_with_same_attribute_set/point_constraint.geojson @@ -0,0 +1,13 @@ +{ +"type": "FeatureCollection", +"name": "point_constraint", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -0.580298931890692, 0.616020648792184 ] } }, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -0.771296296296296, 0.164814814814815 ] } }, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.454012345679012, 0.190740740740741 ] } }, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.466975308641975, -0.431481481481481 ] } }, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -0.952777777777778, -0.466049382716049 ] } }, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.026234567901235, -0.320987654320988 ] } } +] +} diff --git a/tests/testing_datasets/GeoJSON/identify_unmerged_lines_with_same_attribute_set/test1.geojson b/tests/testing_datasets/GeoJSON/identify_unmerged_lines_with_same_attribute_set/test1.geojson new file mode 100644 index 000000000..767d3cc17 --- /dev/null +++ b/tests/testing_datasets/GeoJSON/identify_unmerged_lines_with_same_attribute_set/test1.geojson @@ -0,0 +1,25 @@ +{ +"type": "FeatureCollection", +"name": "test1", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "id": 1, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.729938271604938, 0.787037037037037 ], [ -0.580298931890692, 0.616020648792184 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 2, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.060185185185185, 0.04320987654321 ], [ 0.034201701235946, 0.059432622646842 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 3, "attr": "c" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.782407407407407, 0.540123456790123 ], [ 0.834876543209877, 0.361111111111111 ], [ 0.831076252151338, 0.271804271235443 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 4, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.813271604938271, -0.265432098765432 ], [ 0.723765432098765, -0.388888888888889 ], [ 0.474522088609893, -0.422016168719688 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 5, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.813271604938271, -0.265432098765432 ], [ 0.844135802469136, -0.472222222222222 ], [ 0.908950617283951, -0.679012345679012 ], [ 0.924382716049382, -0.768518518518518 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 6, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.580298931890692, 0.616020648792184 ], [ -0.492283950617284, 0.515432098765432 ], [ -0.339498582828625, 0.280874280611013 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 7, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.339498582828625, 0.280874280611013 ], [ -0.273148148148148, 0.179012345679012 ], [ -0.060185185185185, 0.04320987654321 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 8, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.034201701235946, 0.059432622646842 ], [ 0.137345679012346, 0.077160493827161 ], [ 0.21160310475492, 0.139866764454223 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 9, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.21160310475492, 0.139866764454223 ], [ 0.276234567901235, 0.194444444444445 ], [ 0.323077417695473, 0.235243055555556 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 10, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.323077417695473, 0.235243055555556 ], [ 0.47052649033558, 0.36366644140339 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 11, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.47052649033558, 0.36366644140339 ], [ 0.563271604938272, 0.444444444444444 ], [ 0.782407407407407, 0.540123456790123 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 12, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.831076252151338, 0.271804271235443 ], [ 0.821712308267864, 0.051751589973812 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 13, "attr": "c" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.821712308267864, 0.051751589973812 ], [ 0.816358024691358, -0.074074074074074 ], [ 0.814820843850651, -0.169379286197885 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 14, "attr": "c" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.814820843850651, -0.169379286197885 ], [ 0.813271604938271, -0.265432098765432 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 15, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.474522088609893, -0.422016168719688 ], [ 0.236111111111111, -0.453703703703704 ], [ 0.060185185185185, -0.570987654320988 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 16, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.236111111111111, -0.453703703703704 ], [ 0.146604938271605, -0.361111111111111 ], [ 0.109687375547591, -0.348805256869773 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 17, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.109687375547591, -0.348805256869773 ], [ 0.026234567901235, -0.320987654320988 ], [ -0.121574747395765, -0.294493531767752 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 18, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.121574747395765, -0.294493531767752 ], [ -0.300925925925926, -0.262345679012346 ] ] ] } } +] +} diff --git a/tests/testing_datasets/GeoJSON/merge_lines/test1.geojson b/tests/testing_datasets/GeoJSON/merge_lines/test1.geojson new file mode 100644 index 000000000..767d3cc17 --- /dev/null +++ b/tests/testing_datasets/GeoJSON/merge_lines/test1.geojson @@ -0,0 +1,25 @@ +{ +"type": "FeatureCollection", +"name": "test1", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "id": 1, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.729938271604938, 0.787037037037037 ], [ -0.580298931890692, 0.616020648792184 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 2, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.060185185185185, 0.04320987654321 ], [ 0.034201701235946, 0.059432622646842 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 3, "attr": "c" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.782407407407407, 0.540123456790123 ], [ 0.834876543209877, 0.361111111111111 ], [ 0.831076252151338, 0.271804271235443 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 4, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.813271604938271, -0.265432098765432 ], [ 0.723765432098765, -0.388888888888889 ], [ 0.474522088609893, -0.422016168719688 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 5, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.813271604938271, -0.265432098765432 ], [ 0.844135802469136, -0.472222222222222 ], [ 0.908950617283951, -0.679012345679012 ], [ 0.924382716049382, -0.768518518518518 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 6, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.580298931890692, 0.616020648792184 ], [ -0.492283950617284, 0.515432098765432 ], [ -0.339498582828625, 0.280874280611013 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 7, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.339498582828625, 0.280874280611013 ], [ -0.273148148148148, 0.179012345679012 ], [ -0.060185185185185, 0.04320987654321 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 8, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.034201701235946, 0.059432622646842 ], [ 0.137345679012346, 0.077160493827161 ], [ 0.21160310475492, 0.139866764454223 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 9, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.21160310475492, 0.139866764454223 ], [ 0.276234567901235, 0.194444444444445 ], [ 0.323077417695473, 0.235243055555556 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 10, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.323077417695473, 0.235243055555556 ], [ 0.47052649033558, 0.36366644140339 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 11, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.47052649033558, 0.36366644140339 ], [ 0.563271604938272, 0.444444444444444 ], [ 0.782407407407407, 0.540123456790123 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 12, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.831076252151338, 0.271804271235443 ], [ 0.821712308267864, 0.051751589973812 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 13, "attr": "c" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.821712308267864, 0.051751589973812 ], [ 0.816358024691358, -0.074074074074074 ], [ 0.814820843850651, -0.169379286197885 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 14, "attr": "c" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.814820843850651, -0.169379286197885 ], [ 0.813271604938271, -0.265432098765432 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 15, "attr": "a" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.474522088609893, -0.422016168719688 ], [ 0.236111111111111, -0.453703703703704 ], [ 0.060185185185185, -0.570987654320988 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 16, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.236111111111111, -0.453703703703704 ], [ 0.146604938271605, -0.361111111111111 ], [ 0.109687375547591, -0.348805256869773 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 17, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ 0.109687375547591, -0.348805256869773 ], [ 0.026234567901235, -0.320987654320988 ], [ -0.121574747395765, -0.294493531767752 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 18, "attr": "b" }, "geometry": { "type": "MultiLineString", "coordinates": [ [ [ -0.121574747395765, -0.294493531767752 ], [ -0.300925925925926, -0.262345679012346 ] ] ] } } +] +}