Skip to content

Commit

Permalink
Move some temp file/folder utils to c++
Browse files Browse the repository at this point in the history
  • Loading branch information
nyalldawson committed Jun 21, 2017
1 parent e8a03b9 commit a87ca09
Show file tree
Hide file tree
Showing 11 changed files with 99 additions and 47 deletions.
16 changes: 16 additions & 0 deletions python/core/processing/qgsprocessingutils.sip
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,22 @@ class QgsProcessingUtils
:rtype: QVariant
%End

static QString tempFolder();
%Docstring
Returns a session specific processing temporary folder for use in processing algorithms.
.. seealso:: generateTempFileName()
:rtype: str
%End

static QString generateTempFilename( const QString &basename );
%Docstring
Returns a temporary filename for a given file, putting it into
a temporary folder (creating that folder in the process),
but not changing the ``basename``.
.. seealso:: tempFolder()
:rtype: str
%End

};


Expand Down
7 changes: 4 additions & 3 deletions python/plugins/processing/ProcessingPlugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@
import os
import sys

from qgis.core import QgsApplication
from qgis.core import (QgsApplication,
QgsProcessingUtils)
from qgis.gui import QgsOptionsWidgetFactory
from qgis.PyQt.QtCore import Qt, QCoreApplication, QDir
from qgis.PyQt.QtWidgets import QMenu, QAction
Expand All @@ -44,7 +45,7 @@
from processing.gui.ResultsDock import ResultsDock
from processing.gui.AlgorithmLocatorFilter import AlgorithmLocatorFilter
from processing.modeler.ModelerDialog import ModelerDialog
from processing.tools.system import tempFolder, tempHelpFolder
from processing.tools.system import tempHelpFolder
from processing.gui.menus import removeMenus, initializeMenus, createMenus
from processing.core.ProcessingResults import resultsList

Expand Down Expand Up @@ -142,7 +143,7 @@ def unload(self):
self.menu.deleteLater()

# delete temporary output files
folder = tempFolder()
folder = QgsProcessingUtils.tempFolder()
if QDir(folder).exists():
shutil.rmtree(folder, True)

Expand Down
4 changes: 2 additions & 2 deletions python/plugins/processing/algs/gdal/buildvrt.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@

from qgis.PyQt.QtGui import QIcon

from qgis.core import QgsProcessingUtils
from processing.algs.gdal.GdalAlgorithm import GdalAlgorithm
from processing.core.outputs import OutputRaster
from processing.core.parameters import ParameterBoolean
from processing.core.parameters import ParameterMultipleInput
from processing.core.parameters import ParameterSelection
from processing.algs.gdal.GdalUtils import GdalUtils
from processing.tools.system import tempFolder
from processing.tools import dataobjects

pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0]
Expand Down Expand Up @@ -85,7 +85,7 @@ def getConsoleCommands(self, parameters):
arguments.append('-allow_projection_difference')
# Always write input files to a text file in case there are many of them and the
# length of the command will be longer then allowed in command prompt
listFile = os.path.join(tempFolder(), 'buildvrtInputFiles.txt')
listFile = os.path.join(QgsProcessingUtils.tempFolder(), 'buildvrtInputFiles.txt')
with open(listFile, 'w') as f:
f.write(self.getParameterValue(buildvrt.INPUT).replace(';', '\n'))
arguments.append('-input_file_list')
Expand Down
4 changes: 2 additions & 2 deletions python/plugins/processing/algs/grass7/Grass7Utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
QgsMessageLog)
from qgis.PyQt.QtCore import QCoreApplication
from processing.core.ProcessingConfig import ProcessingConfig
from processing.tools.system import userFolder, isWindows, isMac, tempFolder, mkdir
from processing.tools.system import userFolder, isWindows, isMac, mkdir
from processing.tests.TestData import points


Expand Down Expand Up @@ -205,7 +205,7 @@ def grassMapsetFolder():

@staticmethod
def grassDataFolder():
tempfolder = os.path.join(tempFolder(), 'grassdata')
tempfolder = os.path.join(QgsProcessingUtils.tempFolder(), 'grassdata')
mkdir(tempfolder)
return tempfolder

Expand Down
4 changes: 2 additions & 2 deletions python/plugins/processing/algs/saga/SagaAlgorithm.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
OutputRaster)
from processing.tools import dataobjects
from processing.algs.help import shortHelp
from processing.tools.system import getTempFilename, getTempFilenameInTempFolder
from processing.tools.system import getTempFilename
from processing.algs.saga.SagaNameDecorator import decoratedAlgorithmName, decoratedGroupName
from . import SagaUtils

Expand Down Expand Up @@ -342,7 +342,7 @@ def exportRasterLayer(self, source):
filename = ''.join(c for c in filename if c in validChars)
if len(filename) == 0:
filename = 'layer'
destFilename = getTempFilenameInTempFolder(filename + '.sgrd')
destFilename = QgsProcessingUtils.generateTempFilename(filename + '.sgrd')
self.exportedLayers[source] = destFilename
sessionExportedLayers[source] = destFilename
return 'io_gdal 0 -TRANSFORM 1 -RESAMPLING 3 -GRIDS "' + destFilename + '" -FILES "' + source + '"'
Expand Down
12 changes: 6 additions & 6 deletions python/plugins/processing/core/outputs.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from qgis.PyQt.QtCore import QCoreApplication

from processing.core.ProcessingConfig import ProcessingConfig
from processing.tools.system import isWindows, getTempFilenameInTempFolder, getTempDirInTempFolder
from processing.tools.system import isWindows, getTempDirInTempFolder
from processing.tools.vector import TableWriter, NOGEOMETRY_EXTENSIONS
from processing.tools import dataobjects

Expand Down Expand Up @@ -93,7 +93,7 @@ def setValue(self, value):

def _resolveTemporary(self, alg):
ext = self.getDefaultFileExtension()
return getTempFilenameInTempFolder(self.name + '.' + ext)
return QgsProcessingUtils.generateTempFilename(self.name + '.' + ext)

def _supportedExtensions(self):
return []
Expand Down Expand Up @@ -225,7 +225,7 @@ def getCompatibleFileName(self, alg):
supported = alg.provider().supportedOutputRasterLayerExtensions()
default = ProcessingConfig.getSetting(ProcessingConfig.DEFAULT_OUTPUT_RASTER_LAYER_EXT, True)
ext = default if default in supported else supported[0]
self.compatible = getTempFilenameInTempFolder(self.name + '.' + ext)
self.compatible = QgsProcessingUtils.generateTempFilename(self.name + '.' + ext)
return self.compatible


Expand Down Expand Up @@ -264,7 +264,7 @@ def getCompatibleFileName(self, alg):
return self.value
else:
if self.compatible is None:
self.compatible = getTempFilenameInTempFolder(
self.compatible = QgsProcessingUtils.generateTempFilename(
self.name + '.' + alg.provider().supportedOutputTableExtensions()[0])
return self.compatible

Expand Down Expand Up @@ -342,7 +342,7 @@ def getCompatibleFileName(self, alg):
default = self.getDefaultFileExtension()
supported = alg.provider().supportedOutputVectorLayerExtensions()
ext = default if default in supported else supported[0]
self.compatible = getTempFilenameInTempFolder(self.name + '.' + ext)
self.compatible = QgsProcessingUtils.generateTempFilename(self.name + '.' + ext)
return self.compatible

def getVectorWriter(self, fields, geomType, crs, context):
Expand Down Expand Up @@ -381,7 +381,7 @@ def _resolveTemporary(self, alg):
return "memory:"
else:
ext = self.getDefaultFileExtension()
return getTempFilenameInTempFolder(self.name + '.' + ext)
return QgsProcessingUtils.generateTempFilename(self.name + '.' + ext)


def getOutputFromString(s):
Expand Down
8 changes: 3 additions & 5 deletions python/plugins/processing/tools/dataobjects.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,8 @@

from processing.core.ProcessingConfig import ProcessingConfig
from processing.algs.gdal.GdalUtils import GdalUtils
from processing.tools.system import (getTempFilenameInTempFolder,
getTempFilename,
removeInvalidChars,
isWindows)
from processing.tools.system import (getTempFilename,
removeInvalidChars)
from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException

ALL_TYPES = [-1]
Expand Down Expand Up @@ -198,7 +196,7 @@ def exportVectorLayer(layer, supported=None):
if basename:
if not basename.endswith("shp"):
basename = os.path.splitext(basename)[0] + ".shp"
output = getTempFilenameInTempFolder(basename)
output = QgsProcessingUtils.generateTempFilename(basename)
else:
output = getTempFilename("shp")
useSelection = False # TODO ProcessingConfig.getSetting(ProcessingConfig.USE_SELECTED)
Expand Down
31 changes: 4 additions & 27 deletions python/plugins/processing/tools/system.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@
import math

from qgis.PyQt.QtCore import QDir
from qgis.core import QgsApplication
from qgis.core import (QgsApplication,
QgsProcessingUtils)

numExported = 1

Expand Down Expand Up @@ -63,19 +64,8 @@ def isMac():
return sys.platform == 'darwin'


_tempFolderSuffix = uuid.uuid4().hex


def tempFolder():
tempDir = os.path.join(str(QDir.tempPath()), 'processing' + _tempFolderSuffix)
if not QDir(tempDir).exists():
QDir().mkpath(tempDir)

return str(os.path.abspath(tempDir))


def getTempFilename(ext=None):
tmpPath = tempFolder()
tmpPath = QgsProcessingUtils.tempFolder()
t = time.time()
m = math.floor(t)
uid = '{:8x}{:05x}'.format(m, int((t - m) * 1000000))
Expand All @@ -86,24 +76,11 @@ def getTempFilename(ext=None):
return filename


def getTempFilenameInTempFolder(basename):
"""Returns a temporary filename for a given file, putting it into
a temp folder but not changing its basename.
"""

path = tempFolder()
path = os.path.join(path, uuid.uuid4().hex)
mkdir(path)
basename = removeInvalidChars(basename)
filename = os.path.join(path, basename)
return filename


def getTempDirInTempFolder():
"""Returns a temporary directory, putting it into a temp folder.
"""

path = tempFolder()
path = QgsProcessingUtils.tempFolder()
path = os.path.join(path, uuid.uuid4().hex)
mkdir(path)
return path
Expand Down
28 changes: 28 additions & 0 deletions src/core/processing/qgsprocessingutils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,34 @@ QVariant QgsProcessingUtils::generateIteratingDestination( const QVariant &input
}
}

QString QgsProcessingUtils::tempFolder()
{
static QString sFolder;
static QMutex sMutex;
sMutex.lock();
if ( sFolder.isEmpty() )
{
QString subPath = QUuid::createUuid().toString();
sFolder = QDir::tempPath() + QStringLiteral( "/processing_" ) + subPath;
if ( !QDir( sFolder ).exists() )
QDir().mkpath( sFolder );
}
sMutex.unlock();
return sFolder;
}

QString QgsProcessingUtils::generateTempFilename( const QString &basename )
{
QString subPath = QUuid::createUuid().toString();
QString path = tempFolder() + '/' + subPath;
if ( !QDir( path ).exists() ) //make sure the directory exists - it shouldn't, but lets be safe...
{
QDir tmpDir( QDir::tempPath() );
tmpDir.mkdir( subPath );
}
return path + '/' + basename;
}


//
// QgsProcessingFeatureSource
Expand Down
14 changes: 14 additions & 0 deletions src/core/processing/qgsprocessingutils.h
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,20 @@ class CORE_EXPORT QgsProcessingUtils
*/
static QVariant generateIteratingDestination( const QVariant &input, const QVariant &id, QgsProcessingContext &context );

/**
* Returns a session specific processing temporary folder for use in processing algorithms.
* \see generateTempFileName()
*/
static QString tempFolder();

/**
* Returns a temporary filename for a given file, putting it into
* a temporary folder (creating that folder in the process),
* but not changing the \a basename.
* \see tempFolder()
*/
static QString generateTempFilename( const QString &basename );

private:

static bool canUseLayer( const QgsRasterLayer *layer );
Expand Down
18 changes: 18 additions & 0 deletions tests/src/core/testqgsprocessing.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ class TestQgsProcessing: public QObject
void generateIteratingDestination();
void asPythonCommand();
void modelerAlgorithm();
void tempUtils();

private:

Expand Down Expand Up @@ -3830,5 +3831,22 @@ void TestQgsProcessing::modelerAlgorithm()
QCOMPARE( alg7.outputDefinitions().at( 0 )->description(), QStringLiteral( "my output2" ) );
}

void TestQgsProcessing::tempUtils()
{
QString tempFolder = QgsProcessingUtils::tempFolder();
// tempFolder should remain constant for session
QCOMPARE( QgsProcessingUtils::tempFolder(), tempFolder );

QString tempFile1 = QgsProcessingUtils::generateTempFilename( "test.txt" );
QVERIFY( tempFile1.endsWith( "test.txt" ) );
QVERIFY( tempFile1.startsWith( tempFolder ) );

// expect a different file
QString tempFile2 = QgsProcessingUtils::generateTempFilename( "test.txt" );
QVERIFY( tempFile1 != tempFile2 );
QVERIFY( tempFile2.endsWith( "test.txt" ) );
QVERIFY( tempFile2.startsWith( tempFolder ) );
}

QGSTEST_MAIN( TestQgsProcessing )
#include "testqgsprocessing.moc"

0 comments on commit a87ca09

Please sign in to comment.