diff --git a/avaframe/com3Hybrid/com3Hybrid.py b/avaframe/com3Hybrid/com3Hybrid.py index a57eb4ad0..577f0f54b 100644 --- a/avaframe/com3Hybrid/com3Hybrid.py +++ b/avaframe/com3Hybrid/com3Hybrid.py @@ -14,12 +14,16 @@ # import config and init tools import avaframe.in2Trans.shpConversion as shpConv from avaframe.in3Utils import cfgUtils +from avaframe.in3Utils import cfgHandling +from avaframe.in3Utils import fileHandlerUtils as fU from avaframe.in1Data import getInput +import avaframe.in3Utils.initializeProject as initProj # import computation modules from avaframe.com1DFA import com1DFA, particleTools import avaframe.ana5Utils.DFAPathGeneration as DFAPath from avaframe.com2AB import com2AB +from avaframe.com3Hybrid import com3Hybrid # import analysis tools from avaframe.out3Plot import outAB @@ -41,11 +45,32 @@ def maincom3Hybrid(cfgMain, cfgHybrid): """ avalancheDir = cfgMain['MAIN']['avalancheDir'] demOri = getInput.readDEM(avalancheDir) - # get comDFA configuration path for hybrid model - hybridModelDFACfg = pathlib.Path('com3Hybrid', 'hybridModel_com1DFACfg.ini') + + # setup outputs folder and work folder + workPath = pathlib.Path(avalancheDir, 'Work', 'com3Hybrid') + oPath = pathlib.Path(avalancheDir, 'Outputs', 'com3Hybrid') + fU.makeADir(workPath) + fU.makeADir(oPath) + + # ++++++++++ set configurations for all the used modules and override ++++++++++++ + # get comDFA configuration and save to file + com1DFACfg = cfgUtils.getModuleConfig(com1DFA, fileOverride='', modInfo=False, toPrint=False, + onlyDefault=cfgHybrid['com1DFA_override']['defaultConfig']) + com1DFACfg, cfgHybrid = cfgHandling.applyCfgOverride(com1DFACfg, cfgHybrid, com1DFA, addModValues=False) + com1DFACfgFile = cfgUtils.writeCfgFile(avalancheDir, com1DFA, com1DFACfg, fileName='com1DFA_settings', filePath=workPath) + + # fetch configuration for DFAPathGeneration + DFAPathGenerationCfg= cfgUtils.getModuleConfig(DFAPath, fileOverride='', modInfo=False, toPrint=False, + onlyDefault=cfgHybrid['DFAPathGeneration_override']['defaultConfig']) + DFAPathGenerationCfg, cfgHybrid = cfgHandling.applyCfgOverride(DFAPathGenerationCfg, cfgHybrid, DFAPath, addModValues=False) + + # first create configuration object for com2AB + com2ABCfg = cfgUtils.getModuleConfig(com2AB, fileOverride='', modInfo=False, toPrint=False, + onlyDefault=cfgHybrid['com1DFA_override']['defaultConfig']) + com2ABCfg, cfgHybrid = cfgHandling.applyCfgOverride(com2ABCfg, cfgHybrid, com2AB, addModValues=False) # get initial mu value - muArray = np.array([cfgHybrid.getfloat('DFA', 'mu')]) + muArray = np.array([cfgHybrid.getfloat('com1DFA_override', 'mu')]) alphaArray = np.array([np.degrees(np.arctan(muArray[0]))]) # prepare for iterating @@ -54,21 +79,27 @@ def maincom3Hybrid(cfgMain, cfgHybrid): iterate = True resultsHybrid = {} while iteration < nIterMax and iterate: - # update the com1DFA mu value + # update the com1DFA mu value in configuration file updater = ConfigUpdater() - updater.read(hybridModelDFACfg) + updater.read(com1DFACfgFile) updater['GENERAL']['mu'].value = ('%.4f' % muArray[-1]) updater.update_file() + log.info('Mu is set to: %f' % muArray[-1]) + # ++++++++++ RUN COM1DFA +++++++++++ # Run dense flow with coulomb friction - dem, _, _, simDF = com1DFA.com1DFAMain(avalancheDir, cfgMain, cfgFile=hybridModelDFACfg) + # Clean input directory of old work and output files from module + initProj.cleanModuleFiles(avalancheDir, com1DFA, deleteOutput=False) + dem, _, _, simDF = com1DFA.com1DFAMain(avalancheDir, cfgMain, cfgFile=com1DFACfgFile) simID = simDF.index[0] particlesList, timeStepInfo = particleTools.readPartFromPickle(avalancheDir, simName=simID, flagAvaDir=True, comModule='com1DFA') + + # ++++++++++ GENERATE PATH +++++++++++ # postprocess to extract path and energy line avaProfileMass = DFAPath.getDFAPathFromPart(particlesList, addVelocityInfo=True) # make a copy because extendDFAPathKernel might modify avaProfileMass - avaProfileMassExt = DFAPath.extendDFAPath(cfgHybrid['PATH'], avaProfileMass.copy(), dem, particlesList[0]) + avaProfileMassExt = DFAPath.extendDFAPath(DFAPathGenerationCfg['PATH'], avaProfileMass.copy(), dem, particlesList[0]) avaProfileMassExtOri = copy.deepcopy(avaProfileMassExt) avaProfileMassExtOri['x'] = avaProfileMassExtOri['x'] + demOri['header']['xllcenter'] avaProfileMassExtOri['y'] = avaProfileMassExtOri['y'] + demOri['header']['yllcenter'] @@ -77,15 +108,13 @@ def maincom3Hybrid(cfgMain, cfgHybrid): name = 'massAvaPath' shpConv.writeLine2SHPfile(avaProfileMassExtOri, name, pathAB) - # Run Alpha Beta - hybridModelABCfg = pathlib.Path('com3Hybrid', 'hybridModel_com2ABCfg.ini') - cfgAB = cfgUtils.getModuleConfig(com2AB, fileOverride=hybridModelABCfg) - cfgAB['ABSETUP']['path2Line'] = str(pathAB) + '.shp' + # ++++++++++ RUN COM2AB +++++++++++ + com2ABCfg['ABSETUP']['path2Line'] = str(pathAB) + '.shp' # take the path extracted from the DFA model as input - pathDict, demAB, splitPoint, eqParams, resAB = com2AB.com2ABMain(cfgAB, avalancheDir) + pathDict, demAB, splitPoint, eqParams, resAB = com2AB.com2ABMain(com2ABCfg, avalancheDir) # make AB plot reportDictList = [] - _, plotFile, writeFile = outAB.writeABpostOut(pathDict, demAB, splitPoint, eqParams, resAB, cfgAB, reportDictList) + _, plotFile, writeFile = outAB.writeABpostOut(pathDict, demAB, splitPoint, eqParams, resAB, com2ABCfg, reportDictList) # make custom com3 profile plot alpha = resAB[name]['alpha'] @@ -113,6 +142,9 @@ def maincom3Hybrid(cfgMain, cfgHybrid): outCom3Plots.hybridProfilePlot(avalancheDir, resultsHybrid) outCom3Plots.hybridPathPlot(avalancheDir, dem, resultsHybrid, fields[0], particlesList[-1], muArray) + # write com3HYbrid configuration settings to file for reproduction use + com3HybridCfgFile = cfgUtils.writeCfgFile(avalancheDir, com3Hybrid, cfgHybrid, fileName='com3Hybrid_settings', filePath=oPath) + log.debug('Alpha array is %s' % alphaArray) log.debug('mu array is %s' % muArray) diff --git a/avaframe/com3Hybrid/com3HybridCfg.ini b/avaframe/com3Hybrid/com3HybridCfg.ini index 5bcd2e37a..f4f6c5124 100644 --- a/avaframe/com3Hybrid/com3HybridCfg.ini +++ b/avaframe/com3Hybrid/com3HybridCfg.ini @@ -9,29 +9,99 @@ nIterMax = 4 alphaThreshold = 0.1 -# settings for path finding (these setting are used for path generation, as set in ana5Utils/DFAPathGenerationCfg.ini) -[PATH] +[DFAPathGeneration_override] +# use default DFAPathGeneration config as base configuration (True) and override following parameters +# if False and local is available use local +defaultConfig = True # the path extracted from the com1DFA simulation is re-sampled # re-sampling step size is defined resampleDistance = nCellsResample x cellSize) # this results in a path with a horizontal distance between points <= resampleDistance nCellsResample = 5 -# extension method at the top -# option 0: take the highest particle in the release -# option 1: find the point that will lead to the longest runout -extTopOption = 1 - # for extending the path at the bottom, extend path towards the bottom of the runout in the # direction extracted form the first/last points of the path (all points at a distance # nCellsMinExtend x cellSize < distance < nCellsMaxExtend x cellSize from the start/end) nCellsMinExtend = 2 -nCellsMaxExtend = 20 # for the extrapolation at the bottom, add factBottomExt * sMax to the path factBottomExt = 0.2 maxIterationExtBot = 10 nBottomExtPrecision = 10 -[DFA] -# starting value for bottom bed friction +[com1DFA_override] +# use default com1DFA config as base configuration (True) and override following parameters +# if False and local is available use local +defaultConfig = True +#++++++++++++++++ Simulation type +# list of simulations that shall be performed (null, ent, res, entres, available (use all available input data)) +simTypeList = null + +#+++++++++++++ Output++++++++++++ +# desired result Parameters (ppr, pft, pfv, pta, FT, FV, P, Vx, Vy, Vz, TA, particles) - separated by | +resType = particles +# saving time step, i.e. time in seconds (first and last time step are always saved) +# option 1: give an interval with start:interval in seconds (tStep = 0:5 - this will save desired results every 5 seconds for the full simulation) +# option 2: explicitly list all desired time steps (closest to actual computational time step) separated by | (example tSteps = 1|50.2|100) +# NOTE: initial and last time step are always saved! +tSteps = 0:5 + +#+++++Release thickness++++ +# True if release thickness should be read from shapefile file; if False - relTh read from ini file +relThFromShp = False +# release thickness (only considered if relThFromShp=False) +relTh = 1 +#+++++Entrainment thickness++++ +# True if entrainment thickness should be read from shapefile file; if False - entTh read from ini file +entThFromShp = True +# entrainment thickness (only considered if entThFromShp=False) +entTh = + +#++++++++++++Time stepping parameters +# to use a variable time step (time step depends on kernel radius) +sphKernelRadiusTimeStepping = True +# courant number if option cflTimeStepping is chosen. +# Upper time step limit coefficient if option sphKernelRadiusTimeStepping is chosen. +cMax = 0.02 +# stopCriterion (based on massFlowing or kinEnergy) +stopCritType = massFlowing +# if based on massFlowing, specify the velocity threshold for flowing mass (m/s) +uFlowingThreshold = 0.0 + +#+++++++++++++SPH parameters +# SPH gradient option +# 0) No pressure gradients computed +# 1) SamosAT style (no reprojecion on the surface, dz = 0 and gz is used) +# 2) SamosAT done in the cartesian coord system (reprojecion on the surface, dz != 0 and g3 is used) +# 3) SamosAT but done in the local coord system (will hopefully allow us to add the earth pressure coef) +# and this time reprojecion on the surface, dz not 0 and g3 is used +sphOption = 2 + +#++++++++++++++++ Particles +# number of particles defined by: MPPDIR= mass per particle direct, MPPDH= mass per particles through release thickness, +# MPPKR= mass per particles through number of particles per kernel radius +massPerParticleDeterminationMethod = MPPKR + +#+++++++++++++Flow model parameters+++++++++ +# curvature acceleration coefficient +# take curvature term into account in the gravity acceleration term +# 0 if deactivated, 1 if activated +curvAcceleration = 0 + +#++++++++++++Friction model +# add the friction using an explicit formulation (1) +# 0 use an implicit method +explicitFriction = 1 +# friction type (samosAT, Coulomb, Voellmy) +frictModel = Coulomb +#+++++++++++++General Friction parameters +# tan of bed friction angle used for: samosAT, Coulomb, Voellmy mu = 0.4 + +# which result parameters shall be included as plots in report, - separated by | +plotFields = ppr|pft|pfv|TA|pta + + +[com2AB_override] +# use default com2AB config as base configuration (True) and override following parameters +# if False and local is available use local +defaultConfig = True diff --git a/avaframe/com3Hybrid/hybridModel_com1DFACfg.ini b/avaframe/com3Hybrid/hybridModel_com1DFACfg.ini deleted file mode 100644 index 5772d1bc3..000000000 --- a/avaframe/com3Hybrid/hybridModel_com1DFACfg.ini +++ /dev/null @@ -1,103 +0,0 @@ -### Config File - This file is used as configuration file for running com1DFA -## It replaces the local_com1DFACfg.ini - -[GENERAL] -#++++++++++++++++ Simulation type -# list of simulations that shall be performed (null, ent, res, entres, available (use all available input data)) -simTypeList = null - -#+++++++++++++ Output++++++++++++ -# desired result Parameters (ppr, pft, pfv, pta, FT, FV, P, Vx, Vy, Vz, TA, particles) - separated by | -resType = particles -# saving time step, i.e. time in seconds (first and last time step are always saved) -# option 1: give an interval with start:interval in seconds (tStep = 0:5 - this will save desired results every 5 seconds for the full simulation) -# option 2: explicitly list all desired time steps (closest to actual computational time step) separated by | (example tSteps = 1|50.2|100) -# NOTE: initial and last time step are always saved! -tSteps = 0:5 - -#+++++Release thickness++++ -# True if release thickness should be read from shapefile file; if False - relTh read from ini file -relThFromShp = False -# release thickness (only considered if relThFromShp=False) -relTh = 1 -#+++++Entrainment thickness++++ -# True if entrainment thickness should be read from shapefile file; if False - entTh read from ini file -entThFromShp = True -# entrainment thickness (only considered if entThFromShp=False) -entTh = - -#++++++++++++Time stepping parameters -# End time [s] -tEnd = 400 -# to use a variable time step (time step depends on kernel radius) -sphKernelRadiusTimeStepping = True -# courant number if option cflTimeStepping is chosen. -# Upper time step limit coefficient if option sphKernelRadiusTimeStepping is chosen. -cMax = 0.02 -# stopCriterion (based on massFlowing or kinEnergy) -stopCritType = massFlowing -# if based on massFlowing, specify the velocity threshold for flowing mass (m/s) -uFlowingThreshold = 0.0 -# stopCriterion (stops when massFlowing<0.01*peakMassFlowing or ke<0.01*pke) -stopCrit = 0.01 - -#+++++++++++++SPH parameters -# SPH gradient option -# 0) No pressure gradients computed -# 1) SamosAT style (no reprojecion on the surface, dz = 0 and gz is used) -# 2) SamosAT done in the cartesian coord system (reprojecion on the surface, dz != 0 and g3 is used) -# 3) SamosAT but done in the local coord system (will hopefully allow us to add the earth pressure coef) -# and this time reprojecion on the surface, dz not 0 and g3 is used -sphOption = 2 -# sph kernel smoothing length [m] -sphKernelRadius = 5 -# Choice of artificial viscosity -# 0) No artificial viscosity -# 1) SAMOS artificial viscosity -# 2) Ata artificial viscosity -viscOption = 1 - -#++++++++++++++++ Particles -# number of particles defined by: MPPDIR= mass per particle direct, MPPDH= mass per particles through release thickness, -# MPPKR= mass per particles through number of particles per kernel radius -massPerParticleDeterminationMethod = MPPKR -# is computed with: nPPK = nPPK0 * (sphKR/sphKR0)^aPPK -# where sphKR is the sphKernelRadius specified further up -# reference kernel radius -sphKR0 = 5 -# reference number of particles per kernel radius -nPPK0 = 15 -# variation of nppK exponent -aPPK = -1.5 - - -#+++++++++++++Mesh and interpolation -# remesh the input DEM -# expected mesh size [m] -meshCellSize = 5 - -#+++++++++++++Flow model parameters+++++++++ -# subgridMixingFactor -subgridMixingFactor = 100. -# curvature acceleration coefficient -# take curvature term into account in the gravity acceleration term -# 0 if deactivated, 1 if activated -curvAcceleration = 0 - -#++++++++++++Friction model -# add the friction using an explicit formulation (1) -# 0 use an implicit method -explicitFriction = 1 -# friction type (samosAT, Coulomb, Voellmy) -frictModel = Coulomb -#+++++++++++++General Friction parameters -# tan of bed friction angle used for: samosAT, Coulomb, Voellmy -mu = 0.6073 - -[REPORT] -# which result parameters shall be included as plots in report, - separated by | -plotFields = ppr|pft|pfv|TA|pta -# units for output variables -unitppr = kPa -unitpft = m -unitpfv = ms-1 diff --git a/avaframe/com3Hybrid/hybridModel_com2ABCfg.ini b/avaframe/com3Hybrid/hybridModel_com2ABCfg.ini deleted file mode 100644 index 5101cf823..000000000 --- a/avaframe/com3Hybrid/hybridModel_com2ABCfg.ini +++ /dev/null @@ -1,14 +0,0 @@ -### Config File - This file is used as configuration file for running com2AB -## It replaces the local_com2ABCfg.ini - -# Optional settings------------------------------- -[ABSETUP] - # if custom avalanche set is wanted switch to True and specify your parameters -customParam = False - - # resampling step [m] -distance = 10 - -# threshold distance [m]. When looking for the beta point make sure at least -# dsMin meters after the beta point also have an angle bellow 10° -dsMin = 30 diff --git a/avaframe/in3Utils/cfgHandling.py b/avaframe/in3Utils/cfgHandling.py index 1b48935c9..eb401b9d0 100644 --- a/avaframe/in3Utils/cfgHandling.py +++ b/avaframe/in3Utils/cfgHandling.py @@ -6,6 +6,7 @@ import logging import numpy as np +import pathlib # Local imports from avaframe.in3Utils import cfgUtils @@ -66,7 +67,7 @@ def addInfoToSimName(avalancheDir, csvString=''): csvString: comma separated list with parameter names, as found in com1DFA ini file eg. 'mu,tau0,tEnd' - + Returns -------- simDF: dataframe @@ -364,3 +365,66 @@ def filterCom1DFAThicknessValues(key, value, simDF): log.info('simulations for %s found with values: %s' % (key, simDF[allThNames])) return simDF + + +def applyCfgOverride(cfgToOverride, cfgWithOverrideParameters, module, addModValues=False): + """ override configuration parameter values with the values provided in cfgWithOverrideParameters[modName_override] + if addModValues True update the cfgWithOverrideParameters with the values for all parameters that are not + provided in the override parameters + + Parameters + ---------- + cfgToOverride: configparer object + configuration of module of interest + cfgWithOverrideParameters: configparser object + full configuration settings containing a section modName_override with parameter values + that should be overriden in the cfgToOverride + module + module of the cfgToOverride configuration + addModValues: bool + if True add all parameters from cfgToOverride module to cfgWithOverrideParameters override + section + + Returns + -------- + cfgToOverride: configparser object + updated configuration of module + cfgWithOverrideParameters: configparser object + updated configuration of module + """ + + # get path of module + modPath = pathlib.Path(module.__file__).resolve().parent + + # get filename of module + modName = str(pathlib.Path(module.__file__).stem) + + # create list with parameters that become overridden + overrideParameters = cfgWithOverrideParameters['%s_override' % modName] + overrideKeys = [item for item in overrideParameters] + overrideKeys.remove('defaultConfig') + + # loop through sections of the configuration of the module + foundKeys = [] + for section in cfgToOverride.sections(): + for key in overrideKeys: + if cfgToOverride.has_option(section, key): + cfgToOverride.set(section, key, overrideParameters[key]) + log.info('Override %s parameter: %s in section: %s with %s' % (modName, key, section, str(overrideParameters[key]))) + foundKeys.append(key) + if addModValues: + for section in cfgToOverride.sections(): + for key in cfgToOverride[section]: + if key not in overrideKeys: + # if no override value is provided add actual configuration parameter to override section + # useful for reproduction if onlyDefault = False and modName config was read from local + cfgWithOverrideParameters['%s_override' % modName][key] = cfgToOverride[section][key] + log.debug('Added %s: %s to override parameters ' % (key, cfgToOverride[section][key])) + + # log warning if parameter in override was not found in modName configuration + notOverride = set(foundKeys).symmetric_difference(set(overrideKeys)) + for item in notOverride: + if item != 'defaultConfig': + log.warning('Additional Key [\'%s\'] in section %s_override is ignored.' % (item, modName)) + + return cfgToOverride, cfgWithOverrideParameters diff --git a/avaframe/in3Utils/cfgUtils.py b/avaframe/in3Utils/cfgUtils.py index 44cde1254..3ef383a22 100644 --- a/avaframe/in3Utils/cfgUtils.py +++ b/avaframe/in3Utils/cfgUtils.py @@ -240,9 +240,9 @@ def compareConfig(iniFile, modName, compare, modInfo=False, toPrint=True): return cfg, modDict -def writeCfgFile(avaDir, module, cfg, fileName=''): - """ Save configuration used to text file in Outputs as moduleName_settings.ini - or optional in Outputs/moduleName/configurationFiles/filenName.ini +def writeCfgFile(avaDir, module, cfg, fileName='', filePath=''): + """ Save configuration used to text file in Outputs/moduleName/configurationFiles/modName.ini + or optional to filePath and with fileName Parameters ----------- @@ -254,6 +254,8 @@ def writeCfgFile(avaDir, module, cfg, fileName=''): configuration settings fileName: str name of saved configuration file - optional + filePath: str or pathlib path + path where file should be saved to except file name - optional """ @@ -261,20 +263,28 @@ def writeCfgFile(avaDir, module, cfg, fileName=''): name = pathlib.Path(module.__file__).name modName = name.split('.')[0] - # write to file - if fileName != '': - # set outputs + # set outputs + if filePath == '': outDir = pathlib.Path(avaDir, 'Outputs', modName, 'configurationFiles') fU.makeADir(outDir) - cfg.optionxform = str - with open(pathlib.Path(outDir, '%s.ini' % (fileName)), 'w') as conf: - cfg.write(conf) else: - # set outputs - outDir = pathlib.Path(avaDir, 'Outputs') - cfg.optionxform = str - with open(pathlib.Path(outDir, '%s_settings.ini' % (modName)), 'w') as conf: - cfg.write(conf) + if filePath.is_dir(): + outDir = pathlib.Path(filePath) + else: + message = '%s is not a valid location for saving cfg file' % str(filePath) + log.error(message) + raise NotADirectoryError(message) + + # set path to file + if fileName == '': + fileName = modName + pathToFile = pathlib.Path(outDir, '%s.ini' % (fileName)) + + # write file + with open(pathToFile, 'w') as conf: + cfg.write(conf) + + return pathToFile def readCfgFile(avaDir, module='', fileName=''): diff --git a/avaframe/in3Utils/initialiseDirs.py b/avaframe/in3Utils/initialiseDirs.py index 1122aa1ab..4d60996d4 100644 --- a/avaframe/in3Utils/initialiseDirs.py +++ b/avaframe/in3Utils/initialiseDirs.py @@ -41,7 +41,9 @@ def initialiseRunDirs(avaDir, modName, cleanDEMremeshed): workDir = pathlib.Path(avaDir, 'Work', modName) # If Work directory already exists - error if workDir.is_dir(): - log.error('Work directory %s already exists - delete first!' % (workDir)) + message = 'Work directory %s already exists - delete first!' % (workDir) + log.error(message) + raise AssertionError(message) else: workDir.mkdir(parents=True, exist_ok=False) log.debug('Directory: %s created' % workDir) diff --git a/avaframe/tests/test_cfgHandling.py b/avaframe/tests/test_cfgHandling.py new file mode 100644 index 000000000..63f10dab4 --- /dev/null +++ b/avaframe/tests/test_cfgHandling.py @@ -0,0 +1,196 @@ +"""Tests for module cfgHandling""" + + +import pathlib +import pytest +import configparser +import logging + +from avaframe.in3Utils import cfgUtils +from avaframe.in3Utils import cfgHandling +from avaframe.tests import test_logUtils +from avaframe.com1DFA import com1DFA + +log = logging.getLogger(__name__) + + +def test_addInfoToSimName(): + '''Test for addInfoToSimname''' + avaTestDir = 'avaParabolaStatsTest' + dirPath = pathlib.Path(__file__).parents[0] + avaDir = dirPath / '..' / '..' / 'benchmarks' / avaTestDir + renameDF = cfgHandling.addInfoToSimName(avaDir,'mu') + assert renameDF.loc['c4f3a000c3']['newName'] == 'release1PF_c4f3a000c3_mu_0.155_null_dfa' + renameDF = cfgHandling.addInfoToSimName(avaDir,'mu,tEnd') + assert renameDF.loc['c4f3a000c3']['newName'] == 'release1PF_c4f3a000c3_mu_0.155_tEnd_400_null_dfa' + + +def test_orderSimFiles(): + """ test generating order of simulation results """ + + avaTestDir = 'avaHockeyChannelPytest' + dirPath = pathlib.Path(__file__).parents[0] + avaDir = dirPath / '..' / '..' / 'benchmarks' / avaTestDir + + varParList = 'releaseScenario' + + simDF = cfgUtils.createConfigurationInfo(avaDir, specDir='') + + varParList, simDF = cfgHandling.orderSimulations(varParList, True, simDF) + + assert simDF['simName'][0] == 'release1HS_d10bdc1e81_ent_dfa' + + varParList = 'releaseSenario' + message = ('Choose a valid parameter for sorting the simulations. \'releaseSenario\' is not valid.') + with pytest.raises(KeyError) as e: + assert cfgHandling.orderSimulations(varParList, True, simDF) + assert message in str(e.value) + + +def test_fetchAndOrderSimFiles(): + """ test generating order of simulation results """ + + avaTestDir = 'avaHockeyChannelPytest' + dirPath = pathlib.Path(__file__).parents[0] + avaDir = dirPath / '..' / '..' / 'benchmarks' / avaTestDir + inputDir = avaDir / 'Outputs' / 'com1DFA' / 'peakFiles' + + varParList = 'releaseScenario' + + simDF = cfgHandling.fetchAndOrderSimFiles(avaDir, inputDir, varParList, True, specDir='', resFiles=True) + + assert simDF['simName'][0] == 'release1HS_d10bdc1e81_ent_dfa' + + varParList = 'releaseSenario' + message = ('Choose a valid parameter for sorting the simulations. \'releaseSenario\' is not valid.') + with pytest.raises(KeyError) as e: + simDF = cfgHandling.fetchAndOrderSimFiles(avaDir, 'inputDir', varParList, True, specDir='', resFiles=False) + assert message in str(e.value) + + +def test_filterSims(tmp_path): + """ test filtering of simulations using configuration files """ + + avaTestDir = 'avaHockeyChannelPytest' + dirPath = pathlib.Path(__file__).parents[0] + avaDir = dirPath / '..' / '..' / 'benchmarks' / avaTestDir + avaDir2 = dirPath / 'data' / 'avaFilterTest' / 'com1DFA' + + parametersDict = {'releaseScenario': 'release1HS'} + + simNames = cfgHandling.filterSims(avaDir, parametersDict, specDir='') + + testRel = False + testRel2 = False + if 'release1HS' in simNames[0]: + testRel = True + if 'release2HS' in simNames[0]: + testRel2 = True + + assert testRel + assert testRel2 is False + assert len(simNames) == 1 + + parametersDict = {'relTh': 1.} + simNames2 = cfgHandling.filterSims(avaDir, parametersDict, specDir='') + simNames2 = sorted(simNames2) + assert len(simNames2) == 2 + assert simNames2[0] == 'release1HS_d10bdc1e81_ent_dfa' + assert simNames2[1] == 'release2HS_e2145362b7_ent_dfa' + + parametersDict = {'~relTh': 1.} + simNames3 = cfgHandling.filterSims(avaDir, parametersDict, specDir='') + + assert len(simNames3) == 0 + + parametersDict = {'~releaseScenario': 'release1HS'} + simNames4 = cfgHandling.filterSims(avaDir, parametersDict, specDir='') + + assert len(simNames4) == 1 + assert simNames4[0] == 'release2HS_e2145362b7_ent_dfa' + + parametersDict = {'relTh': 1.} + + simNames = cfgHandling.filterSims(avaDir2, parametersDict, specDir=avaDir2) + simNames = sorted(simNames) + + assert len(simNames) == 2 + assert simNames == ['relGar_6f35cbd808_null_dfa', 'relGar_b9b17dd019_ent_dfa'] + + parametersDict = {'relTh': [1., 1.5]} + + simNames = cfgHandling.filterSims(avaDir2, parametersDict, specDir=avaDir2) + simNames = sorted(simNames) + + assert len(simNames) == 4 + assert simNames == ['relGar_1022880a70_null_dfa', 'relGar_6f35cbd808_null_dfa', + 'relGar_b9b17dd019_ent_dfa', 'relGar_c4337e50ac_null_dfa'] + + +def test_applyCfgOverride(caplog): + """ test overriding cfg parameters in a cfg object from another cfg with an override section """ + + cfgWithOverrideParameters = configparser.ConfigParser() + cfgWithOverrideParameters.optionxform = str + cfgWithOverrideParameters['GENERAL'] = {'testp1': 1., 'testp2': 'testValue'} + cfgWithOverrideParameters['com1DFA_override'] = {'defaultConfig': True, 'mu': 0.7, 'tStep': 100., 'notParameter': 1.} + + cfgToOverride = configparser.ConfigParser() + cfgToOverride.optionxform = str + cfgToOverride['GENERAL'] = {'mu': 1., 'tEnd': 400., 'relTh': 1.} + cfgToOverride['DFA'] = {'tStep': 0.3, 'dt1': 6., 'flowF': 3.} + + cfgToOverride, cfgWithOverrideParameters = cfgHandling.applyCfgOverride(cfgToOverride, cfgWithOverrideParameters, com1DFA, addModValues=False) + + assert cfgToOverride['GENERAL']['mu'] == '0.7' + assert cfgToOverride['GENERAL']['tEnd'] == '400.0' + assert cfgToOverride['GENERAL']['relTh'] == '1.0' + assert cfgToOverride['DFA']['tStep'] == '100.0' + assert cfgToOverride['DFA']['dt1'] == '6.0' + assert cfgToOverride['DFA']['flowF'] == '3.0' + assert cfgWithOverrideParameters['GENERAL']['testp1'] == '1.0' + assert cfgWithOverrideParameters['GENERAL']['testp2'] == 'testValue' + assert cfgWithOverrideParameters['com1DFA_override']['defaultConfig'] == 'True' + assert cfgWithOverrideParameters['com1DFA_override']['mu'] == '0.7' + assert cfgWithOverrideParameters['com1DFA_override']['tStep'] == '100.0' + assert cfgWithOverrideParameters['com1DFA_override']['notParameter'] == '1.0' + assert len(cfgWithOverrideParameters.items('GENERAL')) == 2 + assert len(cfgWithOverrideParameters.items('com1DFA_override')) == 4 + assert len(cfgToOverride.items('GENERAL')) == 3 + assert len(cfgToOverride.items('GENERAL')) == 3 + + + cfgWithOverrideParameters = configparser.ConfigParser() + cfgWithOverrideParameters.optionxform = str + cfgWithOverrideParameters['GENERAL'] = {'testp1': 1., 'testp2': 'testValue'} + cfgWithOverrideParameters['com1DFA_override'] = {'defaultConfig': True, 'mu': 0.7, 'tStep': 100., 'notParameter': 1.} + + cfgToOverride = configparser.ConfigParser() + cfgToOverride.optionxform = str + cfgToOverride['GENERAL'] = {'mu': 1., 'tEnd': 400., 'relTh': 1.} + cfgToOverride['DFA'] = {'tStep': 0.3, 'dt1': 6., 'flowF': 3.} + + with caplog.at_level(logging.WARNING): + cfgToOverride, cfgWithOverrideParameters = cfgHandling.applyCfgOverride(cfgToOverride, cfgWithOverrideParameters, com1DFA, addModValues=True) + assert ('Additional Key [\'%s\'] in section %s_override is ignored.' % ('notParameter', 'com1DFA')) in caplog.text + + assert cfgToOverride['GENERAL']['mu'] == '0.7' + assert cfgToOverride['GENERAL']['tEnd'] == '400.0' + assert cfgToOverride['GENERAL']['relTh'] == '1.0' + assert cfgToOverride['DFA']['tStep'] == '100.0' + assert cfgToOverride['DFA']['dt1'] == '6.0' + assert cfgToOverride['DFA']['flowF'] == '3.0' + assert cfgWithOverrideParameters['GENERAL']['testp1'] == '1.0' + assert cfgWithOverrideParameters['GENERAL']['testp2'] == 'testValue' + assert cfgWithOverrideParameters['com1DFA_override']['defaultConfig'] == 'True' + assert cfgWithOverrideParameters['com1DFA_override']['mu'] == '0.7' + assert cfgWithOverrideParameters['com1DFA_override']['tStep'] == '100.0' + assert cfgWithOverrideParameters['com1DFA_override']['notParameter'] == '1.0' + assert cfgWithOverrideParameters['com1DFA_override']['tEnd'] == '400.0' + assert cfgWithOverrideParameters['com1DFA_override']['dt1'] == '6.0' + assert cfgWithOverrideParameters['com1DFA_override']['flowF'] == '3.0' + assert cfgWithOverrideParameters['com1DFA_override']['relTh'] == '1.0' + assert len(cfgWithOverrideParameters.items('GENERAL')) == 2 + assert len(cfgWithOverrideParameters.items('com1DFA_override')) == 8 + assert len(cfgToOverride.items('GENERAL')) == 3 + assert len(cfgToOverride.items('GENERAL')) == 3 diff --git a/avaframe/tests/test_cfgUtils.py b/avaframe/tests/test_cfgUtils.py index 38c597be5..e66ffa309 100644 --- a/avaframe/tests/test_cfgUtils.py +++ b/avaframe/tests/test_cfgUtils.py @@ -1,24 +1,12 @@ """Tests for module cfgUtils""" from avaframe.in3Utils import cfgUtils -from avaframe.in3Utils import cfgHandling from avaframe.tests import test_logUtils import pathlib import pytest import configparser -def test_addInfoToSimName(): - '''Test for addInfoToSimname''' - avaTestDir = 'avaParabolaStatsTest' - dirPath = pathlib.Path(__file__).parents[0] - avaDir = dirPath / '..' / '..' / 'benchmarks' / avaTestDir - renameDF = cfgHandling.addInfoToSimName(avaDir,'mu') - assert renameDF.loc['c4f3a000c3']['newName'] == 'release1PF_c4f3a000c3_mu_0.155_null_dfa' - renameDF = cfgHandling.addInfoToSimName(avaDir,'mu,tEnd') - assert renameDF.loc['c4f3a000c3']['newName'] == 'release1PF_c4f3a000c3_mu_0.155_tEnd_400_null_dfa' - - def test_getModuleConfig(): '''Test for module getModuleConfig''' @@ -96,49 +84,6 @@ def test_convertDictToConfigParser(): assert cfgDict['TEST1']['mu'] == cfg['TEST1'].getfloat('mu') -def test_orderSimFiles(): - """ test generating order of simulation results """ - - avaTestDir = 'avaHockeyChannelPytest' - dirPath = pathlib.Path(__file__).parents[0] - avaDir = dirPath / '..' / '..' / 'benchmarks' / avaTestDir - - varParList = 'releaseScenario' - - simDF = cfgUtils.createConfigurationInfo(avaDir, specDir='') - - varParList, simDF = cfgHandling.orderSimulations(varParList, True, simDF) - - assert simDF['simName'][0] == 'release1HS_d10bdc1e81_ent_dfa' - - varParList = 'releaseSenario' - message = ('Choose a valid parameter for sorting the simulations. \'releaseSenario\' is not valid.') - with pytest.raises(KeyError) as e: - assert cfgHandling.orderSimulations(varParList, True, simDF) - assert message in str(e.value) - - -def test_fetchAndOrderSimFiles(): - """ test generating order of simulation results """ - - avaTestDir = 'avaHockeyChannelPytest' - dirPath = pathlib.Path(__file__).parents[0] - avaDir = dirPath / '..' / '..' / 'benchmarks' / avaTestDir - inputDir = avaDir / 'Outputs' / 'com1DFA' / 'peakFiles' - - varParList = 'releaseScenario' - - simDF = cfgHandling.fetchAndOrderSimFiles(avaDir, inputDir, varParList, True, specDir='', resFiles=True) - - assert simDF['simName'][0] == 'release1HS_d10bdc1e81_ent_dfa' - - varParList = 'releaseSenario' - message = ('Choose a valid parameter for sorting the simulations. \'releaseSenario\' is not valid.') - with pytest.raises(KeyError) as e: - simDF = cfgHandling.fetchAndOrderSimFiles(avaDir, 'inputDir', varParList, True, specDir='', resFiles=False) - assert message in str(e.value) - - def test_createConfigurationInfo(tmp_path): """ test configuration info generation as DF """ @@ -198,62 +143,3 @@ def test_appendCgf2DF(tmp_path): assert simDF.loc['d10bdc1e81']['mu'] == 0.155 assert simDF.loc['e2145362b7']['releaseScenario'] != 'release1HS' assert simDF.loc['e2145362b7']['relTh0'] == 1. - - -def test_filterSims(tmp_path): - """ test filtering of simulations using configuration files """ - - avaTestDir = 'avaHockeyChannelPytest' - dirPath = pathlib.Path(__file__).parents[0] - avaDir = dirPath / '..' / '..' / 'benchmarks' / avaTestDir - avaDir2 = dirPath / 'data' / 'avaFilterTest' / 'com1DFA' - - parametersDict = {'releaseScenario': 'release1HS'} - - simNames = cfgHandling.filterSims(avaDir, parametersDict, specDir='') - - testRel = False - testRel2 = False - if 'release1HS' in simNames[0]: - testRel = True - if 'release2HS' in simNames[0]: - testRel2 = True - - assert testRel - assert testRel2 is False - assert len(simNames) == 1 - - parametersDict = {'relTh': 1.} - simNames2 = cfgHandling.filterSims(avaDir, parametersDict, specDir='') - simNames2 = sorted(simNames2) - assert len(simNames2) == 2 - assert simNames2[0] == 'release1HS_d10bdc1e81_ent_dfa' - assert simNames2[1] == 'release2HS_e2145362b7_ent_dfa' - - parametersDict = {'~relTh': 1.} - simNames3 = cfgHandling.filterSims(avaDir, parametersDict, specDir='') - - assert len(simNames3) == 0 - - parametersDict = {'~releaseScenario': 'release1HS'} - simNames4 = cfgHandling.filterSims(avaDir, parametersDict, specDir='') - - assert len(simNames4) == 1 - assert simNames4[0] == 'release2HS_e2145362b7_ent_dfa' - - parametersDict = {'relTh': 1.} - - simNames = cfgHandling.filterSims(avaDir2, parametersDict, specDir=avaDir2) - simNames = sorted(simNames) - - assert len(simNames) == 2 - assert simNames == ['relGar_6f35cbd808_null_dfa', 'relGar_b9b17dd019_ent_dfa'] - - parametersDict = {'relTh': [1., 1.5]} - - simNames = cfgHandling.filterSims(avaDir2, parametersDict, specDir=avaDir2) - simNames = sorted(simNames) - - assert len(simNames) == 4 - assert simNames == ['relGar_1022880a70_null_dfa', 'relGar_6f35cbd808_null_dfa', - 'relGar_b9b17dd019_ent_dfa', 'relGar_c4337e50ac_null_dfa'] diff --git a/avaframe/tests/test_com1DFA.py b/avaframe/tests/test_com1DFA.py index c4c587792..b011ba769 100644 --- a/avaframe/tests/test_com1DFA.py +++ b/avaframe/tests/test_com1DFA.py @@ -16,6 +16,7 @@ from avaframe.com1DFA import com1DFA import avaframe.in2Trans.ascUtils as IOf +import avaframe.in3Utils.initializeProject as initProj from avaframe.in3Utils import cfgUtils @@ -1624,6 +1625,7 @@ def test_runCom1DFA(tmp_path, caplog): assert (outDir / 'configurationFiles' / ('%s.ini' % (simDF['simName'][1]))).is_file() assert (outDir / 'configurationFiles' / ('allConfigurations.csv')).is_file() + initProj.cleanModuleFiles(avaDir, com1DFA, deleteOutput=False) with caplog.at_level(logging.WARNING): dem, plotDict, reportDictList, simDF = com1DFA.com1DFAMain( avaDir, cfgMain, cfgFile=cfgFile) diff --git a/avaframe/tests/test_initializeDirs.py b/avaframe/tests/test_initializeDirs.py index 3346fd14b..62330c136 100644 --- a/avaframe/tests/test_initializeDirs.py +++ b/avaframe/tests/test_initializeDirs.py @@ -5,6 +5,7 @@ # Load modules import pathlib +import pytest from avaframe.in3Utils import initialiseDirs as iD from avaframe.in3Utils import fileHandlerUtils as fU @@ -32,6 +33,16 @@ def test_initialiseRunDirs(tmp_path): assert outputDir.is_dir() assert DEMremeshedTest.is_dir() is False + # call function to be tested + with pytest.raises(AssertionError) as e: + assert iD.initialiseRunDirs(avaDirtmp, modName, cleanDEMremeshed=False) + assert str(e.value) == 'Work directory %s already exists - delete first!' % (workDirTest) + + # set directory + avaName = 'avaTest2' + avaDirtmp = pathlib.Path(tmp_path, avaName) + modName = 'com1DFA' + # define results workDirTest = avaDirtmp / 'Work' / modName outputDirTest = avaDirtmp / 'Outputs' / modName