diff --git a/avaframe/ana3AIMEC/dfa2Aimec.py b/avaframe/ana3AIMEC/dfa2Aimec.py index a246eb7d1..f96b6f669 100644 --- a/avaframe/ana3AIMEC/dfa2Aimec.py +++ b/avaframe/ana3AIMEC/dfa2Aimec.py @@ -146,6 +146,8 @@ def dfaBench2Aimec(avaDir, cfg, simNameRef='', simNameComp=''): pathDict['colorParameter'] = colorParameter pathDict['refSimRowHash'] = refSimRowHash pathDict['refSimName'] = refSimName + # set pathDict ValRef to empty as reference sim is not chosen on a value + pathDict['valRef'] = '' # if desired set path to mass log files if cfg['FLAGS'].getboolean('flagMass'): refData = getMassInfoInDF(avaDir, refData, comModules[0], sim=refSimName, testName=cfgSetup['testName']) diff --git a/avaframe/com1DFA/com1DFA.py b/avaframe/com1DFA/com1DFA.py index d3bbb7ac7..bd64cf154 100644 --- a/avaframe/com1DFA/com1DFA.py +++ b/avaframe/com1DFA/com1DFA.py @@ -319,16 +319,13 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=''): vector of saving time steps """ - # load GENERAL configuration - cfgGen = cfg['GENERAL'] - # select release area input data according to chosen release scenario inputSimFiles = gI.selectReleaseFile(inputSimFiles, cfg['INPUT']['releaseScenario']) # create required input from input files demOri, inputSimLines = prepareInputData(inputSimFiles, cfg) - if cfgGen.getboolean('iniStep'): + if cfg['GENERAL'].getboolean('iniStep'): # append buffered release Area inputSimLines = pI.createReleaseBuffer(cfg, inputSimLines) @@ -377,7 +374,7 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=''): exportFields(cfg, Tsave, fieldsList, dem, outDir, cuSimName) # write report dictionary - reportDict = createReportDict(avaDir, cuSimName, relName, inputSimLines, cfgGen, reportAreaInfo) + reportDict = createReportDict(avaDir, cuSimName, relName, inputSimLines, cfg, reportAreaInfo) # add time and mass info to report reportDict = reportAddTimeMassInfo(reportDict, tCPUDFA, infoDict) @@ -612,7 +609,7 @@ def prepareInputData(inputSimFiles, cfg): return demOri, inputSimLines -def createReportDict(avaDir, logName, relName, inputSimLines, cfgGen, reportAreaInfo): +def createReportDict(avaDir, logName, relName, inputSimLines, cfg, reportAreaInfo): """ create simulaton report dictionary Parameters @@ -623,8 +620,8 @@ def createReportDict(avaDir, logName, relName, inputSimLines, cfgGen, reportArea release name relDict : dict release dictionary - cfgGen : configparser - general configuration file + cfg : configparser + simulation configuration entrainmentArea : str entrainment file name resistanceArea : str @@ -644,9 +641,12 @@ def createReportDict(avaDir, logName, relName, inputSimLines, cfgGen, reportArea resistanceArea = inputSimLines['resistanceArea'] relDict = inputSimLines['releaseLine'] + + # Get default cfg and convert to dict for comparison + cfgGen = cfg['GENERAL'] + # Create dictionary reportST = {} - reportST = {} reportST = {'headerLine': {'type': 'title', 'title': 'com1DFA Simulation'}, 'avaName': {'type': 'avaName', 'name': str(avaDir)}, 'simName': {'type': 'simName', 'name': logName}, @@ -654,18 +654,36 @@ def createReportDict(avaDir, logName, relName, inputSimLines, cfgGen, reportArea 'Simulation Parameters': { 'type': 'list', 'Program version': getVersion(), - 'Parameter set': '', + 'Parameter set': 'Default', 'Release Area Scenario': relName, 'Entrainment': entInfo, 'Resistance': resInfo, - 'Parameter variation on': '', - 'Parameter value': '', 'Mu': cfgGen['mu'], 'Density [kgm-3]': cfgGen['rho'], 'Friction model': cfgGen['frictModel']}, 'Release Area': {'type': 'columns', 'Release area scenario': relName, 'Release Area': relDict['Name'], 'Release thickness [m]': relDict['thickness']}} + # Check if parameter set is modified from default, and add section to report + if '_C_' in logName: + reportST['Simulation Parameters']['Parameter set'] = 'Changed' + reportST.update({'Parameters changed from default': { + 'type': 'list' + }}) + cfgDict = cfgUtils.convertConfigParserToDict(cfg) + _, changedVals = com1DFATools.compareSimCfgToDefaultCfgCom1DFA(cfgDict) + + for key, val in changedVals.items(): + # Format key string for better readability (without loosing the parameter path) + keyStr = key.replace('root','') + keyStr = keyStr.replace("\'][\'","->") + keyStr = keyStr.replace("[\'","") + keyStr = keyStr.replace("\']","") + + valStr = val['new_value'] + ' (default is ' + val['old_value'] +')' + + reportST['Parameters changed from default'][keyStr] = valStr + if entInfo == 'Yes': entDict = inputSimLines['entLine'] reportST.update({'Entrainment area': @@ -2405,8 +2423,12 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting cfgSimObject = cfgUtils.convertDictToConfigParser(cfgSim) # create unique hash for simulation configuration simHash = cfgUtils.cfgHash(cfgSimObject) - simName = (relNameSim + '_' + simHash + '_' + row._asdict()['simTypeList'] + '_' - + cfgSim['GENERAL']['modelType']) + + # check differences to default and add indicator to name + defID, _ = com1DFATools.compareSimCfgToDefaultCfgCom1DFA(cfgSim) + + simName = '_'.join([relNameSim, simHash, defID, row._asdict()['simTypeList'], cfgSim['GENERAL']['modelType']]) + # check if simulation exists. If yes do not append it if simName not in simNameExisting: simDict[simName] = {'simHash': simHash, 'releaseScenario': relName, diff --git a/avaframe/com1DFA/com1DFATools.py b/avaframe/com1DFA/com1DFATools.py index 423607ff5..f3806934e 100644 --- a/avaframe/com1DFA/com1DFATools.py +++ b/avaframe/com1DFA/com1DFATools.py @@ -5,6 +5,9 @@ # Load modules import logging import math +from avaframe.com1DFA import com1DFA +from avaframe.in3Utils import cfgUtils +from deepdiff import DeepDiff # local imports import avaframe.com1DFA.deriveParameterSet as dP @@ -60,7 +63,6 @@ def getPartInitMethod(cfg, csz, relThForPart): return massPerPart, nPPK - def setRelThIni(avaDir, modName, cfgInitial): """ Add thickness values in configuration file according to thickness flags, ini settings or shapefile attributes and create inputSimFiles dictionary with file paths to input data @@ -91,3 +93,57 @@ def setRelThIni(avaDir, modName, cfgInitial): inputSimFilesAll, cfgInitial = gI.getThickness(inputSimFilesAll, avaDir, modName, cfgInitial) return inputSimFilesAll, cfgInitial + + +def compareSimCfgToDefaultCfgCom1DFA(simCfg): + """ Compares the given simulation configuration (as dict) to the default + com1DFA configuration. Disregards values like avalancheDir that are expected to + change. Returns True if it is the default + an indentifier string: D = Default and + C = Changed + + Parameters + ----------- + simCfg: dict + simulation configuration + + Returns + -------- + isDefault: bool + True if it is the same as the default configuration + defaultIdentifierString: str + D if default and C if changed + + """ + + defaultIdentifierString = 'D' + + # Get default cfg and convert to dict for comparison + defCfgObject = cfgUtils.getDefaultModuleConfig(com1DFA, toPrint=False) + defCfg = cfgUtils.convertConfigParserToDict(defCfgObject) + + # Which changes to ignore (in the case of com1DFA). These are expected + # to change... + excludeItems = ["root['GENERAL']['avalancheDir']", + "root['GENERAL']['secRelArea']", + "root['GENERAL']['simTypeList']", + "root['INPUT']['releaseScenario']", + ] + + # do the diff and analyse + diff = DeepDiff(defCfg, simCfg, exclude_paths=excludeItems) + + + if 'values_changed' in diff: + log.info('Comparing to default cfg, values changed:') + log.info(diff['values_changed']) + log.info('Modified from default') + isDefault = False + defaultIdentifierString = 'C' + else: + diff['values_changed'] = None + + if 'dictionary_item_added' in diff: + log.debug('Comparing to default cfg, added items:') + log.debug(diff['dictionary_item_added']) + + return defaultIdentifierString, diff['values_changed'] diff --git a/avaframe/in3Utils/cfgUtils.py b/avaframe/in3Utils/cfgUtils.py index b2eaa6a5d..532d049b7 100644 --- a/avaframe/in3Utils/cfgUtils.py +++ b/avaframe/in3Utils/cfgUtils.py @@ -140,6 +140,7 @@ def getDefaultModuleConfig(module, toPrint=True): defaultFile = modPath / (modName+'Cfg.ini') + log.info('Getting the default config for %s', modName) log.debug('defaultFile: %s', defaultFile) # Finally read it @@ -199,6 +200,7 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): return modCfg, modDict + def _splitDeepDiffValuesChangedItem(inKey, inVal): """ splits one item of a deepdiff result into section, key, old value, new value @@ -252,6 +254,7 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False): """ + log.info('Comparing two configs') # initialize modInfo and printOutInfo modInfo = dict() @@ -298,7 +301,6 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False): return modInfo, modCfg - def writeCfgFile(avaDir, module, cfg, fileName='', filePath=''): """ Save configuration used to text file in Outputs/moduleName/configurationFiles/modName.ini or optional to filePath and with fileName @@ -597,7 +599,7 @@ def convertDF2numerics(simDF): simDF: pandas DataFrame """ - for name, values in simDF.iteritems(): + for name, values in simDF.items(): simDFTest = simDF[name].str.replace('.', '', regex=True) # allow for - sign too simDFTest = simDFTest.replace('-', '', regex=True) @@ -679,7 +681,6 @@ def writeAllConfigurationInfo(avaDir, simDF, specDir=''): return configFiles - def convertToCfgList(parameterList): """ convert a list into a string where inidividual list items are separated by | diff --git a/avaframe/in3Utils/fileHandlerUtils.py b/avaframe/in3Utils/fileHandlerUtils.py index 70a63a902..3de30a6ad 100644 --- a/avaframe/in3Utils/fileHandlerUtils.py +++ b/avaframe/in3Utils/fileHandlerUtils.py @@ -545,7 +545,7 @@ def makeSimDF(inputDir, avaDir='', simID='simID'): # Set name of avalanche if avaDir is given # Make dictionary of input data info - data = {'files': [], 'names': [], 'resType': [], 'simType': [], 'simName': [], + data = {'files': [], 'names': [], 'resType': [], 'simType': [], 'isDefault': [], 'simName': [], 'modelType': [], 'releaseArea': [], 'cellSize': [], simID: [], 'timeStep': []} # Set name of avalanche if avaDir is given @@ -572,17 +572,37 @@ def makeSimDF(inputDir, avaDir='', simID='simID'): data['releaseArea'].append(relNameSim) data[simID].append(infoParts[0]) - data['simType'].append(infoParts[1]) - data['modelType'].append(infoParts[2]) - data['resType'].append(infoParts[3]) - data['simName'].append(fNamePart + '_' + ('_'.join(infoParts[0:3]))) - - header = IOf.readASCheader(datafiles[m]) - data['cellSize'].append(header['cellsize']) - if len(infoParts) == 5: - data['timeStep'].append(infoParts[4]) + + indiStr = ['_C_', '_D_'] + if any(x in name for x in indiStr): + data['isDefault'].append(infoParts[1]) + data['simType'].append(infoParts[2]) + data['modelType'].append(infoParts[3]) + data['resType'].append(infoParts[4]) + data['simName'].append(fNamePart + '_' + ('_'.join(infoParts[0:4]))) + + header = IOf.readASCheader(datafiles[m]) + data['cellSize'].append(header['cellsize']) + if len(infoParts) == 6: + data['timeStep'].append(infoParts[5]) + else: + data['timeStep'].append('') + + # If it still is an 'old' simname + # This can be removed at one point else: - data['timeStep'].append('') + data['isDefault'].append(None) + data['simType'].append(infoParts[1]) + data['modelType'].append(infoParts[2]) + data['resType'].append(infoParts[3]) + data['simName'].append(fNamePart + '_' + ('_'.join(infoParts[0:3]))) + + header = IOf.readASCheader(datafiles[m]) + data['cellSize'].append(header['cellsize']) + if len(infoParts) == 5: + data['timeStep'].append(infoParts[4]) + else: + data['timeStep'].append('') # Set name of avalanche if avaDir is given if avaDir != '': diff --git a/avaframe/runScripts/runAna3AIMECCompMods.py b/avaframe/runScripts/runAna3AIMECCompMods.py index 6187f617a..84ed483f3 100644 --- a/avaframe/runScripts/runAna3AIMECCompMods.py +++ b/avaframe/runScripts/runAna3AIMECCompMods.py @@ -62,9 +62,6 @@ def runAna3AIMECCompMods(avalancheDir=''): cfgSetup = cfg['AIMECSETUP'] comModules = cfgSetup['comModules'].split('|') pathDict = aimecTools.readAIMECinputs(avalancheDir, pathDict, dirName=(comModules[0] + '_' + comModules[1])) - - # set pathDict ValRef to empty as reference sim is not chosen on a value - pathDict['valRef'] = '' startTime = time.time() diff --git a/avaframe/tests/data/testCFGRef.tog b/avaframe/tests/data/testCFGRef.tog index 59d6a8184..7092ff06d 100644 --- a/avaframe/tests/data/testCFGRef.tog +++ b/avaframe/tests/data/testCFGRef.tog @@ -1,3 +1,4 @@ +cfgUtils - INFO - Comparing two configs cfgUtils - INFO - {'GENERAL': {'inputDir': 'path/to/avalanche', 'fullLog': 'True'}, cfgUtils - INFO - 'FLAGS': {'plotFigure': 'False', 'savePlot': 'False'}, cfgUtils - INFO - 'GOODSECTION1': {'goodKey1': '1', 'goodKey2': 'False', 'testKey1': 'Bonjour'}, diff --git a/avaframe/tests/test_com1DFA.py b/avaframe/tests/test_com1DFA.py index d940099d2..24522e9d7 100644 --- a/avaframe/tests/test_com1DFA.py +++ b/avaframe/tests/test_com1DFA.py @@ -351,7 +351,7 @@ def test_createReportDict(): # call function to be tested reportST = com1DFA.createReportDict( - avaDir, logName, relName, inputSimLines, cfg['GENERAL'], reportAreaInfo) + avaDir, logName, relName, inputSimLines, cfg, reportAreaInfo) assert 'Simulation Parameters' in reportST assert 'Program version' in reportST['Simulation Parameters'] @@ -1422,11 +1422,13 @@ def test_prepareVarSimDict(tmp_path, caplog): testCfg['GENERAL']['avalancheDir'] = str(avaDirTest) simHash = cfgUtils.cfgHash(testCfg) - simName1 = 'relAlr_' + simHash + '_entres_dfa' + print(simHash) + simName1 = 'relAlr_' + simHash + '_C_entres_dfa' testDict = {simName1: {'simHash': simHash, 'releaseScenario': 'relAlr', 'simType': 'entres', 'relFile': relPath, 'cfgSim': testCfg}} for key in testDict[simName1]: + print(simDict[simName1][key]) assert simDict[simName1][key] == testDict[simName1][key] for section in testCfg.sections(): @@ -1457,10 +1459,13 @@ def test_prepareVarSimDict(tmp_path, caplog): testCfg2['INPUT']['DEM'] = 'avaAlr.asc' testCfg2['GENERAL']['avalancheDir'] = str(avaDirTest) simHash2 = cfgUtils.cfgHash(testCfg2) - simName2 = 'relAlr_' + simHash2 + '_entres_dfa' + simName2 = 'relAlr_' + simHash2 + '_C_entres_dfa' testDict2 = {simName2: {'simHash': simHash2, 'releaseScenario': 'relAlr', 'simType': 'entres', 'relFile': relPath, 'cfgSim': testCfg2}} + + print(simDict2) + print(testDict2) for key in testDict2[simName2]: assert simDict2[simName2][key] == testDict2[simName2][key]