Add indicator of default/changed to simulationname and report

avaframe/ana3AIMEC/dfa2Aimec.py

@@ -146,6 +146,8 @@ def dfaBench2Aimec(avaDir, cfg, simNameRef='', simNameComp=''):
     pathDict['colorParameter'] = colorParameter
     pathDict['refSimRowHash'] = refSimRowHash
     pathDict['refSimName'] = refSimName
+    # set pathDict ValRef to empty as reference sim is not chosen on a value
+    pathDict['valRef'] = ''
     # if desired set path to mass log files
     if cfg['FLAGS'].getboolean('flagMass'):
         refData = getMassInfoInDF(avaDir, refData, comModules[0], sim=refSimName, testName=cfgSetup['testName'])

avaframe/com1DFA/com1DFA.py

@@ -319,16 +319,13 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=''):
         vector of saving time steps
     """
 
-    # load GENERAL configuration
-    cfgGen = cfg['GENERAL']
-
     # select release area input data according to chosen release scenario
     inputSimFiles = gI.selectReleaseFile(inputSimFiles, cfg['INPUT']['releaseScenario'])
 
     # create required input from input files
     demOri, inputSimLines = prepareInputData(inputSimFiles, cfg)
 
-    if cfgGen.getboolean('iniStep'):
+    if cfg['GENERAL'].getboolean('iniStep'):
         # append buffered release Area
         inputSimLines = pI.createReleaseBuffer(cfg, inputSimLines)
 
@@ -377,7 +374,7 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=''):
     exportFields(cfg, Tsave, fieldsList, dem, outDir, cuSimName)
 
     # write report dictionary
-    reportDict = createReportDict(avaDir, cuSimName, relName, inputSimLines, cfgGen, reportAreaInfo)
+    reportDict = createReportDict(avaDir, cuSimName, relName, inputSimLines, cfg, reportAreaInfo)
     # add time and mass info to report
     reportDict = reportAddTimeMassInfo(reportDict, tCPUDFA, infoDict)
 
@@ -612,7 +609,7 @@ def prepareInputData(inputSimFiles, cfg):
     return demOri, inputSimLines
 
 
-def createReportDict(avaDir, logName, relName, inputSimLines, cfgGen, reportAreaInfo):
+def createReportDict(avaDir, logName, relName, inputSimLines, cfg, reportAreaInfo):
     """ create simulaton report dictionary
 
     Parameters
@@ -623,8 +620,8 @@ def createReportDict(avaDir, logName, relName, inputSimLines, cfgGen, reportArea
         release name
     relDict : dict
         release dictionary
-    cfgGen : configparser
-        general configuration file
+    cfg : configparser
+        simulation configuration 
     entrainmentArea : str
         entrainment file name
     resistanceArea : str
@@ -644,28 +641,49 @@ def createReportDict(avaDir, logName, relName, inputSimLines, cfgGen, reportArea
     resistanceArea = inputSimLines['resistanceArea']
     relDict = inputSimLines['releaseLine']
 
+
+    # Get default cfg and convert to dict for comparison
+    cfgGen = cfg['GENERAL']
+
     # Create dictionary
     reportST = {}
-    reportST = {}
     reportST = {'headerLine': {'type': 'title', 'title': 'com1DFA Simulation'},
                 'avaName': {'type': 'avaName', 'name': str(avaDir)},
                 'simName': {'type': 'simName', 'name': logName},
                 'time': {'type': 'time', 'time': dateTimeInfo},
                 'Simulation Parameters': {
                 'type': 'list',
                 'Program version': getVersion(),
-                'Parameter set': '',
+                'Parameter set': 'Default',
                 'Release Area Scenario': relName,
                 'Entrainment': entInfo,
                 'Resistance': resInfo,
-                'Parameter variation on': '',
-                'Parameter value': '',
                 'Mu': cfgGen['mu'],
                 'Density [kgm-3]': cfgGen['rho'],
                 'Friction model': cfgGen['frictModel']},
                 'Release Area': {'type': 'columns', 'Release area scenario': relName, 'Release Area': relDict['Name'],
                                  'Release thickness [m]': relDict['thickness']}}
 
+    # Check if parameter set is modified from default, and add section to report
+    if '_C_' in logName:
+        reportST['Simulation Parameters']['Parameter set'] = 'Changed'
+        reportST.update({'Parameters changed from default': {
+                        'type': 'list'
+                         }})
+        cfgDict = cfgUtils.convertConfigParserToDict(cfg)
+        _, changedVals = com1DFATools.compareSimCfgToDefaultCfgCom1DFA(cfgDict)
+
+        for key, val in changedVals.items():
+            # Format key string for better readability (without loosing the parameter path)
+            keyStr = key.replace('root','')
+            keyStr = keyStr.replace("\'][\'","->")
+            keyStr = keyStr.replace("[\'","")
+            keyStr = keyStr.replace("\']","")
+
+            valStr = val['new_value'] + ' (default is ' + val['old_value'] +')'
+
+            reportST['Parameters changed from default'][keyStr] = valStr
+
     if entInfo == 'Yes':
         entDict = inputSimLines['entLine']
         reportST.update({'Entrainment area':
@@ -2405,8 +2423,12 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting
         cfgSimObject = cfgUtils.convertDictToConfigParser(cfgSim)
         # create unique hash for simulation configuration
         simHash = cfgUtils.cfgHash(cfgSimObject)
-        simName = (relNameSim + '_' + simHash + '_' + row._asdict()['simTypeList'] + '_'
-                   + cfgSim['GENERAL']['modelType'])
+
+        # check differences to default and add indicator to name
+        defID, _ = com1DFATools.compareSimCfgToDefaultCfgCom1DFA(cfgSim)
+
+        simName = '_'.join([relNameSim, simHash, defID, row._asdict()['simTypeList'], cfgSim['GENERAL']['modelType']])
+
         # check if simulation exists. If yes do not append it
         if simName not in simNameExisting:
             simDict[simName] = {'simHash': simHash, 'releaseScenario': relName,

avaframe/com1DFA/com1DFATools.py

@@ -5,6 +5,9 @@
 # Load modules
 import logging
 import math
+from avaframe.com1DFA import com1DFA
+from avaframe.in3Utils import cfgUtils
+from deepdiff import DeepDiff
 
 # local imports
 import avaframe.com1DFA.deriveParameterSet as dP
@@ -60,7 +63,6 @@ def getPartInitMethod(cfg, csz, relThForPart):
 
     return massPerPart, nPPK
 
-
 def setRelThIni(avaDir, modName, cfgInitial):
     """ Add thickness values in configuration file according to thickness flags, ini settings or shapefile attributes
         and create inputSimFiles dictionary with file paths to input data
@@ -91,3 +93,57 @@ def setRelThIni(avaDir, modName, cfgInitial):
     inputSimFilesAll, cfgInitial = gI.getThickness(inputSimFilesAll, avaDir, modName, cfgInitial)
 
     return inputSimFilesAll, cfgInitial
+
+
+def compareSimCfgToDefaultCfgCom1DFA(simCfg):
+    """ Compares the given simulation configuration (as dict) to the default 
+        com1DFA configuration. Disregards values like avalancheDir that are expected to 
+        change. Returns True if it is the default + an indentifier string: D = Default and 
+        C = Changed
+
+        Parameters
+        -----------
+        simCfg: dict
+            simulation configuration
+
+        Returns
+        --------
+        isDefault: bool
+            True if it is the same as the default configuration 
+        defaultIdentifierString: str
+            D if default and C if changed
+
+    """
+
+    defaultIdentifierString = 'D'
+
+    # Get default cfg and convert to dict for comparison
+    defCfgObject = cfgUtils.getDefaultModuleConfig(com1DFA, toPrint=False)
+    defCfg = cfgUtils.convertConfigParserToDict(defCfgObject)
+
+    # Which changes to ignore (in the case of com1DFA). These are expected
+    # to change...
+    excludeItems = ["root['GENERAL']['avalancheDir']", 
+                   "root['GENERAL']['secRelArea']",
+                   "root['GENERAL']['simTypeList']",
+                   "root['INPUT']['releaseScenario']",
+                   ]
+
+    # do the diff and analyse
+    diff = DeepDiff(defCfg, simCfg, exclude_paths=excludeItems)
+
+
+    if 'values_changed' in diff:
+        log.info('Comparing to default cfg, values changed:')
+        log.info(diff['values_changed'])
+        log.info('Modified from default')
+        isDefault = False 
+        defaultIdentifierString = 'C'
+    else: 
+        diff['values_changed'] = None
+
+    if 'dictionary_item_added' in diff:
+        log.debug('Comparing to default cfg, added items:')
+        log.debug(diff['dictionary_item_added'])
+
+    return defaultIdentifierString, diff['values_changed']

avaframe/in3Utils/cfgUtils.py

@@ -140,6 +140,7 @@ def getDefaultModuleConfig(module, toPrint=True):
 
     defaultFile = modPath / (modName+'Cfg.ini')
 
+    log.info('Getting the default config for %s', modName)
     log.debug('defaultFile: %s', defaultFile)
 
     # Finally read it
@@ -199,6 +200,7 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True):
 
     return modCfg, modDict
 
+
 def _splitDeepDiffValuesChangedItem(inKey, inVal):
     """ splits one item of a deepdiff result into section, key, old value, new value 
 
@@ -252,6 +254,7 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False):
     """
 
 
+    log.info('Comparing two configs')
 
     # initialize modInfo and printOutInfo
     modInfo = dict() 
@@ -298,7 +301,6 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False):
     return modInfo, modCfg
 
 
-
 def writeCfgFile(avaDir, module, cfg, fileName='', filePath=''):
     """ Save configuration used to text file in Outputs/moduleName/configurationFiles/modName.ini
         or optional to filePath and with fileName
@@ -597,7 +599,7 @@ def convertDF2numerics(simDF):
         simDF: pandas DataFrame
     """
 
-    for name, values in simDF.iteritems():
+    for name, values in simDF.items():
         simDFTest = simDF[name].str.replace('.', '', regex=True)
         # allow for - sign too
         simDFTest = simDFTest.replace('-', '', regex=True)
@@ -679,7 +681,6 @@ def writeAllConfigurationInfo(avaDir, simDF, specDir=''):
 
     return configFiles
 
-
 def convertToCfgList(parameterList):
     """ convert a list into a string where inidividual list items are separated by |
 

avaframe/in3Utils/fileHandlerUtils.py

@@ -545,7 +545,7 @@ def makeSimDF(inputDir, avaDir='', simID='simID'):
 
     # Set name of avalanche if avaDir is given
     # Make dictionary of input data info
-    data = {'files': [], 'names': [], 'resType': [], 'simType': [], 'simName': [],
+    data = {'files': [], 'names': [], 'resType': [], 'simType': [], 'isDefault': [], 'simName': [],
             'modelType': [], 'releaseArea': [], 'cellSize': [], simID: [], 'timeStep': []}
 
     # Set name of avalanche if avaDir is given
@@ -572,17 +572,37 @@ def makeSimDF(inputDir, avaDir='', simID='simID'):
 
         data['releaseArea'].append(relNameSim)
         data[simID].append(infoParts[0])
-        data['simType'].append(infoParts[1])
-        data['modelType'].append(infoParts[2])
-        data['resType'].append(infoParts[3])
-        data['simName'].append(fNamePart + '_' + ('_'.join(infoParts[0:3])))
-
-        header = IOf.readASCheader(datafiles[m])
-        data['cellSize'].append(header['cellsize'])
-        if len(infoParts) == 5:
-            data['timeStep'].append(infoParts[4])
+
+        indiStr = ['_C_', '_D_']
+        if any(x in name for x in indiStr): 
+            data['isDefault'].append(infoParts[1])
+            data['simType'].append(infoParts[2])
+            data['modelType'].append(infoParts[3])
+            data['resType'].append(infoParts[4])
+            data['simName'].append(fNamePart + '_' + ('_'.join(infoParts[0:4])))
+
+            header = IOf.readASCheader(datafiles[m])
+            data['cellSize'].append(header['cellsize'])
+            if len(infoParts) == 6:
+                data['timeStep'].append(infoParts[5])
+            else:
+                data['timeStep'].append('')
+
+        # If it still is an 'old' simname 
+        # This can be removed at one point
         else:
-            data['timeStep'].append('')
+            data['isDefault'].append(None)
+            data['simType'].append(infoParts[1])
+            data['modelType'].append(infoParts[2])
+            data['resType'].append(infoParts[3])
+            data['simName'].append(fNamePart + '_' + ('_'.join(infoParts[0:3])))
+
+            header = IOf.readASCheader(datafiles[m])
+            data['cellSize'].append(header['cellsize'])
+            if len(infoParts) == 5:
+                data['timeStep'].append(infoParts[4])
+            else:
+                data['timeStep'].append('')
 
         # Set name of avalanche if avaDir is given
         if avaDir != '':

avaframe/runScripts/runAna3AIMECCompMods.py

@@ -62,9 +62,6 @@ def runAna3AIMECCompMods(avalancheDir=''):
     cfgSetup = cfg['AIMECSETUP']
     comModules = cfgSetup['comModules'].split('|')
     pathDict = aimecTools.readAIMECinputs(avalancheDir, pathDict, dirName=(comModules[0] + '_' + comModules[1]))
-
-    # set pathDict ValRef to empty as reference sim is not chosen on a value
-    pathDict['valRef'] = ''
 
     startTime = time.time()
 

avaframe/tests/data/testCFGRef.tog

@@ -1,3 +1,4 @@
+cfgUtils - INFO -  Comparing two configs
 cfgUtils - INFO -  {'GENERAL': {'inputDir': 'path/to/avalanche', 'fullLog': 'True'},
 cfgUtils - INFO -   'FLAGS': {'plotFigure': 'False', 'savePlot': 'False'},
 cfgUtils - INFO -   'GOODSECTION1': {'goodKey1': '1', 'goodKey2': 'False', 'testKey1': 'Bonjour'},

avaframe/tests/test_com1DFA.py

@@ -351,7 +351,7 @@ def test_createReportDict():
 
     # call function to be tested
     reportST = com1DFA.createReportDict(
-        avaDir, logName, relName, inputSimLines, cfg['GENERAL'], reportAreaInfo)
+        avaDir, logName, relName, inputSimLines, cfg, reportAreaInfo)
 
     assert 'Simulation Parameters' in reportST
     assert 'Program version' in reportST['Simulation Parameters']
@@ -1422,11 +1422,13 @@ def test_prepareVarSimDict(tmp_path, caplog):
     testCfg['GENERAL']['avalancheDir'] = str(avaDirTest)
 
     simHash = cfgUtils.cfgHash(testCfg)
-    simName1 = 'relAlr_' + simHash + '_entres_dfa'
+    print(simHash)
+    simName1 = 'relAlr_' + simHash + '_C_entres_dfa'
     testDict = {simName1: {'simHash': simHash, 'releaseScenario': 'relAlr',
                            'simType': 'entres', 'relFile': relPath, 'cfgSim': testCfg}}
 
     for key in testDict[simName1]:
+        print(simDict[simName1][key])
         assert simDict[simName1][key] == testDict[simName1][key]
 
     for section in testCfg.sections():
@@ -1457,10 +1459,13 @@ def test_prepareVarSimDict(tmp_path, caplog):
     testCfg2['INPUT']['DEM'] = 'avaAlr.asc'
     testCfg2['GENERAL']['avalancheDir'] = str(avaDirTest)
     simHash2 = cfgUtils.cfgHash(testCfg2)
-    simName2 = 'relAlr_' + simHash2 + '_entres_dfa'
+    simName2 = 'relAlr_' + simHash2 + '_C_entres_dfa'
     testDict2 = {simName2: {'simHash': simHash2, 'releaseScenario': 'relAlr',
                             'simType': 'entres', 'relFile': relPath, 'cfgSim': testCfg2}}
 
+
+    print(simDict2)
+    print(testDict2)
     for key in testDict2[simName2]:
         assert simDict2[simName2][key] == testDict2[simName2][key]