From 73775184a5bc77963e9cb31c8154f50202afe608 Mon Sep 17 00:00:00 2001 From: Anna Wirbel Date: Fri, 5 Jan 2024 10:35:27 +0100 Subject: [PATCH] add contours plot add runscripts add function to create local cfg files with overrides add collection name to override section update docs move writing of local cfgs to folder add option to read config from dir --- avaframe/ana5Utils/distanceTimeAnalysis.py | 6 +- avaframe/avaframeCfg.ini | 3 +- avaframe/com1DFA/com1DFA.py | 8 +- avaframe/com1DFA/com1DFATools.py | 2 +- avaframe/in3Utils/cfgHandling.py | 8 ++ avaframe/in3Utils/cfgUtils.py | 30 ++++++-- avaframe/out3Plot/plotUtils.py | 2 +- avaframe/runCom1DFA.py | 75 +++++++++++-------- avaframe/runScripts/runPlotContoursFromAsc.py | 2 +- docs/configuration.rst | 10 +++ 10 files changed, 99 insertions(+), 47 deletions(-) diff --git a/avaframe/ana5Utils/distanceTimeAnalysis.py b/avaframe/ana5Utils/distanceTimeAnalysis.py index 1a9cceed4..5117d4f2c 100644 --- a/avaframe/ana5Utils/distanceTimeAnalysis.py +++ b/avaframe/ana5Utils/distanceTimeAnalysis.py @@ -559,7 +559,7 @@ def extractFrontAndMeanValuesTT(cfgRangeTime, flowF, demHeader, mtiInfo): return mtiInfo -def initializeRangeTime(modName, cfg, dem, simHash): +def initializeRangeTime(modName, cfg, dem, simHash, configDir): """ initialize generation of range-time diagram for visualizing simulation data Parameters @@ -572,6 +572,8 @@ def initializeRangeTime(modName, cfg, dem, simHash): dictionary with DEM header and data simHash: str unique simulation ID + configDir: str or pathlib path + path to configuration directory - optional if not provided has to be empty string Returns -------- @@ -583,7 +585,7 @@ def initializeRangeTime(modName, cfg, dem, simHash): """ # fetch configuration and add info - cfgRangeTime = cfgUtils.getModuleConfig(modName) + cfgRangeTime = cfgUtils.getModuleConfig(modName, configDir) cfgRangeTime['GENERAL']['tEnd'] = cfg['GENERAL']['tEnd'] cfgRangeTime['GENERAL']['avalancheDir'] = cfg['GENERAL']['avalancheDir'] diff --git a/avaframe/avaframeCfg.ini b/avaframe/avaframeCfg.ini index 4572b2567..a10fa9775 100644 --- a/avaframe/avaframeCfg.ini +++ b/avaframe/avaframeCfg.ini @@ -5,7 +5,8 @@ [MAIN] # Path to avalanche directory avalancheDir = data/avaParabola - +# OPTIONAL Path to configuration file directory +configurationDir = # number of CPU cores to use for the computation of com1DFA # possible values are: # - auto -> takes up to CPUPercent (see below) % of the available CPU cores diff --git a/avaframe/com1DFA/com1DFA.py b/avaframe/com1DFA/com1DFA.py index 021b49b42..a97fc2041 100644 --- a/avaframe/com1DFA/com1DFA.py +++ b/avaframe/com1DFA/com1DFA.py @@ -99,7 +99,7 @@ def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo, module=com1DFA): # read initial configuration if typeCfgInfo in ["cfgFromFile", "cfgFromDefault"]: - cfgStart = cfgUtils.getModuleConfig(module, fileOverride=cfgInfo, toPrint=False) + cfgStart = cfgUtils.getModuleConfig(module, cfgMain['MAIN']['configurationDir'], fileOverride=cfgInfo, toPrint=False) elif typeCfgInfo == "cfgFromObject": cfgStart = cfgInfo @@ -1994,7 +1994,7 @@ def initializeResistance(cfg, dem, simTypeActual, resLine, reportAreaInfo, thres return cResRaster, detRaster, reportAreaInfo -def DFAIterate(cfg, particles, fields, dem, inputSimLines, outDir, cuSimName, simHash=""): +def DFAIterate(cfg, particles, fields, dem, inputSimLines, configDir, outDir, cuSimName, simHash=""): """Perform time loop for DFA simulation Save results at desired intervals @@ -2013,6 +2013,8 @@ def DFAIterate(cfg, particles, fields, dem, inputSimLines, outDir, cuSimName, si dictionary with dem information inputSimLines : dict dictionary with input data dictionaries (releaseLine, entLine, ...) + configDir: str or pathlib Path + path to configuration directory - optional if not provided has to be empty string Returns ------- @@ -2143,7 +2145,7 @@ def DFAIterate(cfg, particles, fields, dem, inputSimLines, outDir, cuSimName, si # check if range-time diagram should be performed, if yes - initialize if cfg["VISUALISATION"].getboolean("createRangeTimeDiagram"): demRT = dtAna.setDemOrigin(dem) - mtiInfo, dtRangeTime, cfgRangeTime = dtAna.initializeRangeTime(dtAna, cfg, demRT, simHash) + mtiInfo, dtRangeTime, cfgRangeTime = dtAna.initializeRangeTime(dtAna, cfg, demRT, simHash, configDir) # fetch initial time step too mtiInfo, dtRangeTime = dtAna.fetchRangeTimeInfo( cfgRangeTime, cfg, dtRangeTime, t, demRT["header"], fields, mtiInfo diff --git a/avaframe/com1DFA/com1DFATools.py b/avaframe/com1DFA/com1DFATools.py index 7c940f61b..91d7dda59 100644 --- a/avaframe/com1DFA/com1DFATools.py +++ b/avaframe/com1DFA/com1DFATools.py @@ -299,7 +299,7 @@ def createSimDictFromCfgs(cfgMain, cfgPath, module=com1DFA): # loop over all cfgFiles and create simDict for index, cfgFile in enumerate(cfgFilesAll): # read configuration - cfgFromFile = cfgUtils.getModuleConfig(module, fileOverride=cfgFile, toPrint=False) + cfgFromFile = cfgUtils.getModuleConfig(module, "", fileOverride=cfgFile, toPrint=False) # create dictionary with one key for each simulation that shall be performed # NOTE: sims that are added don't need to be added to the simNameExisting list as diff --git a/avaframe/in3Utils/cfgHandling.py b/avaframe/in3Utils/cfgHandling.py index 86fef5c4a..589144f48 100644 --- a/avaframe/in3Utils/cfgHandling.py +++ b/avaframe/in3Utils/cfgHandling.py @@ -540,6 +540,11 @@ def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): optional - path to directory to store local_ cfg ini file to if not provided - local_ cfg ini file is saved to avalanche directory + Returns + -------- + locFilePath: pathlib Path + path to directory where local configuration files are written to derived from override sections + """ # if a path is provided - save local cfg ini file there @@ -576,6 +581,7 @@ def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): cfgModule = cfgUtils.getModuleConfig( cfgNamePath, + '', fileOverride="", modInfo=False, toPrint=False, @@ -606,6 +612,8 @@ def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): log.info("%s CONFIGURATION wrote to %s" % (cfgName, str(cfgF))) + return locFilePath + def _removeCfgItemsNotInOverride(cfgModule, overrideKeys): """ remove options of cfgModule if not part of overrideKeys diff --git a/avaframe/in3Utils/cfgUtils.py b/avaframe/in3Utils/cfgUtils.py index d4c9ef4b5..5aa2701fc 100644 --- a/avaframe/in3Utils/cfgUtils.py +++ b/avaframe/in3Utils/cfgUtils.py @@ -95,7 +95,8 @@ def getGeneralConfig(nameFile=""): return cfg -def getModuleConfig(module, fileOverride="", modInfo=False, toPrint=True, onlyDefault=False): + +def getModuleConfig(module, configDir, fileOverride="", modInfo=False, toPrint=True, onlyDefault=False): """Returns the configuration for a given module returns a configParser object @@ -108,6 +109,9 @@ def getModuleConfig(module, fileOverride="", modInfo=False, toPrint=True, onlyDe leads to getModuleConfig(c2) OR: pathlib Path to module (python file) + configDir: pathlib Path or str + directory to local configuration files - if not provided has to be empty str! + Str: fileOverride : allows for a completely different file location. However note: missing values from the default cfg will always be added! @@ -133,11 +137,22 @@ def getModuleConfig(module, fileOverride="", modInfo=False, toPrint=True, onlyDe else: modPath, modName = getModPathName(module) + if configDir != "": + if pathlib.Path(configDir).is_dir() is False: + message = "Provided configurationDir: %s is not a directory!" % str(configDir) + log.error(message) + raise NotADirectoryError(message) + else: + directoryFile = pathlib.Path(configDir, (("local_" + modName + "Cfg.ini"))) + else: + directoryFile = None + localFile = modPath / ("local_" + modName + "Cfg.ini") defaultFile = modPath / (modName + "Cfg.ini") log.debug("localFile: %s", localFile) log.debug("defaultFile: %s", defaultFile) + log.debug("directoryFile: %s", directoryFile) # Decide which one to take if fileOverride: @@ -147,7 +162,9 @@ def getModuleConfig(module, fileOverride="", modInfo=False, toPrint=True, onlyDe compare = True else: raise FileNotFoundError("Provided fileOverride does not exist: " + str(fileOverride)) - + elif (directoryFile is not None) and directoryFile.is_file(): + iniFile = [defaultFile, directoryFile] + compare = True elif localFile.is_file() and not onlyDefault: iniFile = localFile iniFile = [defaultFile, localFile] @@ -182,7 +199,6 @@ def getDefaultModuleConfig(module, toPrint=True): # get path to the module and its name modPath, modName = getModPathName(module) - defaultFile = modPath / (modName + "Cfg.ini") log.info("Getting the default config for %s", modName) @@ -1026,10 +1042,10 @@ def convertToCfgList(parameterList): parameterList: list list of parameter values - Returns - --------- - parameterString: str - str with parameter values separated by | + Returns + --------- + parameterString: str + str with parameter values separated by | """ if len(parameterList) == 0: diff --git a/avaframe/out3Plot/plotUtils.py b/avaframe/out3Plot/plotUtils.py index 5396d361a..1fc88ed1f 100644 --- a/avaframe/out3Plot/plotUtils.py +++ b/avaframe/out3Plot/plotUtils.py @@ -36,7 +36,7 @@ # Load all input Parameters from config file # get the configuration of an already imported module -cfg = cfgUtils.getModuleConfig(plotUtils) +cfg = cfgUtils.getModuleConfig(plotUtils, cfgMain['MAIN']['configurationDir']) cfgPlotUtils = cfg["UNITS"] cfgConstants = cfg["CONSTANTS"] cfg = cfg["MAIN"] diff --git a/avaframe/runCom1DFA.py b/avaframe/runCom1DFA.py index 3d295b088..a57c708fe 100644 --- a/avaframe/runCom1DFA.py +++ b/avaframe/runCom1DFA.py @@ -16,8 +16,8 @@ from avaframe.in3Utils import logUtils from avaframe.in3Utils import fileHandlerUtils as fU -def runCom1DFA(avalancheDir='', calibration=''): - """ Run com1DFA in the default configuration with only an +def runCom1DFA(avalancheDir="", configurationDir="", calibration=""): + """Run com1DFA in the default configuration with only an avalanche directory as input and the (optional) friction calibration size @@ -38,21 +38,25 @@ def runCom1DFA(avalancheDir='', calibration=''): startTime = time.time() # log file name; leave empty to use default runLog.log - logName = 'runCom1DFA' + logName = "runCom1DFA" # Load avalanche directory from general configuration file # More information about the configuration can be found here # on the Configuration page in the documentation cfgMain = cfgUtils.getGeneralConfig() - if avalancheDir != '': - cfgMain['MAIN']['avalancheDir'] = avalancheDir + if avalancheDir != "": + cfgMain["MAIN"]["avalancheDir"] = avalancheDir else: - avalancheDir = cfgMain['MAIN']['avalancheDir'] + avalancheDir = cfgMain["MAIN"]["avalancheDir"] + if configurationDir != "": + cfgMain["MAIN"]["configurationDir"] = configurationDir + else: + configurationDir = cfgMain["MAIN"]["configurationDir"] # Start logging log = logUtils.initiateLogger(avalancheDir, logName) - log.info('MAIN SCRIPT') - log.info('Current avalanche: %s', avalancheDir) + log.info("MAIN SCRIPT") + log.info("Current avalanche: %s", avalancheDir) # ---------------- # Clean input directory(ies) of old work and output files @@ -61,18 +65,18 @@ def runCom1DFA(avalancheDir='', calibration=''): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Set friction model according to cmd argument - cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, toPrint=False) - - if calibration.lower() == 'small': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATSmall' - elif calibration.lower() == 'medium': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATMedium' - elif calibration.lower() == 'large': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosAT' - elif calibration.lower() == 'auto': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATAuto' + cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, configDir=configurationDir, toPrint=False) + + if calibration.lower() == "small": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATSmall" + elif calibration.lower() == "medium": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATMedium" + elif calibration.lower() == "large": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosAT" + elif calibration.lower() == "auto": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATAuto" else: - log.info('no friction calibration override given - using ini') + log.info("no friction calibration override given - using ini") # ---------------- # Run dense flow @@ -80,25 +84,34 @@ def runCom1DFA(avalancheDir='', calibration=''): # Get peakfiles to return to QGIS avaDir = pathlib.Path(avalancheDir) - inputDir = avaDir / 'Outputs' / 'com1DFA' / 'peakFiles' + inputDir = avaDir / "Outputs" / "com1DFA" / "peakFiles" peakFilesDF = fU.makeSimDF(inputDir, avaDir=avaDir) # Print time needed endTime = time.time() - log.info('Took %6.1f seconds to calculate.' % (endTime - startTime)) + log.info("Took %6.1f seconds to calculate." % (endTime - startTime)) return peakFilesDF -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description='Run com1DFA workflow') - parser.add_argument('avadir', metavar='avalancheDir', type=str, nargs='?', default='', - help='the avalanche directory') - parser.add_argument('-fc', '--friction_calibration', choices=['auto', 'large', 'medium', 'small', 'ini'], - type=str, default='ini', - help='friction calibration override, possible values are large, medium , small, auto and ini.' + - 'Overrides default AND local configs') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run com1DFA workflow") + parser.add_argument( + "avadir", metavar="avalancheDir", type=str, nargs="?", default="", help="the avalanche directory" + ) + parser.add_argument( + "configdir", metavar="configurationDir", type=str, nargs="?", default="", + help="the configuration file directory" + ) + parser.add_argument( + "-fc", + "--friction_calibration", + choices=["auto", "large", "medium", "small", "ini"], + type=str, + default="ini", + help="friction calibration override, possible values are large, medium , small, auto and ini." + + "Overrides default AND local configs", + ) args = parser.parse_args() - runCom1DFA(str(args.avadir), str(args.friction_calibration)) + runCom1DFA(str(args.avadir), str(args.configdir),str(args.friction_calibration)) diff --git a/avaframe/runScripts/runPlotContoursFromAsc.py b/avaframe/runScripts/runPlotContoursFromAsc.py index 04dac701f..46911606b 100644 --- a/avaframe/runScripts/runPlotContoursFromAsc.py +++ b/avaframe/runScripts/runPlotContoursFromAsc.py @@ -26,7 +26,7 @@ def plotContoursFromAsc(cfg, avalancheDir): if __name__ == "__main__": # Load configuration for runPlotContour - cfg = cfgUtils.getModuleConfig(rCon, fileOverride="", modInfo=False, toPrint=False, onlyDefault=False) + cfg = cfgUtils.getModuleConfig(rCon, '', fileOverride="", modInfo=False, toPrint=False, onlyDefault=False) # fetch input directory cfgMain = cfgUtils.getGeneralConfig() diff --git a/docs/configuration.rst b/docs/configuration.rst index 402662068..033aeeda0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -61,4 +61,14 @@ in the module **B** configuration file are used to update the configuration sett the configuration parameters used for one task in one configuration file. An example of this usage can be found in ``ana1Tests/energyLineTestCfg.ini``. +Additionally, there is the option to write ``local_moduleNameCfg.ini`` configuration files for all modules +where a corresponding *collectionName_moduleName_override* section is provided by using +the function: :py:func:`in3Utils.cfgHandling.rewriteLocalCfgs`. These ``local_moduleNameCfg.ini`` files are +saved to ``avalancheDir/Inputs/configurationsOverrides`` by default if no ``localCfgPath`` is provided as input. +An example of this functionality is provided in :py:mod:`runScripts.runPlotContoursFromAsc.py` with the respective +``runPlotContoursFromAscCfg.ini`` file. + +.. Note:: + By using :py:func:`in3Utils.cfgHandling.rewriteLocalCfgs`, already existing ``local_moduleNameCfg.ini`` files in the + respective directory will be overwritten.