Merge pull request #512 from tscheypidi/develop

Added restart points and automatic restart support

CHANGELOG.md

@@ -9,7 +9,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### changed
+-
+
+### added
 - **15_food** half_overweight scenario added
+- **config** added setting cfg$keep_restarts which controls whether restart files should be kept after a run finished
+- **config** changed default for `s_use_gdx` from 2 to 0
+- **scripts** added restart points after each time step from which the model can now be restarted if the simulation aborts at some point
+- **scripts** added SLURM dayMax submission type for standby QOS
+
+
+### removed
+-
+
+### fixed
+-
+
+
+## [4.6.3] - 2023-01-19
+
+### changed
+- **15_food** changed `anthro_iso_jun22` realisation such that results in case of `exo_diet = 1/0` and `exo_waste = 1/0` are identical until 2020
+- **30_crop** identical assumptions for bioenergy until 2020
 - **38_factor_costs** changed name of set `req` to `factors` (also used in 11_costs, 57_maccs, 70_livestock)
 - **38_factos_costs** sticky_labor realization: included option to set a labor share target
 - **62_material** Bioplastic demand identical in all scenarios until 2020

config/default.cfg

@@ -125,7 +125,7 @@ cfg$gms$c_timesteps <- "coup2100"
 cfg$gms$c_past <- "till_2010"
 
 # use of gdx files
-cfg$gms$s_use_gdx <- 2  # def = 2
+cfg$gms$s_use_gdx <- 0  # def = 0
 #*                   0:  gdx will not be loaded
 #*                   1:  gdx is loaded in the first time step
 #*                   2:  gdx is loaded in all time steps
@@ -1807,6 +1807,7 @@ cfg$sequential <- NA
 # *              priority (immediate start, but slots limited to 5 in parallel)
 # *              priority_maxMem (same as priority but with 16 CPUs and max Memory)
 # *              standby (1 week max, preemption possible)
+# *              standby_dayMax (24h max, preemption possible)
 # *              standby_maxMem (same as standby but with 16 CPUs and max Memory)
 # *              NULL (educated guess of best option based
 # *                    available resources)
@@ -1868,6 +1869,10 @@ cfg$model_name <- "MAgPIE"
 # configuration
 cfg$info <- list()
 
+# Should the restart files of each iteration be kept in the output folder (TRUE)
+# or deleted after the run finished (FALSE)
+cfg$keep_restarts <- FALSE
+
 # Should the model run in developer mode? This will loosen some restrictions,
 # such as temporary toleration of coding etiquette violations
 # Please make sure to set it to FALSE for production runs!

core/calculations.gms

@@ -33,11 +33,15 @@ file dummy; dummy.pw=2000; put dummy;
 * clear ct set
 ct(t) = no;
 pt(t) = no;
+
 ***************************TIMESTEP LOOP START**********************************
-loop (t,
+$label TimeLoop
+$if not set TIMESTEP $set TIMESTEP 0
+
+loop (t$(m_year(t) > %TIMESTEP%),
 
 * set ct to current time period
-      ct(t) = yes;
+    ct(t) = yes;
 	  pt(t) = yes$(ord(t) = 1);
 	  pt(t-1) = yes$(ord(t) > 1);
 
@@ -90,6 +94,8 @@ $batinclude "./modules/include.gms" postsolve
   ct(t) = no;
   pt(t) = no$(ord(t) = 1);
   pt(t-1) = no$(ord(t) > 1);
+
+   put_utility 'save' / 'restart_' t.tl:0;;
 ********************************************************************************
 );
 ****************************TIMESTEP LOOP END***********************************

main.gms

@@ -5,6 +5,8 @@
 *** |  MAgPIE License Exception, version 1.0 (see LICENSE file).
 *** |  Contact: magpie@pik-potsdam.de
 
+$if set RESTARTPOINT $goto %RESTARTPOINT%
+
 $title magpie
 
 *' @title MAgPIE - Modelling Framework
@@ -147,24 +149,24 @@ $title magpie
 *##################### R SECTION START (VERSION INFO) ##########################
 * 
 * Used data set: rev4.79_h12_magpie.tgz
-* md5sum: 4f3f5fd72716fe371d646c69c30e6fd3
-* Repository: /p/projects/rd3mod/inputdata/output
+* md5sum: NA
+* Repository: https://rse.pik-potsdam.de/data/magpie/public
 * 
 * Used data set: rev4.79_h12_fd712c0b_cellularmagpie_c200_MRI-ESM2-0-ssp370_lpjml-8e6c5eb1.tgz
-* md5sum: a18af444eb4a3d24956f66e31e8634d8
-* Repository: /p/projects/rd3mod/inputdata/output
+* md5sum: NA
+* Repository: https://rse.pik-potsdam.de/data/magpie/public
 * 
 * Used data set: rev4.79_h12_validation.tgz
-* md5sum: 0a617c2999127a50146ac106cd6ee4bf
-* Repository: /p/projects/rd3mod/inputdata/output
+* md5sum: NA
+* Repository: https://rse.pik-potsdam.de/data/magpie/public
 * 
 * Used data set: additional_data_rev4.36.tgz
-* md5sum: e24c46872f77dc15ad8603bdac1e6065
-* Repository: /p/projects/rd3mod/mirror/rse.pik-potsdam.de/data/magpie/public
+* md5sum: NA
+* Repository: https://rse.pik-potsdam.de/data/magpie/public
 * 
 * Used data set: calibration_H12_09Jan23.tgz
-* md5sum: 0fd18901ec047862918bf598ac126411
-* Repository: /p/projects/rd3mod/mirror/rse.pik-potsdam.de/data/magpie/public
+* md5sum: NA
+* Repository: https://rse.pik-potsdam.de/data/magpie/public
 * 
 * Low resolution: c200
 * High resolution: 0.5
@@ -193,7 +195,7 @@ $title magpie
 * * Call: withCallingHandlers(expr, message = messageHandler, warning = warningHandler,     error = errorHandler)
 * 
 * 
-* Last modification (input data): Tue Jan 17 11:25:32 2023
+* Last modification (input data): Wed Jan 18 12:28:24 2023
 * 
 *###################### R SECTION END (VERSION INFO) ###########################
 

scripts/run_submit/submit.R

@@ -12,6 +12,19 @@ library(gms)
 
 #options(error=function()traceback(2))
 
+.getRestartFiles <- function() {
+  restartFiles <- dir(pattern="restart_.*\\.g00")
+  names(restartFiles) <- gsub("restart_y([0-9]*)\\.g00", "\\1", restartFiles)
+  return(restartFiles)
+}
+
+.getRestartCode <- function() {
+  restartFiles <- .getRestartFiles()
+  if (length(restartFiles) == 0) return("")
+  restartFiles <- tail(restartFiles, 1)
+  return(paste0(" --RESTARTPOINT=TimeLoop --TIMESTEP=", names(restartFiles), " r=", restartFiles))
+}
+
 cfg <- gms::loadConfig("config.yml")
 
 maindir <- cfg$magpie_folder
@@ -20,7 +33,9 @@ maindir <- cfg$magpie_folder
 timeGAMSStart <- Sys.time()
 
 cat("\nStarting MAgPIE...\n")
-system(paste("gams full.gms -errmsg=1 -lf=full.log -lo=",cfg$logoption,sep=""))
+system(paste0("gams full.gms -errmsg=1 -lf=full.log -lo=",cfg$logoption, .getRestartCode()))
+
+if (isFALSE(cfg$keep_restarts)) unlink(.getRestartFiles())
 
 # Capture runtimes
 timeGAMSEnd  <- Sys.time()

scripts/run_submit/submit_standby_dayMax.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+#SBATCH --qos=standby
+#SBATCH --job-name=mag-run
+#SBATCH --output=slurm.log
+#SBATCH --mail-type=END
+#SBATCH --cpus-per-task=3
+#SBATCH --partition=priority
+#SBATCH --time=24:00:00
+
+Rscript submit.R