improvements to coupled start scripts

config/tests/scenario_config_coupled_shortCascade.csv

@@ -1,4 +1,4 @@
 title;start;qos;sbatch;magpie_scen;magpie_empty;no_ghgprices_land_until;max_iterations;oldrun;path_gdx;path_gdx_ref;path_gdx_bau;path_report;cm_nash_autoconverge_lastrun;path_mif_ghgprice_land
-TESTTHAT-SSP2EU-Base;1;priority;--wait;SSP2|NPI;TRUE;y2150;2;;;;;;2;
-TESTTHAT-SSP2EU-NDC;1;priority;--wait;SSP2|NDC;TRUE;y2150;2;;;;;;2;TESTTHAT-SSP2EU-Base
-TESTTHAT-SSP2EU-Policy;2;priority;--wait;SSP2|NDC;TRUE;y2150;2;TESTTHAT-SSP2EU-Base;;;;;;output/C_TESTTHAT-SSP2EU-Base-rem-1/REMIND_generic_C_TESTTHAT-SSP2EU-Base-rem-1.mif
+TESTTHAT-SSP2EU-Base;1;auto;--wait;SSP2|NPI;TRUE;y2150;2;;;;;;2;
+TESTTHAT-SSP2EU-NDC;1;auto;--wait;SSP2|NDC;TRUE;y2150;2;;;;;;2;TESTTHAT-SSP2EU-Base
+TESTTHAT-SSP2EU-Policy;2;auto;--wait;SSP2|NDC;TRUE;y2150;2;TESTTHAT-SSP2EU-Base;;;;;;output/C_TESTTHAT-SSP2EU-Base-rem-1/REMIND_generic_C_TESTTHAT-SSP2EU-Base-rem-1.mif

scripts/start/getReportData.R

@@ -123,7 +123,11 @@ getReportData <- function(path_to_report,inputpath_mag="magpie",inputpath_acc="c
   }
   .agriculture_costs <- function(mag){
     notGLO <- getRegions(mag)[!(getRegions(mag)=="GLO")]
-    out <- mag[,,"Costs Without Incentives (million US$05/yr)"]/1000/1000 # with transformation factor from 10E6 US$2005 to 10E12 US$2005
+    if ("Costs Without Incentives (million US$05/yr)" %in% getNames(mag)) {
+      out <- mag[,,"Costs Without Incentives (million US$05/yr)"]/1000/1000 # with transformation factor from 10E6 US$2005 to 10E12 US$2005
+    } else {
+      out <- mag[,,"Costs|MainSolve w/o GHG Emissions (million US$05/yr)"]/1000/1000 # old reporting
+    }
     out["JPN",is.na(out["JPN",,]),] <- 0
     dimnames(out)[[3]] <- NULL #Delete variable name to prevent it from being written into output file
     write.magpie(out[notGLO,,],paste0("./modules/26_agCosts/",inputpath_acc,"/input/p26_totLUcost_coupling.csv"),file_type="csvr")

start_bundle_coupled.R

@@ -52,7 +52,8 @@ helpText <- "
 
 # Please provide all files and paths relative to the folder where start_coupled is executed
 path_remind <- getwd()   # provide path to REMIND. Default: the actual path which the script is started from
-path_magpie <- normalizePath(file.path(getwd(), "..", "magpie"))
+path_magpie <- normalizePath(file.path(getwd(), "magpie"), mustWork = FALSE)
+if (! dir.exists(path_magpie)) path_magpie <- normalizePath(file.path(getwd(), "..", "magpie"))
 
 # Paths to the files where scenarios are defined
 # path_settings_remind contains the detailed configuration of the REMIND scenarios
@@ -84,6 +85,9 @@ max_iterations <- 5
 # Afterwards REMIND runs for "n600_iterations" iterations with results from higher resolution.
 n600_iterations <- 0 # max_iterations
 
+# Use this qos if none is specified in config file. "auto" means use "priority" slot if available, else "short"
+qos_default <- "auto"
+
 # run a compareScenario for each scenario comparing all rem-x: Choose qos (short, priority) or set to FALSE
 run_compareScenarios <- "short"
 
@@ -139,8 +143,9 @@ if (length(argv) > 0) {
   if (!all(file_exists)) stop("Unknown parameter provided: ", paste(argv[!file_exists], collapse = ", "))
   # set config file to not known parameter where the file actually exists
   path_settings_coupled <- file.path(path_remind, argv[[1]])
-  if (! grep("scenario_config_coupled", path_settings_coupled))
-    stop("Enter only a scenario_config_coupled* file via command line or set all files manually in start_bundle_coupled.R")
+  if (! isTRUE(grepl("scenario_config_coupled", path_settings_coupled)))
+    stop("Enter only a scenario_config_coupled* file via command line or set all files manually in start_bundle_coupled.R.\n",
+         "Your command line arguments were: ", paste0(argv, collapse = " "))
   path_settings_remind  <- sub("scenario_config_coupled", "scenario_config", path_settings_coupled)
 } else if (! file.exists(path_settings_coupled)) {
   possiblecsv <- Sys.glob(c(file.path("config", "scenario_config_coupled*.csv"),
@@ -177,6 +182,7 @@ errorsfound <- 0
 startedRuns <- 0
 finishedRuns <- 0
 waitingRuns <- 0
+qosRuns <- NULL
 deletedFolders <- 0
 
 stamp <- format(Sys.time(), "_%Y-%m-%d_%H.%M.%S")
@@ -262,10 +268,11 @@ for (scen in common) {
   }
 }
 
-if (file.exists("/p") && "qos" %in% names(scenarios_coupled)
-    && sum(scenarios_coupled[common, "qos"] == "priority", na.rm = TRUE) > 4) {
-      message("\nAttention, you want to start more than 4 runs with qos=priority mode.")
-      message("They may not be able to run in parallel on the PIK cluster.")
+if (! "qos" %in% names(scenarios_coupled)) scenarios_coupled[, "qos"] <- qos_default
+scenarios_coupled[, "qos"] <- ifelse(is.na(scenarios_coupled[, "qos"]), qos_default, scenarios_coupled[, "qos"])
+if (file.exists("/p") && sum(scenarios_coupled[common, "qos"] == "priority", na.rm = TRUE) > 4) {
+  message("\nAttention, you want to start more than 4 runs with qos=priority mode.")
+  message("They may not be able to run in parallel on the PIK cluster.")
 }
 
 ####################################################
@@ -280,7 +287,7 @@ for(scen in common){
   runname      <- paste0(prefix_runname, scen)            # name of the run that is used for the folder names
   path_report  <- NULL                                    # sets the path to the report REMIND is started with in the first loop
   qos          <- scenarios_coupled[scen, "qos"]          # set the SLURM quality of service (priority/short/medium/...)
-  if(is.null(qos) || is.na(qos)) qos <- "auto"            # if qos could not be found in scenarios_coupled use short/priority
+  qosRuns[qos] <- if (isTRUE(qosRuns[qos] > 0)) qosRuns[qos] + 1 else 1 # count
   sbatch       <- scenarios_coupled[scen, "sbatch"]       # retrieve sbatch options from scenarios_coupled
   if (is.null(sbatch) || is.na(sbatch)) sbatch <- ""      # if sbatch could not be found in scenarios_coupled use empty string
   start_iter_first <- 1                                   # iteration to start the coupling with
@@ -523,10 +530,14 @@ for(scen in common){
     }
     foldername <- file.path("output", fullrunname)
     if ((i > start_iter_first || !scenarios_coupled[scen, "start_magpie"]) && file.exists(foldername)) {
-      if (errorsfound == 0) {
-        if (! "--test" %in% flags) unlink(foldername, recursive = TRUE, force = TRUE)
-        message("Delete ", foldername, if ("--test" %in% flags) " if not in test mode", ". ", appendLF = FALSE)
-        deletedFolders <- deletedFolders + 1
+      if (errorsfound == 0 && ! any(c("--test", "--gamscompile") %in% flags)) {
+        message("Folder ", foldername, " exists but incomplete. Delete it and rerun (else will be skipped)? y/N")
+        if (tolower(gms::getLine()) %in% c("y", "yes")) {
+          unlink(foldername, recursive = TRUE, force = TRUE)
+          deletedFolders <- deletedFolders + 1
+        } else {
+          start_now <- FALSE
+        }
       }
     }
 
@@ -616,8 +627,9 @@ for (scen in common) {
           sq <- system(paste0("squeue -u ", Sys.info()[["user"]], " -o '%q %j'"), intern = TRUE)
           runEnv$qos <- if (is.null(attr(sq, "status")) && sum(grepl("^priority ", sq)) < 4) "priority" else "short"
         }
-        slurm_command <- paste0("sbatch --qos=", runEnv$qos, " --mem=8000 --job-name=", fullrunname,
+        slurm_command <- paste0("sbatch --qos=", runEnv$qos, " --job-name=", fullrunname,
         " --output=", logfile, " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", runEnv$numberOfTasks,
+        if (runEnv$numberOfTasks == 1) " --mem=8000", " ", runEnv$sbatch,
         " ", runEnv$sbatch, " --wrap=\"Rscript start_coupled.R coupled_config=", Rdatafile, "\"")
         message(slurm_command)
         exitCode <- system(slurm_command)
@@ -646,9 +658,13 @@ if (! "--test" %in% flags && ! "--gamscompile" %in% flags) {
   message(cs_command)
 }
 
-message("\nDone: ", finishedRuns, " runs already finished. ", deletedFolders, " folders deleted. ",
-        startedRuns, " runs started. ", waitingRuns, " runs are waiting.",
-        if("--test" %in% flags) "\nYou are in TEST mode, only RData files were written.")
+message("#### Summary ####")
+message("\nDone.", if(any(c("--test", "--gamscompile") %in% flags)) " You are in TEST or gamscompile mode, no runs were actually started.")
+message("- ", finishedRuns, " runs already finished.")
+message("- ", deletedFolders, " folders deleted.")
+message("- ", startedRuns, " runs started.")
+message("- ", waitingRuns, " runs are waiting.")
+message("qos statistics: ", paste0(names(qosRuns), ": ", qosRuns, collapse = ", "), ".")
 # make sure we have a non-zero exit status if there were any errors
 if (0 < errorsfound) {
   stop(red, errorsfound, NC, " errors were identified, check logs above for details.")

start_coupled.R

@@ -300,8 +300,9 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m
           sq <- system(paste0("squeue -u ", Sys.info()[["user"]], " -o '%q %j' | grep -v ", fullrunname), intern = TRUE)
           subseq.env$qos <- if (is.null(attr(sq, "status")) && sum(grepl("^priority ", sq)) < 4) "priority" else "short"
         }
-        subsequentcommand <- paste0("sbatch --qos=", subseq.env$qos, " --mem=8000 --job-name=", subseq.env$fullrunname, " --output=", logfile,
+        subsequentcommand <- paste0("sbatch --qos=", subseq.env$qos, " --job-name=", subseq.env$fullrunname, " --output=", logfile,
         " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", subseq.env$numberOfTasks,
+        if (subseq.env$numberOfTasks == 1) " --mem=8000",
         " ", subseq.env$sbatch, " --wrap=\"Rscript start_coupled.R coupled_config=", RData_file, "\"")
         message(subsequentcommand)
         if (length(needfulldatagdx) > 0) {
@@ -344,7 +345,8 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m
       source_include <- TRUE
       runs <- runname
       folder <- "./output"
-      source("scripts/output/comparison/plot_compare_iterations.R", local = TRUE)
+      pci <- try(source("scripts/output/comparison/plot_compare_iterations.R", local = TRUE))
+      if (inherits(pci, "try-error")) errorsfound <- errorsfound + 1
       cs_runs <- findIterations(runname, modelpath = remindpath, latest = FALSE)
       cs_name <- paste0("compScen-rem-1-", max_iterations, "_", runname)
       cs_qos <- if (!isFALSE(run_compareScenarios)) run_compareScenarios else "short"
@@ -362,8 +364,8 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m
       }
     }
   }
-  if (errorsfound > 0) stop(errorsfound, " errors found, check the logs.")
   message("### start_coupled() finished. ###")
+  if (errorsfound > 0) stop(errorsfound, " errors found, check the logs.")
 }
 
 ##################################################################

tests/testthat/test_20-coupled.R

@@ -7,7 +7,8 @@
 
 skipIfFast()
 coupledConfig <- "config/tests/scenario_config_coupled_shortCascade.csv"
-magpie_folder <- "../../../magpie"
+magpie_folder <- "../../magpie"
+if (! dir.exists(magpie_folder)) magpie_folder <- paste0("../", magpie_folder)
 config <- readCheckScenarioConfig(file.path("..", "..", coupledConfig), remindPath <- file.path("..", ".."))
 max_iterations <- if ("max_iterations" %in% names(config)) max(config$max_iterations) else 5
 # for a fresh run, delete all left-overs from previous test
@@ -62,7 +63,7 @@ test_that("using start_bundle_coupled.R --test works", {
                          env = paste0("R_PROFILE_USER=", Rprofile))
   printIfFailed(output)
   expectSuccessStatus(output)
-  expect_true(any(grepl("TEST mode", output)))
+  expect_true(any(grepl("TEST or gamscompile mode", output)))
   expect_true(any(grepl("NOT submitted", output)))
   for (scen in rownames(config)[config$start == 1]) {
     expect_true(any(grepl(paste0("starting with C_", scen, "-rem-1"), output)))