undo 'increase memory on single node runs' remindmodel#1567

orichters · Mar 12, 2024 · 3df4e94 · 3df4e94
1 parent 34bd6e2
commit 3df4e94
Show file tree

Hide file tree

Showing 7 changed files with 18 additions and 14 deletions.
diff --git a/.gitignore b/.gitignore
@@ -58,6 +58,10 @@ core/magicc/
 !core/magicc/magicc_scen_450.inc
 !core/magicc/magicc_scen_550.inc
 
+# ignore climate assessment data
+scripts/utils/climate_assessment/PYTHONLOG*.out
+scripts/utils/climate_assessment/output_climate/
+
 # ignore GDX files in the gdx-files folder
 config/gdx-files/*.gdx
 

diff --git a/DESCRIPTION b/DESCRIPTION
@@ -51,15 +51,15 @@ Imports:
     nleqslv,
     optparse,
     piamenv (>= 0.4.0),
-    piamInterfaces (>= 0.13.14),
+    piamInterfaces (>= 0.15.1),
     plotly,
     purrr,
     quitte (>= 0.3128.4),
     R.utils,
     raster,
     readr,
     readxl,
-    remind2 (>= 1.135.6),
+    remind2 (>= 1.135.11),
     renv,
     reshape2,
     reticulate,

diff --git a/output.R b/output.R
@@ -47,7 +47,7 @@ helpText <- "
 #'                     folders, comma-separated, such as remind_dir=.,../otherremind
 #'   slurmConfig=      use slurmConfig=priority, short or standby to specify slurm
 #'                     selection. You may also pass complicated arguments such as
-#'                     slurmConfig='--qos=priority --mem=16000'
+#'                     slurmConfig='--qos=priority --mem=8000'
 "
 
 argv <- get0("argv", ifnotfound = commandArgs(trailingOnly = TRUE))
@@ -86,8 +86,8 @@ if ("--help" %in% flags) {
 
 choose_slurmConfig_output <- function(output) {
   slurm_options <- c("--qos=priority", "--qos=short", "--qos=standby",
-                     "--qos=priority --mem=16000", "--qos=short --mem=16000",
-                     "--qos=standby --mem=16000", "--qos=priority --mem=32000")
+                     "--qos=priority --mem=8000", "--qos=short --mem=8000",
+                     "--qos=standby --mem=8000", "--qos=priority --mem=32000")
 
   if (!isSlurmAvailable())
     return("direct")

diff --git a/scripts/start/choose_slurmConfig.R b/scripts/start/choose_slurmConfig.R
@@ -52,16 +52,16 @@ choose_slurmConfig <- function(identifier = FALSE, flags = NULL) {
                     "1" = "--qos=standby --nodes=1 --tasks-per-node=12"  , # SLURM standby  - task per node: 12 (nash H12) [recommended]
                     "2" = "--qos=standby --nodes=1 --tasks-per-node=13"  , # SLURM standby  - task per node: 13 (nash H12 coupled)
                     "3" = "--qos=standby --nodes=1 --tasks-per-node=16"  , # SLURM standby  - task per node: 16 (nash H12+)
-                    "4" = "--qos=standby --nodes=1 --tasks-per-node=1 --mem=16000"   , # SLURM standby  - task per node:  1 (nash debug, test one regi)
+                    "4" = "--qos=standby --nodes=1 --tasks-per-node=1 --mem=8000"   , # SLURM standby  - task per node:  1 (nash debug, test one regi)
                     "5" = "--qos=priority --nodes=1 --tasks-per-node=12" , # SLURM priority - task per node: 12 (nash H12) [recommended]
                     "6" = "--qos=priority --nodes=1 --tasks-per-node=13" , # SLURM priority - task per node: 13 (nash H12 coupled)
                     "7" = "--qos=priority --nodes=1 --tasks-per-node=16" , # SLURM priority - task per node: 16 (nash H12+)
-                    "8" = "--qos=priority --nodes=1 --tasks-per-node=1 --mem=16000"  , # SLURM priority - task per node:  1 (nash debug, test one regi)
+                    "8" = "--qos=priority --nodes=1 --tasks-per-node=1 --mem=8000"  , # SLURM priority - task per node:  1 (nash debug, test one regi)
                     "9" = "--qos=short --nodes=1 --tasks-per-node=12"    , # SLURM short    - task per node: 12 (nash H12)
                    "10" = "--qos=short --nodes=1 --tasks-per-node=16"    , # SLURM short    - task per node: 16 (nash H12+)
-                   "11" = "--qos=short --nodes=1 --tasks-per-node=1 --mem=16000"     , # SLURM short    - task per node:  1 (nash debug, test one regi)
-                   "12" = "--qos=medium --nodes=1 --tasks-per-node=1 --mem=16000"    , # SLURM medium   - task per node:  1 (negishi)
-                   "13" = "--qos=long --nodes=1 --tasks-per-node=1 --mem=16000"      , # SLURM long     - task per node:  1 (negishi)
+                   "11" = "--qos=short --nodes=1 --tasks-per-node=1 --mem=8000"     , # SLURM short    - task per node:  1 (nash debug, test one regi)
+                   "12" = "--qos=medium --nodes=1 --tasks-per-node=1 --mem=8000"    , # SLURM medium   - task per node:  1 (negishi)
+                   "13" = "--qos=long --nodes=1 --tasks-per-node=1 --mem=8000"      , # SLURM long     - task per node:  1 (negishi)
                    "14" = "--qos=medium --nodes=1 --tasks-per-node=12"   , # SLURM medium   - task per node: 12 (nash long calibration)
                    "15" = "--qos=medium --nodes=1 --tasks-per-node=16"   , # SLURM medium   - task per node: 16 (nash long calibration)
                    "16" = "direct")

diff --git a/start_bundle_coupled.R b/start_bundle_coupled.R
@@ -641,7 +641,7 @@ for (scen in common) {
       }
       slurmOptions <- combine_slurmConfig(paste0("--qos=", runEnv$qos, " --job-name=", fullrunname, " --output=", logfile,
         " --open-mode=append --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", runEnv$numberOfTasks,
-        if (runEnv$numberOfTasks == 1) " --mem=16000"), runEnv$sbatch)
+        if (runEnv$numberOfTasks == 1) " --mem=8000"), runEnv$sbatch)
       slurmCommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", Rdatafile, "\"")
       message(slurmCommand)
       exitCode <- system(slurmCommand)

diff --git a/start_coupled.R b/start_coupled.R
@@ -260,7 +260,7 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m
         }
         slurmOptions <- combine_slurmConfig(paste0("--qos=", subseq.env$qos, " --job-name=", subseq.env$fullrunname, " --output=", logfile,
            " --open-mode=append --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", subseq.env$numberOfTasks,
-          if (subseq.env$numberOfTasks == 1) " --mem=16000"), subseq.env$sbatch)
+          if (subseq.env$numberOfTasks == 1) " --mem=8000"), subseq.env$sbatch)
         subsequentcommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", RData_file, "\"")
         message(subsequentcommand)
         if (length(needfulldatagdx) > 0) {

diff --git a/tests/testthat/test_06-output.R b/tests/testthat/test_06-output.R
@@ -4,12 +4,12 @@
 # |  AGPL-3.0, you are granted additional permissions described in the
 # |  REMIND License Exception, version 1.0 (see LICENSE file).
 # |  Contact: remind@pik-potsdam.de
-skipIfFast()
+# skipIfFast()
 skipIfPreviousFailed()
 
 test_that("output.R -> single -> reporting works", {
   output <- localSystem2("Rscript", c("output.R", "comp=single", "output=reporting", "outputdir=output/testOneRegi",
-                                      "slurmConfig='--qos=priority --mem=16000 --wait'"))
+                                      "slurmConfig='--qos=priority --mem=8000 --wait'"))
   printIfFailed(output)
   expectSuccessStatus(output)
   expect_true(file.exists("../../output/testOneRegi/REMIND_generic_testOneRegi.mif"))