allow gamscompile to get input data, update regexp in main.gms

CHANGELOG.md

@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **inputs** update of landuse emissions and costs using MAgPIE 4.6.8, mrcommons 1.32.0, input data rev6.543
 - **scripts** MAgPIE coupling interface: replace old MAgPIE cost variable
 - **scripts** MAgPIE coupling interface: remove filtering of negative LU emissions
+- **scripts** `./start.R --gamscompile` now adjust sets and gets input data
 - **core** MAgPIE coupling: tolerate negative values for `n2ofertsom` and deactivate its MAC
 - **05_initialCap** fix overwriting of investment cost changes from cm_inco0Factor switch
 

core/sets.gms

@@ -741,7 +741,7 @@ teEs(all_teEs)           "ES technologies which are actually used (to be filled
 ***######################## R SECTION START (SETS) ###############################
 *** THIS CODE IS CREATED AUTOMATICALLY, DO NOT MODIFY THESE LINES DIRECTLY
 *** ANY DIRECT MODIFICATION WILL BE LOST AFTER NEXT INPUT DOWNLOAD
-*** CHANGES CAN BE DONE USING THE RESPECTIVE LINES IN scripts/start/prepare.R
+*** CHANGES CAN BE DONE USING THE RESPECTIVE LINES IN scripts/start/updateSets.R
 
 sets
 

main.gms

@@ -156,11 +156,12 @@
 *' ;
 *'   param_name = 0;     !! def = 0  !! regexp = 0|1
 *' --------
-*' * The value behind 'def' contains the default value and is added only for the user to remember if changed manually
-*' * The value behind regexp is read by scripts/start/checkFixCfg.R to check the validity of the input.
+*' * def shows the default value, which is added only for the user to remember if changed manually
+*' * regexp is optional, the value is read by scripts/start/checkFixCfg.R to check the validity of the input.
 *' In this case, it checks whether the value fits this regular expression: ^(0|1)$
 *' Use 'value1|value2' for specific values, use '[1-7]' for a row of integers.
-*' Two shortcut are defined: use 'is.numeric' for numeric values and 'is.share' if the value must be >= 0 and <= 1
+*' Three shortcut are defined: use 'is.numeric' for numeric values, 'is.nonnegative' for >= 0,
+*' and 'is.share' if the value must be >= 0 and <= 1
 *'
 *'
 *' #### Other general rules:
@@ -502,7 +503,7 @@ parameter
 parameter
   cm_co2_tax_2020           "level of co2 tax in year 2020 in $ per t CO2eq, makes sense only for emiscen eq 9 and 45_carbonprice exponential"
 ;
-  cm_co2_tax_2020   = -1;              !! def = -1  !! regexp = is.numeric
+  cm_co2_tax_2020   = -1;              !! def = -1  !! regexp = -1|is.nonnegative
 *' * (-1): default setting equivalent to no carbon tax
 *' * (any number >= 0): tax level in 2020, with 5% exponential increase over time
 *'
@@ -1137,7 +1138,7 @@ parameter
 *'
 *' *  (off): off = REMIND expects to be run standalone (emission factors are used, shiftfactors are set to zero)
 *' *  (on): on  = REMIND expects to be run based on a MAgPIE reporting file (emission factors are set to zero because emissions are retrieved from the MAgPIE reporting, shift factors for supply curves are calculated)
-$setglobal cm_MAgPIE_coupling  off     !! def = "off"  !! regexp = on|off
+$setglobal cm_MAgPIE_coupling  off     !! def = "off"  !! regexp = off|on
 *' cm_rcp_scen       "chooses RCP scenario"
 *'
 *' *  (none): no RCP scenario, standard setting
@@ -1193,7 +1194,7 @@ $setglobal cm_tradbio_phaseout  default  !! def = default  !! regexp = default|f
 ***  (off):             (default) no bound
 ***  (100):             (e.g.) set maximum to 100 EJ per year
 ***  (any value ge 0):  set maximum to that value
-$setglobal cm_maxProdBiolc  off  !! def = off  !! regexp = off|is.numeric
+$setglobal cm_maxProdBiolc  off  !! def = off  !! regexp = off|is.nonnegative
 *** cm_bioprod_regi_lim
 *** limit to total biomass production (including residues) by region to an upper value in EJ/yr from 2035 on
 *** example: "CHA 20, EUR_regi 7.5" limits total biomass production in China to 20 EJ/yr and
@@ -1202,7 +1203,7 @@ $setglobal cm_maxProdBiolc  off  !! def = off  !! regexp = off|is.numeric
 *** If you specify a value for a region within a region group (e.g. DEU in EU27_regi),
 *** then the values from the region group disaggregation will be overwritten by this region-specific value.
 *** For example: "EU27_regi 7.5, DEU 1.5".
-$setGLobal cm_bioprod_regi_lim off !! def off
+$setGLobal cm_bioprod_regi_lim off  !! def off
 *** cm_POPscen      "Population growth scenarios from UN data and IIASA projection used in SSP"
 *** pop_SSP1    "SSP1 population scenario"
 *** pop_SSP2    "SSP2 population scenario"

scripts/start/checkFixCfg.R

@@ -22,11 +22,14 @@ checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) {
   code <- system(paste0("grep regexp ", file.path(remindPath, "main.gms")), intern = TRUE)
   # this is used to replace all 'regexp = is.numeric'
   grepisnum <- "((\\+|-)?[0-9]*([0-9]\\.?|\\.?[0-9])[0-9]*)"
+  grepisnonnegative <- "(\\+?[0-9]*([0-9]\\.?|\\.?[0-9])[0-9]*)"
   grepisshare <-  "(\\+?0?\\.[0-9]+|0|0\\.0*|1|1\\.0*)"
   # some simple tests
   if (testmode) {
     stopifnot(all(  grepl(paste0("^", grepisnum, "$"), c("2", "2.2", "32.", "+32.", "+.05", "-0.5", "-.5", "-5", "-7."))))
     stopifnot(all(! grepl(paste0("^", grepisnum, "$"), c("2.2.", "0a", "1e1", ".2.", "ab", "2.3a", "--a", "++2"))))
+    stopifnot(all(  grepl(paste0("^", grepisnonnegative, "$"), c("2", "2.2", "32.", "+32.", "+.05"))))
+    stopifnot(all(! grepl(paste0("^", grepisnonnegative, "$"), c("2.2.", "0a", "1e1", ".2.", "ab", "2.3a", "--a", "++2", "-0.5", "-.5", "-5", "-7."))))
     stopifnot(all(  grepl(paste0("^", grepisshare, "$"), c("0", "0.0", ".000", "1.0", "1.", "1", "0.12341234"))))
     stopifnot(all(! grepl(paste0("^", grepisshare, "$"), c("1.1", "-0.3", "-0", "."))))
   }
@@ -42,6 +45,7 @@ checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) {
       regexp <- paste0("^(", trimws(gsub("!!.*", "", gsub("^.*regexp[ ]*=", "", filtered))), ")$")
       # replace is.numeric by pattern defined above
       useregexp <- gsub("is.numeric", grepisnum, regexp, fixed = TRUE)
+      useregexp <- gsub("is.nonnegative", grepisnonnegative, useregexp, fixed = TRUE)
       useregexp <- gsub("is.share", grepisshare, useregexp, fixed = TRUE)
       # check whether parameter value fits regular expression
       if (! grepl(useregexp, cfg$gms[[n]])) {

scripts/start/prepare.R

@@ -120,15 +120,6 @@ prepare <- function() {
     create_input_for_45_carbonprice_exogenous(as.character(cfg$files2export$start["input_carbonprice.gdx"]))
   }
 
-  cfg$gms$cm_CES_configuration <- calculate_CES_configuration(cfg)
-
-  # write name of corresponding CES file to datainput.gms
-  replace_in_file(file    = "./modules/29_CES_parameters/load/datainput.gms",
-                  content = paste0('$include "',
-                                   "./modules/29_CES_parameters/load/input/",
-                                   cfg$gms$cm_CES_configuration, ".inc\""),
-                  subject = "CES INPUT")
-
   # If a path to a MAgPIE report is supplied use it as REMIND input (used for REMIND-MAgPIE coupling)
   # ATTENTION: modifying gms files
   if (!is.null(cfg$pathToMagpieReport)) {
@@ -169,99 +160,9 @@ prepare <- function() {
     replace_in_file(tmpModelFile, paste("*", content), subject)
   }
 
-  update_sets <- function(map) {
-     .tmp <- function(x,prefix="", suffix1="", suffix2=" /", collapse=",", n=10) {
-      content <- NULL
-      tmp <- lapply(split(x, ceiling(seq_along(x)/n)),paste,collapse=collapse)
-      end <- suffix1
-      for(i in 1:length(tmp)) {
-        if(i==length(tmp)) end <- suffix2
-        content <- c(content,paste0('       ',prefix,tmp[[i]],end))
-      }
-      return(content)
-    }
-    modification_warning <- c(
-      '*** THIS CODE IS CREATED AUTOMATICALLY, DO NOT MODIFY THESE LINES DIRECTLY',
-      '*** ANY DIRECT MODIFICATION WILL BE LOST AFTER NEXT INPUT DOWNLOAD',
-      '*** CHANGES CAN BE DONE USING THE RESPECTIVE LINES IN scripts/start/prepare.R')
-    content <- c(modification_warning,'','sets')
-    # create iso set with nice formatting (10 countries per line)
-    tmp <- lapply(split(map$CountryCode, ceiling(seq_along(map$CountryCode)/10)),paste,collapse=",")
-    regions <- as.character(unique(map$RegionCode))
-    # Creating sets for H12 subregions
-    subsets <- remind2::toolRegionSubsets(map=cfg$regionmapping,singleMatches=TRUE,removeDuplicates=FALSE)
-    if(grepl("regionmapping_21_EU11", cfg$regionmapping, fixed = TRUE)){ #add EU27 region group
-      subsets <- c(subsets,list(
-        "EU27"=c("ENC","EWN","ECS","ESC","ECE","FRA","DEU","ESW"), #EU27 (without Ireland)
-        "NEU_UKI"=c("NES", "NEN", "UKI") #EU27 (without Ireland)
-      ) )
-    }
-    # declare ext_regi (needs to be declared before ext_regi to keep order of ext_regi)
-    content <- c(content, '')
-    content <- c(content, paste('*** Several parts of the REMIND code relies in the order that the regional set is defined.'))
-    content <- c(content, paste('***   Therefore, you must always abide with the below rules:'))
-    content <- c(content, paste('***   - The first regional set to be declared must be the ext_regi set, which includes the model native regions and all possible regional aggregations considered in REMIND.'))
-    content <- c(content, paste('***   - The ext_regi set needs to be declared in the order of more aggregated to less aggregated region order (e.g. World comes first and country regions goes last).'))
-    content <- c(content, paste('***   - IMPORTANT: You CANNOT use any of the ext_regi set elements in any set definition made prior to the ext_regi set declaration in the code.'))
-    content <- c(content, '')
-    content <- c(content, paste('   ext_regi "extended regions list (includes subsets of H12 regions)"'))
-    content <- c(content, '      /')
-    content <- c(content, '        GLO,')
-    content <- c(content, paste0('        ',paste(paste0(names(subsets),"_regi"),collapse=','),","))
-    content <- c(content, paste0('        ',paste(regions,collapse=',')))
-    content <- c(content, '      /')
-    # declare all_regi
-    content <- c(content, '',paste('   all_regi "all regions" /',paste(regions,collapse=','),'/',sep=''),'')
-    # regi_group
-    content <- c(content, '   regi_group(ext_regi,all_regi) "region groups (regions that together corresponds to a H12 region)"')
-    content <- c(content, '      /')
-    content <- c(content, paste0('        ',paste('GLO.(',paste(regions,collapse=','),')')))
-    for (i in 1:length(subsets)){
-        content <- c(content, paste0('        ', paste(c(paste0(names(subsets)[i],"_regi"))), ' .(',paste(subsets[[i]],collapse=','), ')'))
-    }
-    content <- c(content, '      /')
-    content <- c(content, '')
-    # iso countries set
-    content <- c(content,'   iso "list of iso countries" /')
-    content <- c(content, .tmp(map$CountryCode, suffix1=",", suffix2=" /"),'')
-    content <- c(content,'   regi2iso(all_regi,iso) "mapping regions to iso countries"','      /')
-    for(i in as.character(unique(map$RegionCode))) {
-      content <- c(content, .tmp(map$CountryCode[map$RegionCode==i], prefix=paste0(i," . ("), suffix1=")", suffix2=")"))
-    }
-    content <- c(content,'      /')
-    content <- c(content, 'iso_regi "all iso countries and EU and greater China region" /  EUR,CHA,')
-    content <- c(content, .tmp(map$CountryCode, suffix1=",", suffix2=" /"),'')
-    content <- c(content,'   map_iso_regi(iso_regi,all_regi) "mapping from iso countries to regions that represent country" ','         /')
-    for(i in regions[regions %in% c("EUR","CHA",as.character(unique(map$CountryCode)))]) {
-      content <- c(content, .tmp(i, prefix=paste0(i," . "), suffix1="", suffix2=""))
-    }
-    content <- c(content,'      /',';')
-    replace_in_file('core/sets.gms',content,"SETS",comment="***")
-  }
-
   ############ download and distribute input data ########
   # check whether the regional resolution and input data revision are outdated and update data if needed
-  if(file.exists("input/source_files.log")) {
-      input_old     <- readLines("input/source_files.log")[c(1,2,3)]
-  } else {
-      input_old     <- "no_data"
-  }
-  input_new      <- c(paste0("rev",cfg$inputRevision,"_", madrat::regionscode(cfg$regionmapping),"_", tolower(cfg$model_name),".tgz"),
-                      paste0("rev",cfg$inputRevision,"_", madrat::regionscode(cfg$regionmapping),ifelse(cfg$extramappings_historic == "","",paste0("-", madrat::regionscode(cfg$extramappings_historic))),"_", tolower(cfg$validationmodel_name),".tgz"),
-                      paste0("CESparametersAndGDX_",cfg$CESandGDXversion,".tgz"))
-  # download and distribute needed data
-  if (! setequal(input_new, input_old) || isTRUE(cfg$force_download)) {
-      message(if (isTRUE(cfg$force_download)) "You set 'cfg$force_download = TRUE'"
-              else "Your input data are outdated or in a different regional resolution",
-              ". New input data are downloaded and distributed.")
-      download_distribute(files        = input_new,
-                          repositories = cfg$repositories, # defined in your environment variables
-                          modelfolder  = ".",
-                          debug        = FALSE,
-			  stopOnMissing = TRUE)
-  } else {
-      message("No input data downloaded and distributed. To enable that, delete input/source_files.log or set cfg$force_download to TRUE.")
-  }
+  cfg <- updateInputData(cfg, remindPath = ".")
 
   # extract BAU emissions for NDC runs to set up emission goals for region where only some countries have a target
   if (isTRUE(cfg$gms$carbonprice == "NDC") || isTRUE(cfg$gms$carbonpriceRegi == "NDC")) {
@@ -273,10 +174,9 @@ prepare <- function() {
   ############ update information ########################
   # update_info, which regional resolution and input data revision in tmpModelFile
   update_info(madrat::regionscode(cfg$regionmapping), cfg$inputRevision, cfg$model_version)
-  # update_sets, which is updating the region-depending sets in core/sets.gms
+  # updateSets, which is updating the region-depending sets in core/sets.gms
   #-- load new mapping information
-  map <- read.csv(cfg$regionmapping, sep=";")
-  update_sets(map)
+  updateSets(cfg)
 
   ########################################################
   ### PROCESSING INPUT DATA ###################### END ###

scripts/start/runGamsCompile.R

@@ -9,9 +9,10 @@
 #' @param modelFile filename of model file to be compiled
 #' @param cfg list with REMIND configuration
 #' @param interactive boolean, if TRUE, will ask user to compile again after fails
+#' @param testmode boolean. In test mode, don't update sets and input data
 #' @author Oliver Richters
 #' @return boolean whether compilation was successful
-runGamsCompile <- function(modelFile, cfg, interactive = TRUE) {
+runGamsCompile <- function(modelFile, cfg, interactive = TRUE, testmode = FALSE) {
   # Define colors for output
   red   <- "\033[0;31m"
   green <- "\033[0;32m"
@@ -21,6 +22,11 @@ runGamsCompile <- function(modelFile, cfg, interactive = TRUE) {
   tmpModelFile <- file.path(gcdir, paste0("main_", cfg$title, ".gms"))
   tmpModelLst <- gsub("gms$", "lst", tmpModelFile)
   file.copy(modelFile, tmpModelFile, overwrite = TRUE)
+  if (! testmode) {
+    unlink(file.path("modules", c("45_carbonprice", "46_carbonpriceRegi"), "NDC", "input", "pm_BAU_reg_emi_wo_LU_bunkers.cs4r"))
+    updateSets(cfg)
+    updateInputData(cfg, verbose = FALSE)
+  }
   lucode2::manipulateConfig(tmpModelFile, cfg$gms)
   exitcode <- system2(
     command = cfg$gamsv,

scripts/start/updateInputData.R

@@ -0,0 +1,35 @@
+updateInputData <- function(cfg, remindPath = ".", verbose = TRUE) {
+  # write name of corresponding CES file to datainput.gms
+
+  cfg$gms$cm_CES_configuration <- calculate_CES_configuration(cfg)
+  replace_in_file(file    = file.path(remindPath, "modules/29_CES_parameters/load/datainput.gms"),
+                  content = paste0('$include "',
+                                   "./modules/29_CES_parameters/load/input/",
+                                   cfg$gms$cm_CES_configuration, ".inc\""),
+                  subject = "CES INPUT")
+
+
+  if(file.exists("input/source_files.log")) {
+      input_old     <- readLines(file.path(remindPath, "input/source_files.log"))[c(1,2,3)]
+  } else {
+      input_old     <- "no_data"
+  }
+  input_new      <- c(paste0("rev",cfg$inputRevision,"_", madrat::regionscode(cfg$regionmapping),"_", tolower(cfg$model_name),".tgz"),
+                      paste0("rev",cfg$inputRevision,"_", madrat::regionscode(cfg$regionmapping),ifelse(cfg$extramappings_historic == "","",paste0("-", madrat::regionscode(cfg$extramappings_historic))),"_", tolower(cfg$validationmodel_name),".tgz"),
+                      paste0("CESparametersAndGDX_",cfg$CESandGDXversion,".tgz"))
+  # download and distribute needed data
+  if (! setequal(input_new, input_old) || isTRUE(cfg$force_download)) {
+      message(if (isTRUE(cfg$force_download)) "You set 'cfg$force_download = TRUE'"
+              else "Your input data are outdated or in a different regional resolution",
+              ". New input data are downloaded and distributed.")
+      download_distribute(files        = input_new,
+                          repositories = cfg$repositories, # defined in your environment variables
+                          modelfolder  = remindPath,
+                          debug        = FALSE,
+                          stopOnMissing = TRUE)
+  } else if (verbose) {
+      message("No input data downloaded and distributed. To enable that, delete input/source_files.log or set cfg$force_download to TRUE.")
+  }
+
+  return(cfg)
+}