From da35970b27519b689b90abaf964220e9597073c6 Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Fri, 13 Jan 2023 15:50:30 -0700 Subject: [PATCH 01/11] initial forager configs --- simforager_null.config | 117 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 simforager_null.config diff --git a/simforager_null.config b/simforager_null.config new file mode 100644 index 0000000..f5519c6 --- /dev/null +++ b/simforager_null.config @@ -0,0 +1,117 @@ +; These are version 1.0 test of 'null' forager parameters for SIMCoV +; This means a parameter-based translation of SIMCoV into SIMForager without yet +; changing any of the existing SIMCoV code base + +; Dimensions: x y z (assuming grid spacing of 5 microns) +dim = 15000 15000 1 +; >> resulting in 225000000 total environment (epithelial) cells + +; Whole lung dimensions: x y z (assuming grid spacing of 5 microns) +whole-lung-dim = 48000 40000 20000 + +; Number of timesteps (assuming 1 min per timestep) +; Set to arbitrary small number to start +timesteps = 1000 + +; Location of initial infections (omit for no infections); can be: +; list of space-separated coords x,y,z,timestep +; uniform:N (where N is an int, for N uniformly distributed starting points at time 0) +; random:N (where N is an int, for N randomly distributed starting points at time 0) +infection-coords = uniform:22500 +; >> this is .001% of total grid cells + +; Number of virions at initial infection locations +; Chosen to guarantee that at least one cell is infected at the start - see infectivity +initial-infection = 1000 +; >> combined with number of initial infection sites, this should yield a 1:100 ratio of grid cells to algae cells + +; Average number of time steps to expressing virions after cell is infected +incubation-period = 0 + +; Average number of time steps to death after apoptosis is induced +apoptosis-period = 1000000 + +; Average number of time steps to death after a cell starts expresssing +expressing-period = 1000000 + +; Factor multiplied by number of virions to determine probability of infection +infectivity = 0.0 + +; Multiplier reducing infectivity where inflammatory signal is present +; (the multiplier reducing infectivity should be between 0.9 and 0.7 based on values +; in literature demonstrating a reduction of cell death in vitro with IFN) +infectivity-multiplier = 1.0 +; >> irrelevant + +; Number of virions produced by expressing cell each time step +virion-production = 1.1 +; this is SIMCoV default we're going with for now... + +; Multiplier reducing virion production rate where inflammatory signal is present +; (the reduction should be represented by a multiplier betweeon 0.05 and 0.15) +virion-production-multiplier = 1.0 + +; Fraction by which virion count drops each time step +virion-clearance = 0.0 +; >> effectively algae death rate; not considered here + +; Fraction of virions that diffuse into all neighbors each time step +virion-diffusion = 0.001 + +; Amount of chemokine produced by expressing cells each time step +chemokine-production = 0.0 + +; Amount by which chemokine concentration drops each time step +chemokine-decay = 0.0 + +; Fraction of chemokine concentration that diffuses into all neighbors each time step +chemokine-diffusion = 0.0 + +; Minimum chemokine concentration that triggers a T cell +min-chemokine = 0.0 +; >> this should allow fish (T-cells) to be present at start + +; Impact of antibodies; multiplier for virion decay (setting to 1 means this has no effect) +antibody-factor = 1 + +; Number of time steps before antibodies start to be produced +antibody-period = 1000000 +; >> WHAT!? NOBODY INVITED ANTIBODIES TO A REEF + +; Number of tcells generated at each timestep for the whole lung +; This is scaled up 5000x from the mouse model (20 per minute) +tcell-generation-rate = 0 +; >> for a stable population of fish (T-cells) throughout sim + +; Number of time steps before T cells start to be produced +tcell-initial-delay = 0 + +; Average number of time steps to death for a T cell in the vasculature +tcell-vascular-period = 1 + +; Average number of time steps to death after a T cell extravasates +tcell-tissue-period = 1000000 + +; Number of time steps a T cell is bound to an epithelial cell when inducing apoptosis +tcell-binding-period = 0.5 + +; Max probability of a T cell binding to an infected cell in one time step +max-binding-prob = 1 + +; T cells in tissue follow the chemokine gradient +tcells-follow-gradient = false + +; Random seed +seed = 29 + +; Number of timesteps between samples (set to 0 to disable sampling) +sample-period = 0 + +; Resolution for sampling +sample-resolution = 1 + +; Max. block dimension - larger means more locality but worse load balance. Set to 0 for largest possible. +max-block-dim = 10 + +; Output directory (automatically generated) +; output = results From 7b735fb8d1008484c314a165be9cf4a4dba62cfa Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Fri, 13 Jan 2023 15:58:16 -0700 Subject: [PATCH 02/11] Update README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5e2f00c..de879a8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +This repo is a forked adaptation of: + simcov Copyright (c) 2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy), Arizona State @@ -14,9 +16,9 @@ its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software to reproduce, distribute copies to the public, prepare derivative works, and perform publicly and display publicly, and to permit others to do so. -# SimCov # +# SimForager # -This is a model for simulating the response of the immune system to infections in the lungs. +This is a model for simulating large amounts of [currently homogeneous] foragers. ## Installing and building @@ -24,7 +26,7 @@ It requires [UPC++](https://bitbucket.org/berkeleylab/upcxx/wiki/Home), C++ and This repo contains a submodule, so to install, it's best to run -`git clone --recurse-submodules git@github.com:AdaptiveComputationLab/simcov.git` +`git clone --recurse-submodules git@github.com:cswritlarge/simforager.git` to fully initialize the submodules. From ec56185361f60a1c6e569c113f528d09776d172f Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Fri, 20 Jan 2023 14:24:03 -0700 Subject: [PATCH 03/11] find cov replace forager except version.h --- CMakeLists.txt | 14 +++++++------- README.md | 11 +++++------ build_hopper.sh | 8 ++++---- configs/figure-7/simcov.config | 2 +- include/options.hpp | 20 ++++++++++---------- simforager_null.config | 2 +- slurm_hopper.sh | 4 ++-- src/CMakeLists.txt | 6 +++--- 8 files changed, 33 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb2c3b4..8f82135 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,16 +1,16 @@ # The name of our project is "MHMxx". CMakeLists files in this project can -# refer to the root source directory of the project as ${SIMCOV_SOURCE_DIR} and -# to the root binary directory of the project as ${SIMCOV_BINARY_DIR} +# refer to the root source directory of the project as ${SIMFORAGER_SOURCE_DIR} and +# to the root binary directory of the project as ${SIMFORAGER_BINARY_DIR} cmake_minimum_required(VERSION 3.10 FATAL_ERROR) -project(SimCov) +project(SimForager) #----------------------------------------------------------------------------------------# # project specification #----------------------------------------------------------------------------------------# -file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/VERSION SIMCOV_BASE_VERSION) +file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/VERSION SIMFORAGER_BASE_VERSION) # use PROJECT_SOURCE_DIR, not CMAKE_SOURCE_DIR -set(SIMCOV_SOURCE_DIR ${PROJECT_SOURCE_DIR}) -set(SIMCOV_BINARY_DIR ${PROJECT_BINARY_DIR}) +set(SIMFORAGER_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(SIMFORAGER_BINARY_DIR ${PROJECT_BINARY_DIR}) # include modules set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake/Modules) @@ -55,7 +55,7 @@ CHECK_SUBMODULES(upcxx-utils) # set version include_directories(include) -GET_GIT_VERSION(SIMCOV) +GET_GIT_VERSION(SIMFORAGER) # add upcxx-utils if (DEFINED ENV{UPCXX_UTILS_DIR} AND EXISTS $ENV{UPCXX_UTILS_DIR}) diff --git a/README.md b/README.md index de879a8..38cc6d1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -This repo is a forked adaptation of: +simforager is a forked adaptation of: simcov Copyright (c) 2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of @@ -40,22 +40,21 @@ or The executable will be installed in -`/install/bin` +`/install/bin` ## Running To run, execute -`upcxx-run -n -N -- simcov` +`upcxx-run -n -N -- simforager` To see the parameters available, run with `-h`. -It will create an output directory, which will contain a detailed log file (`simcov.log`). It will also create a file containing -all the configuration parameters (`simcov.config`). +It will create an output directory, which will contain a detailed log file (`simforager.log`). It will also create a file containing all the configuration parameters (`simforager.config`). A run can also be executed with a config file as -`upcxx-run -n -N -- simcov --config ` +`upcxx-run -n -N -- simforager --config ` The config file consists of a list of all the command line options as key-value pairs, with semi-colons denoting comments. For example: diff --git a/build_hopper.sh b/build_hopper.sh index 84a5dfa..2a07591 100644 --- a/build_hopper.sh +++ b/build_hopper.sh @@ -9,8 +9,8 @@ module load cmake/3.11.4-qkyj module load openmpi/4.1.3-j6zb module load upcxx/2020.10.0-6eh2 -if [ -n "$SIMCOV_BUILD_ENV" ]; then - source $SIMCOV_BUILD_ENV +if [ -n "$SIMFORAGER_BUILD_ENV" ]; then + source $SIMFORAGER_BUILD_ENV fi upcxx_exec=`which upcxx` @@ -32,11 +32,11 @@ SECONDS=0 rootdir=`pwd` -INSTALL_PATH=${SIMCOV_INSTALL_PATH:=$rootdir/install} +INSTALL_PATH=${SIMFORAGER_INSTALL_PATH:=$rootdir/install} echo "Installing to $INSTALL_PATH" -rm -rf $INSTALL_PATH/bin/simcov +rm -rf $INSTALL_PATH/bin/simforager if [ "$1" == "clean" ]; then rm -rf .build diff --git a/configs/figure-7/simcov.config b/configs/figure-7/simcov.config index 28d9c15..5150209 100644 --- a/configs/figure-7/simcov.config +++ b/configs/figure-7/simcov.config @@ -92,7 +92,7 @@ output = results ; Directory containing files for lung model - lung-model = /users/projects/simcov/lungmodel/lung_model_data + lung-model = /users/projects/simforager/lungmodel/lung_model_data ; Show progress progress = true diff --git a/include/options.hpp b/include/options.hpp index c9f0dc0..80b17c1 100644 --- a/include/options.hpp +++ b/include/options.hpp @@ -94,10 +94,10 @@ class Options { void setup_log_file() { if (!upcxx::rank_me()) { - string log_fname = output_dir + "/simcov.log"; - // check to see if simcov.log exists. If so, rename it + string log_fname = output_dir + "/simforager.log"; + // check to see if simforager.log exists. If so, rename it if (file_exists(log_fname)) { - string new_log_fname = output_dir + "/simcov-" + get_current_time(true) + ".log"; + string new_log_fname = output_dir + "/simforager-" + get_current_time(true) + ".log"; cerr << KLRED << "WARNING: " << KNORM << log_fname << " exists. Renaming to " << new_log_fname << endl; if (rename(log_fname.c_str(), new_log_fname.c_str()) == -1) @@ -274,7 +274,7 @@ class Options { int antibody_period = 5760; unsigned rnd_seed = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - string output_dir = "simcov-results-n" + to_string(upcxx::rank_n()) + "-N" + + string output_dir = "simforager-results-n" + to_string(upcxx::rank_n()) + "-N" + to_string(upcxx::rank_n() / upcxx::local_team().rank_n()); int sample_period = 0; int sample_resolution = 1; @@ -286,9 +286,9 @@ class Options { bool verbose = false; bool load(int argc, char **argv) { - // SIMCOV version v0.1-a0decc6-master (Release) built on 2020-04-08T22:15:40 with g++ - string full_version_str = "SimCov version " + string(SIMCOV_VERSION) + "-" + - string(SIMCOV_BRANCH) + " built on " + string(SIMCOV_BUILD_DATE); + // SIMFORAGER version v0.1-a0decc6-master (Release) built on 2020-04-08T22:15:40 with g++ + string full_version_str = "SimForager version " + string(SIMFORAGER_VERSION) + "-" + + string(SIMFORAGERR_BRANCH) + " built on " + string(SIMFORAGER_BUILD_DATE); vector infection_coords_strs; CLI::App app(full_version_str); @@ -468,13 +468,13 @@ class Options { setup_output_dir(); setup_log_file(); - init_logger(output_dir + "/simcov.log", verbose); + init_logger(output_dir + "/simforager.log", verbose); #ifdef DEBUG open_dbg("debug"); #endif - SLOG(KLBLUE, "SimCov version ", full_version_str, KNORM, "\n"); + SLOG(KLBLUE, "SimForager version ", full_version_str, KNORM, "\n"); if (upcxx::rank_me() == 0) { // print out all compiler definitions @@ -498,7 +498,7 @@ class Options { #endif if (!upcxx::rank_me()) { // write out configuration file for restarts - ofstream ofs(output_dir + "/simcov.config"); + ofstream ofs(output_dir + "/simforager.config"); ofs << app.config_to_str(true, true); } upcxx::barrier(); diff --git a/simforager_null.config b/simforager_null.config index f5519c6..0dc612f 100644 --- a/simforager_null.config +++ b/simforager_null.config @@ -1,6 +1,6 @@ ; These are version 1.0 test of 'null' forager parameters for SIMCoV ; This means a parameter-based translation of SIMCoV into SIMForager without yet -; changing any of the existing SIMCoV code base +; changing anything about the existing SIMCoV code base ; Dimensions: x y z (assuming grid spacing of 5 microns) dim = 15000 15000 1 diff --git a/slurm_hopper.sh b/slurm_hopper.sh index 688de18..05d71fa 100644 --- a/slurm_hopper.sh +++ b/slurm_hopper.sh @@ -3,7 +3,7 @@ #SBATCH --nodes=2 #SBATCH --ntasks-per-node=32 #SBATCH --time=2-00:00 -#SBATCH --job-name=simcov-tutorial +#SBATCH --job-name=simforagger-tutorial #SBATCH --mail-user= #SBATCH --mail-type=END @@ -27,4 +27,4 @@ module load cmake/3.11.4-qkyj module load openmpi/4.1.3-j6zb module load upcxx/2020.10.0-6eh2 -srun --mpi=pmi2 install/bin/simcov --config covid_default.config +srun --mpi=pmi2 install/bin/simforager --config simforager_null.config diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7aab124..4cbe837 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,9 +30,9 @@ endforeach() add_definitions(-DALL_DEFNS=\"${all_defs}\") -add_executable(simcov main.cpp tissue.cpp utils.cpp) +add_executable(simforager main.cpp tissue.cpp utils.cpp) -target_link_libraries(simcov Threads::Threads ${UPCXX_LIBRARIES} ${UPCXX_UTILS_LIBRARIES} SIMCOV_VERSION_LIB) +target_link_libraries(simforager Threads::Threads ${UPCXX_LIBRARIES} ${UPCXX_UTILS_LIBRARIES} SIMFORAGER_VERSION_LIB) #install(PROGRAMS simcov.py DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) -install(TARGETS simcov DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) +install(TARGETS simforager DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) From d299dc84c1ebb310b6ca14842f01352a227fa6d8 Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Fri, 20 Jan 2023 15:09:05 -0700 Subject: [PATCH 04/11] update version.h --- include/version.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/version.h b/include/version.h index db1ef2a..8404519 100644 --- a/include/version.h +++ b/include/version.h @@ -4,10 +4,10 @@ extern "C" { #endif -extern const char *SIMCOV_VERSION; -extern const char *SIMCOV_VERSION_DATE; -extern const char *SIMCOV_BUILD_DATE; -extern const char *SIMCOV_BRANCH; +extern const char *SIMFORAGER_VERSION; +extern const char *SIMFORAGER_VERSION_DATE; +extern const char *SIMFORAGER_BUILD_DATE; +extern const char *SIMFORAGER_BRANCH; #define MAX_BUILD_KMER_STR "Maximum kmer len=MAX_BUILD_KMER" From e0384d1372a9f017d1927dd65b63f9dc2b4557b0 Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Fri, 20 Jan 2023 15:48:31 -0700 Subject: [PATCH 05/11] update SIMCOV_VERSION --- src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index 68e708f..73b98d2 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -823,7 +823,7 @@ int main(int argc, char **argv) { memory_tracker.stop(); chrono::duration t_elapsed = NOW() - start_t; SLOG("Finished in ", setprecision(2), fixed, t_elapsed.count(), " s at ", get_current_time(), - " for SimCov version ", SIMCOV_VERSION, "\n"); + " for SimCov version ", SIMFORAGER_VERSION, "\n"); barrier(); upcxx::finalize(); return 0; From 3a8d922ad195ec7bcf584a4088f1ca02fc719f59 Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Fri, 20 Jan 2023 17:10:07 -0700 Subject: [PATCH 06/11] most complete find replace cov -> forager --- build_hopper.sh | 8 ++++---- include/options.hpp | 8 ++++---- include/version.h | 8 ++++---- slurm_hopper.sh | 2 +- src/CMakeLists.txt | 6 +++--- src/main.cpp | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) mode change 100644 => 100755 build_hopper.sh diff --git a/build_hopper.sh b/build_hopper.sh old mode 100644 new mode 100755 index 84a5dfa..2a07591 --- a/build_hopper.sh +++ b/build_hopper.sh @@ -9,8 +9,8 @@ module load cmake/3.11.4-qkyj module load openmpi/4.1.3-j6zb module load upcxx/2020.10.0-6eh2 -if [ -n "$SIMCOV_BUILD_ENV" ]; then - source $SIMCOV_BUILD_ENV +if [ -n "$SIMFORAGER_BUILD_ENV" ]; then + source $SIMFORAGER_BUILD_ENV fi upcxx_exec=`which upcxx` @@ -32,11 +32,11 @@ SECONDS=0 rootdir=`pwd` -INSTALL_PATH=${SIMCOV_INSTALL_PATH:=$rootdir/install} +INSTALL_PATH=${SIMFORAGER_INSTALL_PATH:=$rootdir/install} echo "Installing to $INSTALL_PATH" -rm -rf $INSTALL_PATH/bin/simcov +rm -rf $INSTALL_PATH/bin/simforager if [ "$1" == "clean" ]; then rm -rf .build diff --git a/include/options.hpp b/include/options.hpp index c9f0dc0..f922756 100644 --- a/include/options.hpp +++ b/include/options.hpp @@ -94,7 +94,7 @@ class Options { void setup_log_file() { if (!upcxx::rank_me()) { - string log_fname = output_dir + "/simcov.log"; + string log_fname = output_dir + "/simforager.log"; // check to see if simcov.log exists. If so, rename it if (file_exists(log_fname)) { string new_log_fname = output_dir + "/simcov-" + get_current_time(true) + ".log"; @@ -274,7 +274,7 @@ class Options { int antibody_period = 5760; unsigned rnd_seed = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - string output_dir = "simcov-results-n" + to_string(upcxx::rank_n()) + "-N" + + string output_dir = "simforager-results-n" + to_string(upcxx::rank_n()) + "-N" + to_string(upcxx::rank_n() / upcxx::local_team().rank_n()); int sample_period = 0; int sample_resolution = 1; @@ -287,8 +287,8 @@ class Options { bool load(int argc, char **argv) { // SIMCOV version v0.1-a0decc6-master (Release) built on 2020-04-08T22:15:40 with g++ - string full_version_str = "SimCov version " + string(SIMCOV_VERSION) + "-" + - string(SIMCOV_BRANCH) + " built on " + string(SIMCOV_BUILD_DATE); + string full_version_str = "SimForager version " + string(SIMFORAGER_VERSION) + "-" + + string(SIMFORAGER_BRANCH) + " built on " + string(SIMFORAGER_BUILD_DATE); vector infection_coords_strs; CLI::App app(full_version_str); diff --git a/include/version.h b/include/version.h index db1ef2a..8404519 100644 --- a/include/version.h +++ b/include/version.h @@ -4,10 +4,10 @@ extern "C" { #endif -extern const char *SIMCOV_VERSION; -extern const char *SIMCOV_VERSION_DATE; -extern const char *SIMCOV_BUILD_DATE; -extern const char *SIMCOV_BRANCH; +extern const char *SIMFORAGER_VERSION; +extern const char *SIMFORAGER_VERSION_DATE; +extern const char *SIMFORAGER_BUILD_DATE; +extern const char *SIMFORAGER_BRANCH; #define MAX_BUILD_KMER_STR "Maximum kmer len=MAX_BUILD_KMER" diff --git a/slurm_hopper.sh b/slurm_hopper.sh index 688de18..9857004 100644 --- a/slurm_hopper.sh +++ b/slurm_hopper.sh @@ -27,4 +27,4 @@ module load cmake/3.11.4-qkyj module load openmpi/4.1.3-j6zb module load upcxx/2020.10.0-6eh2 -srun --mpi=pmi2 install/bin/simcov --config covid_default.config +srun --mpi=pmi2 install/bin/simforager --config simforager_null.config diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7aab124..4cbe837 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,9 +30,9 @@ endforeach() add_definitions(-DALL_DEFNS=\"${all_defs}\") -add_executable(simcov main.cpp tissue.cpp utils.cpp) +add_executable(simforager main.cpp tissue.cpp utils.cpp) -target_link_libraries(simcov Threads::Threads ${UPCXX_LIBRARIES} ${UPCXX_UTILS_LIBRARIES} SIMCOV_VERSION_LIB) +target_link_libraries(simforager Threads::Threads ${UPCXX_LIBRARIES} ${UPCXX_UTILS_LIBRARIES} SIMFORAGER_VERSION_LIB) #install(PROGRAMS simcov.py DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) -install(TARGETS simcov DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) +install(TARGETS simforager DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) diff --git a/src/main.cpp b/src/main.cpp index 68e708f..2a43e8f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -823,7 +823,7 @@ int main(int argc, char **argv) { memory_tracker.stop(); chrono::duration t_elapsed = NOW() - start_t; SLOG("Finished in ", setprecision(2), fixed, t_elapsed.count(), " s at ", get_current_time(), - " for SimCov version ", SIMCOV_VERSION, "\n"); + " for SimForager version ", SIMFORAGER_VERSION, "\n"); barrier(); upcxx::finalize(); return 0; From ef022c61df263101805a45227700b98d5c7cde8b Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Thu, 23 Feb 2023 14:35:09 -0700 Subject: [PATCH 07/11] min t-fish extravisate params --- simforager_null.config | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/simforager_null.config b/simforager_null.config index 0dc612f..faec21b 100644 --- a/simforager_null.config +++ b/simforager_null.config @@ -3,7 +3,7 @@ ; changing anything about the existing SIMCoV code base ; Dimensions: x y z (assuming grid spacing of 5 microns) -dim = 15000 15000 1 +dim = 1000 1000 1 ; >> resulting in 225000000 total environment (epithelial) cells ; Whole lung dimensions: x y z (assuming grid spacing of 5 microns) @@ -35,7 +35,8 @@ apoptosis-period = 1000000 expressing-period = 1000000 ; Factor multiplied by number of virions to determine probability of infection -infectivity = 0.0 +infectivity = 1.0 +; SIMCoV paper default ; Multiplier reducing infectivity where inflammatory signal is present ; (the multiplier reducing infectivity should be between 0.9 and 0.7 based on values @@ -56,16 +57,16 @@ virion-clearance = 0.0 ; >> effectively algae death rate; not considered here ; Fraction of virions that diffuse into all neighbors each time step -virion-diffusion = 0.001 +virion-diffusion = 0.0 ; Amount of chemokine produced by expressing cells each time step -chemokine-production = 0.0 +chemokine-production = 1.0 ; Amount by which chemokine concentration drops each time step chemokine-decay = 0.0 ; Fraction of chemokine concentration that diffuses into all neighbors each time step -chemokine-diffusion = 0.0 +chemokine-diffusion = 1.0 ; Minimum chemokine concentration that triggers a T cell min-chemokine = 0.0 @@ -80,20 +81,20 @@ antibody-period = 1000000 ; Number of tcells generated at each timestep for the whole lung ; This is scaled up 5000x from the mouse model (20 per minute) -tcell-generation-rate = 0 +tcell-generation-rate = 100000 ; >> for a stable population of fish (T-cells) throughout sim ; Number of time steps before T cells start to be produced tcell-initial-delay = 0 ; Average number of time steps to death for a T cell in the vasculature -tcell-vascular-period = 1 +tcell-vascular-period = 3 ; Average number of time steps to death after a T cell extravasates tcell-tissue-period = 1000000 ; Number of time steps a T cell is bound to an epithelial cell when inducing apoptosis -tcell-binding-period = 0.5 +tcell-binding-period = 2 ; Max probability of a T cell binding to an infected cell in one time step max-binding-prob = 1 @@ -114,4 +115,4 @@ sample-resolution = 1 max-block-dim = 10 ; Output directory (automatically generated) -; output = results +; output = results From 8ea1269ee0ab7c85a50bed591dba711fcbc2a100 Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Tue, 21 Mar 2023 13:22:59 -0600 Subject: [PATCH 08/11] latest config file --- simforager_null.config | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/simforager_null.config b/simforager_null.config index faec21b..f198f29 100644 --- a/simforager_null.config +++ b/simforager_null.config @@ -7,7 +7,7 @@ dim = 1000 1000 1 ; >> resulting in 225000000 total environment (epithelial) cells ; Whole lung dimensions: x y z (assuming grid spacing of 5 microns) -whole-lung-dim = 48000 40000 20000 +whole-lung-dim = 1000 1000 1 ; Number of timesteps (assuming 1 min per timestep) ; Set to arbitrary small number to start @@ -22,14 +22,14 @@ infection-coords = uniform:22500 ; Number of virions at initial infection locations ; Chosen to guarantee that at least one cell is infected at the start - see infectivity -initial-infection = 1000 -; >> combined with number of initial infection sites, this should yield a 1:100 ratio of grid cells to algae cells +initial-infection = 1 +; >> initially 1000 combined with number of initial infection sites, this should yield a 1:100 ratio of grid cells to algae cells ; Average number of time steps to expressing virions after cell is infected incubation-period = 0 ; Average number of time steps to death after apoptosis is induced -apoptosis-period = 1000000 +apoptosis-period = 3 ; Average number of time steps to death after a cell starts expresssing expressing-period = 1000000 @@ -60,7 +60,7 @@ virion-clearance = 0.0 virion-diffusion = 0.0 ; Amount of chemokine produced by expressing cells each time step -chemokine-production = 1.0 +chemokine-production = 0.0 ; Amount by which chemokine concentration drops each time step chemokine-decay = 0.0 @@ -81,7 +81,7 @@ antibody-period = 1000000 ; Number of tcells generated at each timestep for the whole lung ; This is scaled up 5000x from the mouse model (20 per minute) -tcell-generation-rate = 100000 +tcell-generation-rate = 100 ; >> for a stable population of fish (T-cells) throughout sim ; Number of time steps before T cells start to be produced @@ -115,4 +115,4 @@ sample-resolution = 1 max-block-dim = 10 ; Output directory (automatically generated) -; output = results +; output = results From af596f09f30eb936fa31763c478bed8d93986914 Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Thu, 27 Jul 2023 14:46:43 -0600 Subject: [PATCH 09/11] null config file for stable tfish population --- simforager_null.config | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/simforager_null.config b/simforager_null.config index f198f29..f613b55 100644 --- a/simforager_null.config +++ b/simforager_null.config @@ -3,15 +3,16 @@ ; changing anything about the existing SIMCoV code base ; Dimensions: x y z (assuming grid spacing of 5 microns) -dim = 1000 1000 1 +dim = 500 500 1 ; >> resulting in 225000000 total environment (epithelial) cells ; Whole lung dimensions: x y z (assuming grid spacing of 5 microns) -whole-lung-dim = 1000 1000 1 +whole-lung-dim = 500 500 1 ; Number of timesteps (assuming 1 min per timestep) ; Set to arbitrary small number to start -timesteps = 1000 +timesteps = 14400 +; >> ten days ; Location of initial infections (omit for no infections); can be: ; list of space-separated coords x,y,z,timestep @@ -35,7 +36,7 @@ apoptosis-period = 3 expressing-period = 1000000 ; Factor multiplied by number of virions to determine probability of infection -infectivity = 1.0 +infectivity = 0.5 ; SIMCoV paper default ; Multiplier reducing infectivity where inflammatory signal is present @@ -88,7 +89,7 @@ tcell-generation-rate = 100 tcell-initial-delay = 0 ; Average number of time steps to death for a T cell in the vasculature -tcell-vascular-period = 3 +tcell-vascular-period = 2 ; Average number of time steps to death after a T cell extravasates tcell-tissue-period = 1000000 @@ -106,7 +107,7 @@ tcells-follow-gradient = false seed = 29 ; Number of timesteps between samples (set to 0 to disable sampling) -sample-period = 0 +sample-period = 5 ; Resolution for sampling sample-resolution = 1 From ba41b14ff3d20728c2e4f1cf22164753d60afc8e Mon Sep 17 00:00:00 2001 From: Julie Hayes Date: Mon, 9 Oct 2023 15:57:05 -0600 Subject: [PATCH 10/11] simforager null config + '0.1' changes to main.cpp --- src/main.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index 2a43e8f..da1c69b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -188,7 +188,9 @@ void update_circulating_tcells(int time_step, Tissue &tissue, double extravasate tissue.change_num_circulating_tcells(-num_dying); _sim_stats.tcells_vasculature -= num_dying; num_circulating = tissue.get_num_circulating_tcells(); - double portion_xtravasing = extravasate_fraction * num_circulating; + double portion_xtravasing = (time_step <= 15) ? extravasate_fraction * num_circulating : 0; + //while (time_step < 20) double portion_xtravasing = ? extravasate_fraction * num_circulating : 0; + //double portion_xtravasing = ? extravasate_fraction * num_circulating; int num_xtravasing = floor(portion_xtravasing); if (_rnd_gen->trial_success(portion_xtravasing - num_xtravasing)) num_xtravasing++; for (int i = 0; i < num_xtravasing; i++) { @@ -226,6 +228,8 @@ void update_tissue_tcell(int time_step, Tissue &tissue, GridPoint *grid_point, v if (tcell->binding_period != -1) { DBG(time_step, " tcell ", tcell->id, " is bound at ", grid_point->coords.str(), "\n"); // this tcell is bound + //grid_point->nb_virions -= 100; + grid_point->virions -= 42; //arbitrary large bite size tcell->binding_period--; // done with binding when set to -1 if (tcell->binding_period == 0) tcell->binding_period = -1; From 1a6e385de99efc3383c3ceb7556e5513945381aa Mon Sep 17 00:00:00 2001 From: Ariana Villegas Date: Thu, 9 Nov 2023 08:33:36 -0700 Subject: [PATCH 11/11] fix: update upcxx calls --- include/options.hpp | 19 +++++----- src/main.cpp | 84 ++++++++++++++++++++++----------------------- src/tissue.cpp | 32 ++++++++--------- src/utils.cpp | 2 +- 4 files changed, 69 insertions(+), 68 deletions(-) diff --git a/include/options.hpp b/include/options.hpp index ac9def2..0c9261e 100644 --- a/include/options.hpp +++ b/include/options.hpp @@ -109,7 +109,7 @@ class Options { void set_random_infections(int num) { for (int i = 0; i < num; i++) { - if (i % rank_n() != rank_me()) continue; + if (i % upcxx::rank_n() != upcxx::rank_me()) continue; infection_coords.push_back( {_rnd_gen->get(dimensions[0] * 0.1, dimensions[0] * 0.9), _rnd_gen->get(dimensions[1] * 0.1, dimensions[1] * 0.9), @@ -123,7 +123,7 @@ class Options { vector> infections = get_uniform_infections(num, dimensions[0], dimensions[1], dimensions[2]); for (int i = 0; i < infections.size(); i++) { - if (i % rank_n() != rank_me()) continue; + if (i % upcxx::rank_n() != upcxx::rank_me()) continue; infection_coords.push_back({infections[i][0], infections[i][1], infections[i][2], 0}); } } @@ -197,7 +197,7 @@ class Options { } } for (int i = 0; i < coords_strs.size(); i++) { - if (i % rank_n() != rank_me()) continue; + if (i % upcxx::rank_n() != upcxx::rank_me()) continue; auto coords_and_time = splitter(",", coords_strs[i]); if (coords_and_time.size() == 4) { try { @@ -255,7 +255,8 @@ class Options { int tcell_initial_delay = 10080; int tcell_vascular_period = 5760; int tcell_tissue_period = 1440; - int tcell_binding_period = 10; + // int tcell_binding_period = 10; + double tcell_binding_period = 10; double max_binding_prob = 1.0; double infectivity = 0.02; @@ -419,7 +420,7 @@ class Options { upcxx::barrier(); - _rnd_gen = make_shared(rnd_seed + rank_me()); + _rnd_gen = make_shared(rnd_seed + upcxx::rank_me()); if (!lung_model_dir.empty()) { auto model_dims = get_model_dims(lung_model_dir + "/alveolus.dat"); @@ -427,7 +428,7 @@ class Options { for (int i = 0; i < 3; i++) { if (model_dims[i] != dimensions[i]) { dimensions = model_dims; - if (!rank_me()) + if (!upcxx::rank_me()) cerr << KLRED << "WARNING: " << KNORM << "Setting dimensions to model data: " << dimensions[0] << ", " << dimensions[1] << ", " << dimensions[2] << endl; @@ -437,7 +438,7 @@ class Options { } if (virion_clearance_rate * antibody_factor > 1.0) { - if (!rank_me()) + if (!upcxx::rank_me()) cerr << "Invalid parameter settings: virion-clearance * antibody_factor > 1.\n" << "Reduce either or both of those settings\n"; return false; @@ -453,7 +454,7 @@ class Options { if (dimensions[0] % sample_resolution || dimensions[1] % sample_resolution || (dimensions[2] > 1 && dimensions[2] % sample_resolution)) { - if (!rank_me()) + if (!upcxx::rank_me()) cerr << "Error: sample period " << sample_resolution << " must be a factor of all the dimensions\n"; return false; @@ -461,7 +462,7 @@ class Options { for (int i = 0; i < 3; i++) { if (dimensions[i] > whole_lung_dims[i]) { - if (!rank_me()) cerr << "Dimensions must be <= whole lung dimensions\n"; + if (!upcxx::rank_me()) cerr << "Dimensions must be <= whole lung dimensions\n"; return false; } } diff --git a/src/main.cpp b/src/main.cpp index da1c69b..22173a0 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -43,7 +43,7 @@ class SimStats { float virions = 0; void init() { - if (!rank_me()) { + if (!upcxx::rank_me()) { log_file.open(_options->output_dir + "/simcov.stats"); log_file << "# time" << header(0) << endl; } @@ -65,16 +65,16 @@ class SimStats { string to_str(int width) { vector totals; - totals.push_back(reduce_one(incubating, op_fast_add, 0).wait()); - totals.push_back(reduce_one(expressing, op_fast_add, 0).wait()); - totals.push_back(reduce_one(apoptotic, op_fast_add, 0).wait()); - totals.push_back(reduce_one(dead, op_fast_add, 0).wait()); - totals.push_back(reduce_one(tcells_vasculature, op_fast_add, 0).wait()); - totals.push_back(reduce_one(tcells_tissue, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(incubating, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(expressing, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(apoptotic, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(dead, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(tcells_vasculature, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(tcells_tissue, op_fast_add, 0).wait()); vector totals_d; - totals_d.push_back(reduce_one(chemokines, op_fast_add, 0).wait() / get_num_grid_points()); - totals_d.push_back(reduce_one(virions, op_fast_add, 0).wait()); // / get_num_grid_points()); - auto all_chem_pts = reduce_one(num_chemo_pts, op_fast_add, 0).wait(); + totals_d.push_back(upcxx::reduce_one(chemokines, op_fast_add, 0).wait() / get_num_grid_points()); + totals_d.push_back(upcxx::reduce_one(virions, op_fast_add, 0).wait()); // / get_num_grid_points()); + auto all_chem_pts = upcxx::reduce_one(num_chemo_pts, op_fast_add, 0).wait(); totals_d.push_back(all_chem_pts + totals[0] + totals[1] + totals[2] + totals[3]); auto perc_infected = 100.0 * (float)(totals[0] + totals[1] + totals[2] + totals[3]) / get_num_grid_points(); @@ -104,7 +104,7 @@ class SimStats { void log(int time_step) { string s = to_str(0); - if (!rank_me()) log_file << time_step << s << endl; + if (!upcxx::rank_me()) log_file << time_step << s << endl; } }; @@ -151,21 +151,21 @@ void seed_infection(Tissue &tissue, int time_step) { _options->infection_coords.erase(it--); } } - barrier(); + upcxx::barrier(); tissue.add_new_actives(add_new_actives_timer); - barrier(); + upcxx::barrier(); } void generate_tcells(Tissue &tissue, int time_step) { generate_tcell_timer.start(); - int local_num = _options->tcell_generation_rate / rank_n(); - int rem = _options->tcell_generation_rate - local_num * rank_n(); - if (rank_me() < rem) local_num++; + int local_num = _options->tcell_generation_rate / upcxx::rank_n(); + int rem = _options->tcell_generation_rate - local_num * upcxx::rank_n(); + if (upcxx::rank_me() < rem) local_num++; if (time_step == 1) WARN("rem ", rem, " local num ", local_num, "\n"); tissue.change_num_circulating_tcells(local_num); #ifdef DEBUG - auto all_num = reduce_one(local_num, op_fast_add, 0).wait(); - if (!rank_me() && all_num != _options->tcell_generation_rate) + auto all_num = upcxx::reduce_one(local_num, op_fast_add, 0).wait(); + if (!upcxx::rank_me() && all_num != _options->tcell_generation_rate) DIE("num tcells generated ", all_num, " != generation rate ", _options->tcell_generation_rate); #endif generate_tcell_timer.stop(); @@ -194,7 +194,7 @@ void update_circulating_tcells(int time_step, Tissue &tissue, double extravasate int num_xtravasing = floor(portion_xtravasing); if (_rnd_gen->trial_success(portion_xtravasing - num_xtravasing)) num_xtravasing++; for (int i = 0; i < num_xtravasing; i++) { - progress(); + upcxx::progress(); GridCoords coords(_rnd_gen); if (tissue.try_add_new_tissue_tcell(coords.to_1d())) { _sim_stats.tcells_tissue++; @@ -490,7 +490,7 @@ void sample(int time_step, vector &samples, int64_t start_id, ViewOb header_oss << " unsigned_char\n" << "LOOKUP_TABLE default\n"; auto header_str = header_oss.str(); - if (!rank_me()) { + if (!upcxx::rank_me()) { tot_sz += header_str.size(); // rank 0 creates the file and truncates it to the correct length auto fileno = open(fname.c_str(), O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); @@ -563,9 +563,9 @@ int64_t get_samples(Tissue &tissue, vector &samples) { int64_t num_points = get_num_grid_points() / (_options->sample_resolution * _options->sample_resolution); if (_grid_size->z > 1) num_points /= _options->sample_resolution; - int64_t num_points_per_rank = ceil((double)num_points / rank_n()); - int64_t start_id = rank_me() * num_points_per_rank; - int64_t end_id = min((rank_me() + 1) * num_points_per_rank, num_points); + int64_t num_points_per_rank = ceil((double)num_points / upcxx::rank_n()); + int64_t start_id = upcxx::rank_me() * num_points_per_rank; + int64_t end_id = min((upcxx::rank_me() + 1) * num_points_per_rank, num_points); samples.clear(); if (end_id > start_id) { samples.reserve(end_id - start_id); @@ -582,7 +582,7 @@ int64_t get_samples(Tissue &tissue, vector &samples) { break; } if (i >= start_id) { - progress(); + upcxx::progress(); #ifdef AVERAGE_SUBSAMPLE float virions = 0; float chemokine = 0; @@ -650,8 +650,8 @@ int64_t get_samples(Tissue &tissue, vector &samples) { if (done) break; } } - barrier(); - auto samples_written = reduce_one(samples.size(), op_fast_add, 0).wait(); + upcxx::barrier(); + auto samples_written = upcxx::reduce_one(samples.size(), op_fast_add, 0).wait(); if (num_points != samples_written) SWARN("Number of point ", num_points, " != ", samples_written, " samples written"); SLOG_VERBOSE("Number of samples written ", samples_written, "\n"); @@ -684,7 +684,7 @@ void run_sim(Tissue &tissue) { for (int time_step = 0; time_step < _options->num_timesteps; time_step++) { DBG("Time step ", time_step, "\n"); seed_infection(tissue, time_step); - barrier(); + upcxx::barrier(); if (time_step == _options->antibody_period) _options->virion_clearance_rate *= _options->antibody_factor; chemokines_to_update.clear(); @@ -692,7 +692,7 @@ void run_sim(Tissue &tissue) { chemokines_cache.clear(); if (time_step > _options->tcell_initial_delay) { generate_tcells(tissue, time_step); - barrier(); + upcxx::barrier(); } compute_updates_timer.start(); update_circulating_tcells(time_step, tissue, extravasate_fraction); @@ -728,31 +728,31 @@ void run_sim(Tissue &tissue) { update_virions(grid_point, *nbs, virions_to_update); if (grid_point->is_active()) tissue.set_active(grid_point); } - barrier(); + upcxx::barrier(); compute_updates_timer.stop(); tissue.accumulate_chemokines(chemokines_to_update, accumulate_concentrations_timer); tissue.accumulate_virions(virions_to_update, accumulate_concentrations_timer); - barrier(); + upcxx::barrier(); if (time_step % five_perc == 0 || time_step == _options->num_timesteps - 1) { - auto num_actives = reduce_one(tissue.get_num_actives(), op_fast_add, 0).wait(); + auto num_actives = upcxx::reduce_one(tissue.get_num_actives(), op_fast_add, 0).wait(); auto perc_actives = 100.0 * num_actives / get_num_grid_points(); - auto max_actives = reduce_one(tissue.get_num_actives(), op_fast_max, 0).wait(); - auto load_balance = max_actives ? (double)num_actives / rank_n() / max_actives : 1; + auto max_actives = upcxx::reduce_one(tissue.get_num_actives(), op_fast_max, 0).wait(); + auto load_balance = max_actives ? (double)num_actives / upcxx::rank_n() / max_actives : 1; chrono::duration t_elapsed = NOW() - curr_t; curr_t = NOW(); SLOG("[", get_current_time(), " ", setprecision(2), fixed, setw(7), right, t_elapsed.count(), "s]: ", setw(8), left, time_step, _sim_stats.to_str(STATS_COL_WIDTH), setprecision(3), fixed, "< ", perc_actives, " ", load_balance, " >\n"); } - barrier(); + upcxx::barrier(); tissue.add_new_actives(add_new_actives_timer); - barrier(); + upcxx::barrier(); _sim_stats.virions = 0; _sim_stats.chemokines = 0; _sim_stats.num_chemo_pts = 0; set_active_grid_points(tissue); - barrier(); + upcxx::barrier(); if (_options->sample_period > 0 && (time_step % _options->sample_period == 0 || time_step == _options->num_timesteps - 1)) { @@ -768,13 +768,13 @@ void run_sim(Tissue &tissue) { log_timer.start(); _sim_stats.log(time_step); - barrier(); + upcxx::barrier(); log_timer.stop(); #ifdef DEBUG DBG("check actives ", time_step, "\n"); tissue.check_actives(time_step); - barrier(); + upcxx::barrier(); #endif } @@ -803,11 +803,11 @@ int main(int argc, char **argv) { _options = make_shared(); if (!_options->load(argc, argv)) return 0; ProgressBar::SHOW_PROGRESS = _options->show_progress; - if (pin_thread(getpid(), local_team().rank_me()) == -1) - WARN("Could not pin process ", getpid(), " to core ", rank_me()); + if (pin_thread(getpid(), upcxx::local_team().rank_me()) == -1) + WARN("Could not pin process ", getpid(), " to core ", upcxx::rank_me()); else SLOG_VERBOSE("Pinned processes, with process 0 (pid ", getpid(), ") pinned to core ", - local_team().rank_me(), "\n"); + upcxx::local_team().rank_me(), "\n"); #ifdef BLOCK_PARTITION SLOG_VERBOSE("Using block partitioning\n"); #else @@ -828,7 +828,7 @@ int main(int argc, char **argv) { chrono::duration t_elapsed = NOW() - start_t; SLOG("Finished in ", setprecision(2), fixed, t_elapsed.count(), " s at ", get_current_time(), " for SimForager version ", SIMFORAGER_VERSION, "\n"); - barrier(); + upcxx::barrier(); upcxx::finalize(); return 0; } diff --git a/src/tissue.cpp b/src/tissue.cpp index 06400de..4560c18 100644 --- a/src/tissue.cpp +++ b/src/tissue.cpp @@ -170,8 +170,8 @@ static int get_cube_block_dim(int64_t num_grid_points) { size_t cube = (size_t)pow((double)d, 3.0); size_t num_cubes = num_grid_points / cube; DBG("cube size ", cube, " num cubes ", num_cubes, "\n"); - if (num_cubes < rank_n() * MIN_BLOCKS_PER_PROC) { - DBG("not enough cubes ", num_cubes, " < ", rank_n() * MIN_BLOCKS_PER_PROC, "\n"); + if (num_cubes < upcxx::rank_n() * MIN_BLOCKS_PER_PROC) { + DBG("not enough cubes ", num_cubes, " < ", upcxx::rank_n() * MIN_BLOCKS_PER_PROC, "\n"); break; } // there is a remainder - this is not a perfect division @@ -180,7 +180,7 @@ static int get_cube_block_dim(int64_t num_grid_points) { continue; } // skip sizes that distribute the blocks in columns - if (d > 1 && (_grid_size->x % (d * rank_n()) == 0 || _grid_size->y % (d * rank_n()) == 0)) { + if (d > 1 && (_grid_size->x % (d * upcxx::rank_n()) == 0 || _grid_size->y % (d * upcxx::rank_n()) == 0)) { DBG("dim ", d, " gives perfect division of all blocks into x axis - skip\n"); continue; } @@ -201,8 +201,8 @@ static int get_square_block_dim(int64_t num_grid_points) { size_t square = (size_t)pow((double)d, 2.0); size_t num_squares = num_grid_points / square; DBG("square size ", square, " num squares ", num_squares, "\n"); - if (num_squares < rank_n() * MIN_BLOCKS_PER_PROC) { - DBG("not enough squares ", num_squares, " < ", rank_n() * MIN_BLOCKS_PER_PROC, "\n"); + if (num_squares < upcxx::rank_n() * MIN_BLOCKS_PER_PROC) { + DBG("not enough squares ", num_squares, " < ", upcxx::rank_n() * MIN_BLOCKS_PER_PROC, "\n"); break; } // there is a remainder - this is not a perfect division @@ -211,7 +211,7 @@ static int get_square_block_dim(int64_t num_grid_points) { continue; } // skip sizes that distribute the blocks in columns - if (d > 1 && _grid_size->x % (d * rank_n()) == 0) { + if (d > 1 && _grid_size->x % (d * upcxx::rank_n()) == 0) { DBG("dim ", d, " gives perfect division of all blocks into x axis - skip\n"); continue; } @@ -259,7 +259,7 @@ Tissue::Tissue() int64_t num_blocks = num_grid_points / _grid_blocks.block_size; - int64_t blocks_per_rank = ceil((double)num_blocks / rank_n()); + int64_t blocks_per_rank = ceil((double)num_blocks / upcxx::rank_n()); bool threeD = _grid_size->z > 1; SLOG("Dividing ", num_grid_points, " grid points into ", num_blocks, @@ -288,7 +288,7 @@ Tissue::Tissue() // FIXME: these blocks need to be stride distributed to better load balance grid_points->reserve(blocks_per_rank * _grid_blocks.block_size); for (int64_t i = 0; i < blocks_per_rank; i++) { - int64_t start_id = (i * rank_n() + rank_me()) * _grid_blocks.block_size; + int64_t start_id = (i * upcxx::rank_n() + upcxx::rank_me()) * _grid_blocks.block_size; if (start_id >= num_grid_points) break; for (auto id = start_id; id < start_id + _grid_blocks.block_size; id++) { assert(id < num_grid_points); @@ -322,7 +322,7 @@ Tissue::Tissue() #endif } } - barrier(); + upcxx::barrier(); } int Tissue::load_data_file(const string &fname, int num_grid_points, EpiCellType epicell_type) { @@ -353,11 +353,11 @@ int Tissue::load_data_file(const string &fname, int num_grid_points, EpiCellType intrank_t Tissue::get_rank_for_grid_point(int64_t grid_i) { int64_t block_i = grid_i / _grid_blocks.block_size; - return block_i % rank_n(); + return block_i % upcxx::rank_n(); } GridPoint *Tissue::get_local_grid_point(grid_points_t &grid_points, int64_t grid_i) { - int64_t block_i = grid_i / _grid_blocks.block_size / rank_n(); + int64_t block_i = grid_i / _grid_blocks.block_size / upcxx::rank_n(); int64_t i = grid_i % _grid_blocks.block_size + block_i * _grid_blocks.block_size; assert(i < grid_points->size()); GridPoint *grid_point = &(*grid_points)[i]; @@ -441,13 +441,13 @@ void Tissue::accumulate_chemokines(HASH_TABLE &chemokines_to_upd // accumulate updates for each target rank HASH_TABLE>> target_rank_updates; for (auto &[coords_1d, chemokines] : chemokines_to_update) { - progress(); + upcxx::progress(); target_rank_updates[get_rank_for_grid_point(coords_1d)].push_back({coords_1d, chemokines}); } future<> fut_chain = make_future<>(); // dispatch all updates to each target rank in turn for (auto &[target_rank, update_vector] : target_rank_updates) { - progress(); + upcxx::progress(); auto fut = rpc( target_rank, [](grid_points_t &grid_points, new_active_grid_points_t &new_active_grid_points, @@ -473,13 +473,13 @@ void Tissue::accumulate_virions(HASH_TABLE &virions_to_update, // accumulate updates for each target rank HASH_TABLE>> target_rank_updates; for (auto &[coords_1d, virions] : virions_to_update) { - progress(); + upcxx::progress(); target_rank_updates[get_rank_for_grid_point(coords_1d)].push_back({coords_1d, virions}); } future<> fut_chain = make_future<>(); // dispatch all updates to each target rank in turn for (auto &[target_rank, update_vector] : target_rank_updates) { - progress(); + upcxx::progress(); auto fut = rpc( target_rank, [](grid_points_t &grid_points, new_active_grid_points_t &new_active_grid_points, @@ -525,7 +525,7 @@ bool Tissue::try_add_new_tissue_tcell(int64_t grid_i) { if (grid_point->tcell) return false; if (grid_point->chemokine < _options->min_chemokine) return false; new_active_grid_points->insert({grid_point, true}); - string tcell_id = to_string(rank_me()) + "-" + to_string(*tcells_generated); + string tcell_id = to_string(upcxx::rank_me()) + "-" + to_string(*tcells_generated); (*tcells_generated)++; grid_point->tcell = new TCell(tcell_id); grid_point->tcell->moved = true; diff --git a/src/utils.cpp b/src/utils.cpp index 997fcec..26bec92 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -42,7 +42,7 @@ void dump_single_file(const string &fname, const string &out_str) { upcxx::barrier(); ad.destroy(); // wait until rank 0 has finished setting up the file - if (rank_me()) fileno = open(fname.c_str(), O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (upcxx::rank_me()) fileno = open(fname.c_str(), O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fileno == -1) WARN("Error trying to open file ", fname, ": ", strerror(errno), "\n"); auto bytes_written = pwrite(fileno, out_str.c_str(), sz, my_fpos); close(fileno);