diff --git a/CMakeLists.txt b/CMakeLists.txt index fb2c3b4..8f82135 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,16 +1,16 @@ # The name of our project is "MHMxx". CMakeLists files in this project can -# refer to the root source directory of the project as ${SIMCOV_SOURCE_DIR} and -# to the root binary directory of the project as ${SIMCOV_BINARY_DIR} +# refer to the root source directory of the project as ${SIMFORAGER_SOURCE_DIR} and +# to the root binary directory of the project as ${SIMFORAGER_BINARY_DIR} cmake_minimum_required(VERSION 3.10 FATAL_ERROR) -project(SimCov) +project(SimForager) #----------------------------------------------------------------------------------------# # project specification #----------------------------------------------------------------------------------------# -file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/VERSION SIMCOV_BASE_VERSION) +file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/VERSION SIMFORAGER_BASE_VERSION) # use PROJECT_SOURCE_DIR, not CMAKE_SOURCE_DIR -set(SIMCOV_SOURCE_DIR ${PROJECT_SOURCE_DIR}) -set(SIMCOV_BINARY_DIR ${PROJECT_BINARY_DIR}) +set(SIMFORAGER_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(SIMFORAGER_BINARY_DIR ${PROJECT_BINARY_DIR}) # include modules set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake/Modules) @@ -55,7 +55,7 @@ CHECK_SUBMODULES(upcxx-utils) # set version include_directories(include) -GET_GIT_VERSION(SIMCOV) +GET_GIT_VERSION(SIMFORAGER) # add upcxx-utils if (DEFINED ENV{UPCXX_UTILS_DIR} AND EXISTS $ENV{UPCXX_UTILS_DIR}) diff --git a/README.md b/README.md index 5e2f00c..38cc6d1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +simforager is a forked adaptation of: + simcov Copyright (c) 2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy), Arizona State @@ -14,9 +16,9 @@ its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software to reproduce, distribute copies to the public, prepare derivative works, and perform publicly and display publicly, and to permit others to do so. -# SimCov # +# SimForager # -This is a model for simulating the response of the immune system to infections in the lungs. +This is a model for simulating large amounts of [currently homogeneous] foragers. ## Installing and building @@ -24,7 +26,7 @@ It requires [UPC++](https://bitbucket.org/berkeleylab/upcxx/wiki/Home), C++ and This repo contains a submodule, so to install, it's best to run -`git clone --recurse-submodules git@github.com:AdaptiveComputationLab/simcov.git` +`git clone --recurse-submodules git@github.com:cswritlarge/simforager.git` to fully initialize the submodules. @@ -38,22 +40,21 @@ or The executable will be installed in -`/install/bin` +`/install/bin` ## Running To run, execute -`upcxx-run -n -N -- simcov` +`upcxx-run -n -N -- simforager` To see the parameters available, run with `-h`. -It will create an output directory, which will contain a detailed log file (`simcov.log`). It will also create a file containing -all the configuration parameters (`simcov.config`). +It will create an output directory, which will contain a detailed log file (`simforager.log`). It will also create a file containing all the configuration parameters (`simforager.config`). A run can also be executed with a config file as -`upcxx-run -n -N -- simcov --config ` +`upcxx-run -n -N -- simforager --config ` The config file consists of a list of all the command line options as key-value pairs, with semi-colons denoting comments. For example: diff --git a/build_hopper.sh b/build_hopper.sh old mode 100644 new mode 100755 index 84a5dfa..2a07591 --- a/build_hopper.sh +++ b/build_hopper.sh @@ -9,8 +9,8 @@ module load cmake/3.11.4-qkyj module load openmpi/4.1.3-j6zb module load upcxx/2020.10.0-6eh2 -if [ -n "$SIMCOV_BUILD_ENV" ]; then - source $SIMCOV_BUILD_ENV +if [ -n "$SIMFORAGER_BUILD_ENV" ]; then + source $SIMFORAGER_BUILD_ENV fi upcxx_exec=`which upcxx` @@ -32,11 +32,11 @@ SECONDS=0 rootdir=`pwd` -INSTALL_PATH=${SIMCOV_INSTALL_PATH:=$rootdir/install} +INSTALL_PATH=${SIMFORAGER_INSTALL_PATH:=$rootdir/install} echo "Installing to $INSTALL_PATH" -rm -rf $INSTALL_PATH/bin/simcov +rm -rf $INSTALL_PATH/bin/simforager if [ "$1" == "clean" ]; then rm -rf .build diff --git a/configs/figure-7/simcov.config b/configs/figure-7/simcov.config index 28d9c15..5150209 100644 --- a/configs/figure-7/simcov.config +++ b/configs/figure-7/simcov.config @@ -92,7 +92,7 @@ output = results ; Directory containing files for lung model - lung-model = /users/projects/simcov/lungmodel/lung_model_data + lung-model = /users/projects/simforager/lungmodel/lung_model_data ; Show progress progress = true diff --git a/include/options.hpp b/include/options.hpp index c9f0dc0..0c9261e 100644 --- a/include/options.hpp +++ b/include/options.hpp @@ -94,10 +94,10 @@ class Options { void setup_log_file() { if (!upcxx::rank_me()) { - string log_fname = output_dir + "/simcov.log"; + string log_fname = output_dir + "/simforager.log"; // check to see if simcov.log exists. If so, rename it if (file_exists(log_fname)) { - string new_log_fname = output_dir + "/simcov-" + get_current_time(true) + ".log"; + string new_log_fname = output_dir + "/simforager-" + get_current_time(true) + ".log"; cerr << KLRED << "WARNING: " << KNORM << log_fname << " exists. Renaming to " << new_log_fname << endl; if (rename(log_fname.c_str(), new_log_fname.c_str()) == -1) @@ -109,7 +109,7 @@ class Options { void set_random_infections(int num) { for (int i = 0; i < num; i++) { - if (i % rank_n() != rank_me()) continue; + if (i % upcxx::rank_n() != upcxx::rank_me()) continue; infection_coords.push_back( {_rnd_gen->get(dimensions[0] * 0.1, dimensions[0] * 0.9), _rnd_gen->get(dimensions[1] * 0.1, dimensions[1] * 0.9), @@ -123,7 +123,7 @@ class Options { vector> infections = get_uniform_infections(num, dimensions[0], dimensions[1], dimensions[2]); for (int i = 0; i < infections.size(); i++) { - if (i % rank_n() != rank_me()) continue; + if (i % upcxx::rank_n() != upcxx::rank_me()) continue; infection_coords.push_back({infections[i][0], infections[i][1], infections[i][2], 0}); } } @@ -197,7 +197,7 @@ class Options { } } for (int i = 0; i < coords_strs.size(); i++) { - if (i % rank_n() != rank_me()) continue; + if (i % upcxx::rank_n() != upcxx::rank_me()) continue; auto coords_and_time = splitter(",", coords_strs[i]); if (coords_and_time.size() == 4) { try { @@ -255,7 +255,8 @@ class Options { int tcell_initial_delay = 10080; int tcell_vascular_period = 5760; int tcell_tissue_period = 1440; - int tcell_binding_period = 10; + // int tcell_binding_period = 10; + double tcell_binding_period = 10; double max_binding_prob = 1.0; double infectivity = 0.02; @@ -274,7 +275,7 @@ class Options { int antibody_period = 5760; unsigned rnd_seed = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - string output_dir = "simcov-results-n" + to_string(upcxx::rank_n()) + "-N" + + string output_dir = "simforager-results-n" + to_string(upcxx::rank_n()) + "-N" + to_string(upcxx::rank_n() / upcxx::local_team().rank_n()); int sample_period = 0; int sample_resolution = 1; @@ -287,8 +288,8 @@ class Options { bool load(int argc, char **argv) { // SIMCOV version v0.1-a0decc6-master (Release) built on 2020-04-08T22:15:40 with g++ - string full_version_str = "SimCov version " + string(SIMCOV_VERSION) + "-" + - string(SIMCOV_BRANCH) + " built on " + string(SIMCOV_BUILD_DATE); + string full_version_str = "SimForager version " + string(SIMFORAGER_VERSION) + "-" + + string(SIMFORAGER_BRANCH) + " built on " + string(SIMFORAGER_BUILD_DATE); vector infection_coords_strs; CLI::App app(full_version_str); @@ -419,7 +420,7 @@ class Options { upcxx::barrier(); - _rnd_gen = make_shared(rnd_seed + rank_me()); + _rnd_gen = make_shared(rnd_seed + upcxx::rank_me()); if (!lung_model_dir.empty()) { auto model_dims = get_model_dims(lung_model_dir + "/alveolus.dat"); @@ -427,7 +428,7 @@ class Options { for (int i = 0; i < 3; i++) { if (model_dims[i] != dimensions[i]) { dimensions = model_dims; - if (!rank_me()) + if (!upcxx::rank_me()) cerr << KLRED << "WARNING: " << KNORM << "Setting dimensions to model data: " << dimensions[0] << ", " << dimensions[1] << ", " << dimensions[2] << endl; @@ -437,7 +438,7 @@ class Options { } if (virion_clearance_rate * antibody_factor > 1.0) { - if (!rank_me()) + if (!upcxx::rank_me()) cerr << "Invalid parameter settings: virion-clearance * antibody_factor > 1.\n" << "Reduce either or both of those settings\n"; return false; @@ -453,7 +454,7 @@ class Options { if (dimensions[0] % sample_resolution || dimensions[1] % sample_resolution || (dimensions[2] > 1 && dimensions[2] % sample_resolution)) { - if (!rank_me()) + if (!upcxx::rank_me()) cerr << "Error: sample period " << sample_resolution << " must be a factor of all the dimensions\n"; return false; @@ -461,20 +462,20 @@ class Options { for (int i = 0; i < 3; i++) { if (dimensions[i] > whole_lung_dims[i]) { - if (!rank_me()) cerr << "Dimensions must be <= whole lung dimensions\n"; + if (!upcxx::rank_me()) cerr << "Dimensions must be <= whole lung dimensions\n"; return false; } } setup_output_dir(); setup_log_file(); - init_logger(output_dir + "/simcov.log", verbose); + init_logger(output_dir + "/simforager.log", verbose); #ifdef DEBUG open_dbg("debug"); #endif - SLOG(KLBLUE, "SimCov version ", full_version_str, KNORM, "\n"); + SLOG(KLBLUE, "SimForager version ", full_version_str, KNORM, "\n"); if (upcxx::rank_me() == 0) { // print out all compiler definitions @@ -498,7 +499,7 @@ class Options { #endif if (!upcxx::rank_me()) { // write out configuration file for restarts - ofstream ofs(output_dir + "/simcov.config"); + ofstream ofs(output_dir + "/simforager.config"); ofs << app.config_to_str(true, true); } upcxx::barrier(); diff --git a/include/version.h b/include/version.h index db1ef2a..8404519 100644 --- a/include/version.h +++ b/include/version.h @@ -4,10 +4,10 @@ extern "C" { #endif -extern const char *SIMCOV_VERSION; -extern const char *SIMCOV_VERSION_DATE; -extern const char *SIMCOV_BUILD_DATE; -extern const char *SIMCOV_BRANCH; +extern const char *SIMFORAGER_VERSION; +extern const char *SIMFORAGER_VERSION_DATE; +extern const char *SIMFORAGER_BUILD_DATE; +extern const char *SIMFORAGER_BRANCH; #define MAX_BUILD_KMER_STR "Maximum kmer len=MAX_BUILD_KMER" diff --git a/simforager_null.config b/simforager_null.config new file mode 100644 index 0000000..f613b55 --- /dev/null +++ b/simforager_null.config @@ -0,0 +1,119 @@ +; These are version 1.0 test of 'null' forager parameters for SIMCoV +; This means a parameter-based translation of SIMCoV into SIMForager without yet +; changing anything about the existing SIMCoV code base + +; Dimensions: x y z (assuming grid spacing of 5 microns) +dim = 500 500 1 +; >> resulting in 225000000 total environment (epithelial) cells + +; Whole lung dimensions: x y z (assuming grid spacing of 5 microns) +whole-lung-dim = 500 500 1 + +; Number of timesteps (assuming 1 min per timestep) +; Set to arbitrary small number to start +timesteps = 14400 +; >> ten days + +; Location of initial infections (omit for no infections); can be: +; list of space-separated coords x,y,z,timestep +; uniform:N (where N is an int, for N uniformly distributed starting points at time 0) +; random:N (where N is an int, for N randomly distributed starting points at time 0) +infection-coords = uniform:22500 +; >> this is .001% of total grid cells + +; Number of virions at initial infection locations +; Chosen to guarantee that at least one cell is infected at the start - see infectivity +initial-infection = 1 +; >> initially 1000 combined with number of initial infection sites, this should yield a 1:100 ratio of grid cells to algae cells + +; Average number of time steps to expressing virions after cell is infected +incubation-period = 0 + +; Average number of time steps to death after apoptosis is induced +apoptosis-period = 3 + +; Average number of time steps to death after a cell starts expresssing +expressing-period = 1000000 + +; Factor multiplied by number of virions to determine probability of infection +infectivity = 0.5 +; SIMCoV paper default + +; Multiplier reducing infectivity where inflammatory signal is present +; (the multiplier reducing infectivity should be between 0.9 and 0.7 based on values +; in literature demonstrating a reduction of cell death in vitro with IFN) +infectivity-multiplier = 1.0 +; >> irrelevant + +; Number of virions produced by expressing cell each time step +virion-production = 1.1 +; this is SIMCoV default we're going with for now... + +; Multiplier reducing virion production rate where inflammatory signal is present +; (the reduction should be represented by a multiplier betweeon 0.05 and 0.15) +virion-production-multiplier = 1.0 + +; Fraction by which virion count drops each time step +virion-clearance = 0.0 +; >> effectively algae death rate; not considered here + +; Fraction of virions that diffuse into all neighbors each time step +virion-diffusion = 0.0 + +; Amount of chemokine produced by expressing cells each time step +chemokine-production = 0.0 + +; Amount by which chemokine concentration drops each time step +chemokine-decay = 0.0 + +; Fraction of chemokine concentration that diffuses into all neighbors each time step +chemokine-diffusion = 1.0 + +; Minimum chemokine concentration that triggers a T cell +min-chemokine = 0.0 +; >> this should allow fish (T-cells) to be present at start + +; Impact of antibodies; multiplier for virion decay (setting to 1 means this has no effect) +antibody-factor = 1 + +; Number of time steps before antibodies start to be produced +antibody-period = 1000000 +; >> WHAT!? NOBODY INVITED ANTIBODIES TO A REEF + +; Number of tcells generated at each timestep for the whole lung +; This is scaled up 5000x from the mouse model (20 per minute) +tcell-generation-rate = 100 +; >> for a stable population of fish (T-cells) throughout sim + +; Number of time steps before T cells start to be produced +tcell-initial-delay = 0 + +; Average number of time steps to death for a T cell in the vasculature +tcell-vascular-period = 2 + +; Average number of time steps to death after a T cell extravasates +tcell-tissue-period = 1000000 + +; Number of time steps a T cell is bound to an epithelial cell when inducing apoptosis +tcell-binding-period = 2 + +; Max probability of a T cell binding to an infected cell in one time step +max-binding-prob = 1 + +; T cells in tissue follow the chemokine gradient +tcells-follow-gradient = false + +; Random seed +seed = 29 + +; Number of timesteps between samples (set to 0 to disable sampling) +sample-period = 5 + +; Resolution for sampling +sample-resolution = 1 + +; Max. block dimension - larger means more locality but worse load balance. Set to 0 for largest possible. +max-block-dim = 10 + +; Output directory (automatically generated) +; output = results diff --git a/slurm_hopper.sh b/slurm_hopper.sh index 688de18..05d71fa 100644 --- a/slurm_hopper.sh +++ b/slurm_hopper.sh @@ -3,7 +3,7 @@ #SBATCH --nodes=2 #SBATCH --ntasks-per-node=32 #SBATCH --time=2-00:00 -#SBATCH --job-name=simcov-tutorial +#SBATCH --job-name=simforagger-tutorial #SBATCH --mail-user= #SBATCH --mail-type=END @@ -27,4 +27,4 @@ module load cmake/3.11.4-qkyj module load openmpi/4.1.3-j6zb module load upcxx/2020.10.0-6eh2 -srun --mpi=pmi2 install/bin/simcov --config covid_default.config +srun --mpi=pmi2 install/bin/simforager --config simforager_null.config diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7aab124..4cbe837 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,9 +30,9 @@ endforeach() add_definitions(-DALL_DEFNS=\"${all_defs}\") -add_executable(simcov main.cpp tissue.cpp utils.cpp) +add_executable(simforager main.cpp tissue.cpp utils.cpp) -target_link_libraries(simcov Threads::Threads ${UPCXX_LIBRARIES} ${UPCXX_UTILS_LIBRARIES} SIMCOV_VERSION_LIB) +target_link_libraries(simforager Threads::Threads ${UPCXX_LIBRARIES} ${UPCXX_UTILS_LIBRARIES} SIMFORAGER_VERSION_LIB) #install(PROGRAMS simcov.py DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) -install(TARGETS simcov DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) +install(TARGETS simforager DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) diff --git a/src/main.cpp b/src/main.cpp index 68e708f..22173a0 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -43,7 +43,7 @@ class SimStats { float virions = 0; void init() { - if (!rank_me()) { + if (!upcxx::rank_me()) { log_file.open(_options->output_dir + "/simcov.stats"); log_file << "# time" << header(0) << endl; } @@ -65,16 +65,16 @@ class SimStats { string to_str(int width) { vector totals; - totals.push_back(reduce_one(incubating, op_fast_add, 0).wait()); - totals.push_back(reduce_one(expressing, op_fast_add, 0).wait()); - totals.push_back(reduce_one(apoptotic, op_fast_add, 0).wait()); - totals.push_back(reduce_one(dead, op_fast_add, 0).wait()); - totals.push_back(reduce_one(tcells_vasculature, op_fast_add, 0).wait()); - totals.push_back(reduce_one(tcells_tissue, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(incubating, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(expressing, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(apoptotic, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(dead, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(tcells_vasculature, op_fast_add, 0).wait()); + totals.push_back(upcxx::reduce_one(tcells_tissue, op_fast_add, 0).wait()); vector totals_d; - totals_d.push_back(reduce_one(chemokines, op_fast_add, 0).wait() / get_num_grid_points()); - totals_d.push_back(reduce_one(virions, op_fast_add, 0).wait()); // / get_num_grid_points()); - auto all_chem_pts = reduce_one(num_chemo_pts, op_fast_add, 0).wait(); + totals_d.push_back(upcxx::reduce_one(chemokines, op_fast_add, 0).wait() / get_num_grid_points()); + totals_d.push_back(upcxx::reduce_one(virions, op_fast_add, 0).wait()); // / get_num_grid_points()); + auto all_chem_pts = upcxx::reduce_one(num_chemo_pts, op_fast_add, 0).wait(); totals_d.push_back(all_chem_pts + totals[0] + totals[1] + totals[2] + totals[3]); auto perc_infected = 100.0 * (float)(totals[0] + totals[1] + totals[2] + totals[3]) / get_num_grid_points(); @@ -104,7 +104,7 @@ class SimStats { void log(int time_step) { string s = to_str(0); - if (!rank_me()) log_file << time_step << s << endl; + if (!upcxx::rank_me()) log_file << time_step << s << endl; } }; @@ -151,21 +151,21 @@ void seed_infection(Tissue &tissue, int time_step) { _options->infection_coords.erase(it--); } } - barrier(); + upcxx::barrier(); tissue.add_new_actives(add_new_actives_timer); - barrier(); + upcxx::barrier(); } void generate_tcells(Tissue &tissue, int time_step) { generate_tcell_timer.start(); - int local_num = _options->tcell_generation_rate / rank_n(); - int rem = _options->tcell_generation_rate - local_num * rank_n(); - if (rank_me() < rem) local_num++; + int local_num = _options->tcell_generation_rate / upcxx::rank_n(); + int rem = _options->tcell_generation_rate - local_num * upcxx::rank_n(); + if (upcxx::rank_me() < rem) local_num++; if (time_step == 1) WARN("rem ", rem, " local num ", local_num, "\n"); tissue.change_num_circulating_tcells(local_num); #ifdef DEBUG - auto all_num = reduce_one(local_num, op_fast_add, 0).wait(); - if (!rank_me() && all_num != _options->tcell_generation_rate) + auto all_num = upcxx::reduce_one(local_num, op_fast_add, 0).wait(); + if (!upcxx::rank_me() && all_num != _options->tcell_generation_rate) DIE("num tcells generated ", all_num, " != generation rate ", _options->tcell_generation_rate); #endif generate_tcell_timer.stop(); @@ -188,11 +188,13 @@ void update_circulating_tcells(int time_step, Tissue &tissue, double extravasate tissue.change_num_circulating_tcells(-num_dying); _sim_stats.tcells_vasculature -= num_dying; num_circulating = tissue.get_num_circulating_tcells(); - double portion_xtravasing = extravasate_fraction * num_circulating; + double portion_xtravasing = (time_step <= 15) ? extravasate_fraction * num_circulating : 0; + //while (time_step < 20) double portion_xtravasing = ? extravasate_fraction * num_circulating : 0; + //double portion_xtravasing = ? extravasate_fraction * num_circulating; int num_xtravasing = floor(portion_xtravasing); if (_rnd_gen->trial_success(portion_xtravasing - num_xtravasing)) num_xtravasing++; for (int i = 0; i < num_xtravasing; i++) { - progress(); + upcxx::progress(); GridCoords coords(_rnd_gen); if (tissue.try_add_new_tissue_tcell(coords.to_1d())) { _sim_stats.tcells_tissue++; @@ -226,6 +228,8 @@ void update_tissue_tcell(int time_step, Tissue &tissue, GridPoint *grid_point, v if (tcell->binding_period != -1) { DBG(time_step, " tcell ", tcell->id, " is bound at ", grid_point->coords.str(), "\n"); // this tcell is bound + //grid_point->nb_virions -= 100; + grid_point->virions -= 42; //arbitrary large bite size tcell->binding_period--; // done with binding when set to -1 if (tcell->binding_period == 0) tcell->binding_period = -1; @@ -486,7 +490,7 @@ void sample(int time_step, vector &samples, int64_t start_id, ViewOb header_oss << " unsigned_char\n" << "LOOKUP_TABLE default\n"; auto header_str = header_oss.str(); - if (!rank_me()) { + if (!upcxx::rank_me()) { tot_sz += header_str.size(); // rank 0 creates the file and truncates it to the correct length auto fileno = open(fname.c_str(), O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); @@ -559,9 +563,9 @@ int64_t get_samples(Tissue &tissue, vector &samples) { int64_t num_points = get_num_grid_points() / (_options->sample_resolution * _options->sample_resolution); if (_grid_size->z > 1) num_points /= _options->sample_resolution; - int64_t num_points_per_rank = ceil((double)num_points / rank_n()); - int64_t start_id = rank_me() * num_points_per_rank; - int64_t end_id = min((rank_me() + 1) * num_points_per_rank, num_points); + int64_t num_points_per_rank = ceil((double)num_points / upcxx::rank_n()); + int64_t start_id = upcxx::rank_me() * num_points_per_rank; + int64_t end_id = min((upcxx::rank_me() + 1) * num_points_per_rank, num_points); samples.clear(); if (end_id > start_id) { samples.reserve(end_id - start_id); @@ -578,7 +582,7 @@ int64_t get_samples(Tissue &tissue, vector &samples) { break; } if (i >= start_id) { - progress(); + upcxx::progress(); #ifdef AVERAGE_SUBSAMPLE float virions = 0; float chemokine = 0; @@ -646,8 +650,8 @@ int64_t get_samples(Tissue &tissue, vector &samples) { if (done) break; } } - barrier(); - auto samples_written = reduce_one(samples.size(), op_fast_add, 0).wait(); + upcxx::barrier(); + auto samples_written = upcxx::reduce_one(samples.size(), op_fast_add, 0).wait(); if (num_points != samples_written) SWARN("Number of point ", num_points, " != ", samples_written, " samples written"); SLOG_VERBOSE("Number of samples written ", samples_written, "\n"); @@ -680,7 +684,7 @@ void run_sim(Tissue &tissue) { for (int time_step = 0; time_step < _options->num_timesteps; time_step++) { DBG("Time step ", time_step, "\n"); seed_infection(tissue, time_step); - barrier(); + upcxx::barrier(); if (time_step == _options->antibody_period) _options->virion_clearance_rate *= _options->antibody_factor; chemokines_to_update.clear(); @@ -688,7 +692,7 @@ void run_sim(Tissue &tissue) { chemokines_cache.clear(); if (time_step > _options->tcell_initial_delay) { generate_tcells(tissue, time_step); - barrier(); + upcxx::barrier(); } compute_updates_timer.start(); update_circulating_tcells(time_step, tissue, extravasate_fraction); @@ -724,31 +728,31 @@ void run_sim(Tissue &tissue) { update_virions(grid_point, *nbs, virions_to_update); if (grid_point->is_active()) tissue.set_active(grid_point); } - barrier(); + upcxx::barrier(); compute_updates_timer.stop(); tissue.accumulate_chemokines(chemokines_to_update, accumulate_concentrations_timer); tissue.accumulate_virions(virions_to_update, accumulate_concentrations_timer); - barrier(); + upcxx::barrier(); if (time_step % five_perc == 0 || time_step == _options->num_timesteps - 1) { - auto num_actives = reduce_one(tissue.get_num_actives(), op_fast_add, 0).wait(); + auto num_actives = upcxx::reduce_one(tissue.get_num_actives(), op_fast_add, 0).wait(); auto perc_actives = 100.0 * num_actives / get_num_grid_points(); - auto max_actives = reduce_one(tissue.get_num_actives(), op_fast_max, 0).wait(); - auto load_balance = max_actives ? (double)num_actives / rank_n() / max_actives : 1; + auto max_actives = upcxx::reduce_one(tissue.get_num_actives(), op_fast_max, 0).wait(); + auto load_balance = max_actives ? (double)num_actives / upcxx::rank_n() / max_actives : 1; chrono::duration t_elapsed = NOW() - curr_t; curr_t = NOW(); SLOG("[", get_current_time(), " ", setprecision(2), fixed, setw(7), right, t_elapsed.count(), "s]: ", setw(8), left, time_step, _sim_stats.to_str(STATS_COL_WIDTH), setprecision(3), fixed, "< ", perc_actives, " ", load_balance, " >\n"); } - barrier(); + upcxx::barrier(); tissue.add_new_actives(add_new_actives_timer); - barrier(); + upcxx::barrier(); _sim_stats.virions = 0; _sim_stats.chemokines = 0; _sim_stats.num_chemo_pts = 0; set_active_grid_points(tissue); - barrier(); + upcxx::barrier(); if (_options->sample_period > 0 && (time_step % _options->sample_period == 0 || time_step == _options->num_timesteps - 1)) { @@ -764,13 +768,13 @@ void run_sim(Tissue &tissue) { log_timer.start(); _sim_stats.log(time_step); - barrier(); + upcxx::barrier(); log_timer.stop(); #ifdef DEBUG DBG("check actives ", time_step, "\n"); tissue.check_actives(time_step); - barrier(); + upcxx::barrier(); #endif } @@ -799,11 +803,11 @@ int main(int argc, char **argv) { _options = make_shared(); if (!_options->load(argc, argv)) return 0; ProgressBar::SHOW_PROGRESS = _options->show_progress; - if (pin_thread(getpid(), local_team().rank_me()) == -1) - WARN("Could not pin process ", getpid(), " to core ", rank_me()); + if (pin_thread(getpid(), upcxx::local_team().rank_me()) == -1) + WARN("Could not pin process ", getpid(), " to core ", upcxx::rank_me()); else SLOG_VERBOSE("Pinned processes, with process 0 (pid ", getpid(), ") pinned to core ", - local_team().rank_me(), "\n"); + upcxx::local_team().rank_me(), "\n"); #ifdef BLOCK_PARTITION SLOG_VERBOSE("Using block partitioning\n"); #else @@ -823,8 +827,8 @@ int main(int argc, char **argv) { memory_tracker.stop(); chrono::duration t_elapsed = NOW() - start_t; SLOG("Finished in ", setprecision(2), fixed, t_elapsed.count(), " s at ", get_current_time(), - " for SimCov version ", SIMCOV_VERSION, "\n"); - barrier(); + " for SimForager version ", SIMFORAGER_VERSION, "\n"); + upcxx::barrier(); upcxx::finalize(); return 0; } diff --git a/src/tissue.cpp b/src/tissue.cpp index 06400de..4560c18 100644 --- a/src/tissue.cpp +++ b/src/tissue.cpp @@ -170,8 +170,8 @@ static int get_cube_block_dim(int64_t num_grid_points) { size_t cube = (size_t)pow((double)d, 3.0); size_t num_cubes = num_grid_points / cube; DBG("cube size ", cube, " num cubes ", num_cubes, "\n"); - if (num_cubes < rank_n() * MIN_BLOCKS_PER_PROC) { - DBG("not enough cubes ", num_cubes, " < ", rank_n() * MIN_BLOCKS_PER_PROC, "\n"); + if (num_cubes < upcxx::rank_n() * MIN_BLOCKS_PER_PROC) { + DBG("not enough cubes ", num_cubes, " < ", upcxx::rank_n() * MIN_BLOCKS_PER_PROC, "\n"); break; } // there is a remainder - this is not a perfect division @@ -180,7 +180,7 @@ static int get_cube_block_dim(int64_t num_grid_points) { continue; } // skip sizes that distribute the blocks in columns - if (d > 1 && (_grid_size->x % (d * rank_n()) == 0 || _grid_size->y % (d * rank_n()) == 0)) { + if (d > 1 && (_grid_size->x % (d * upcxx::rank_n()) == 0 || _grid_size->y % (d * upcxx::rank_n()) == 0)) { DBG("dim ", d, " gives perfect division of all blocks into x axis - skip\n"); continue; } @@ -201,8 +201,8 @@ static int get_square_block_dim(int64_t num_grid_points) { size_t square = (size_t)pow((double)d, 2.0); size_t num_squares = num_grid_points / square; DBG("square size ", square, " num squares ", num_squares, "\n"); - if (num_squares < rank_n() * MIN_BLOCKS_PER_PROC) { - DBG("not enough squares ", num_squares, " < ", rank_n() * MIN_BLOCKS_PER_PROC, "\n"); + if (num_squares < upcxx::rank_n() * MIN_BLOCKS_PER_PROC) { + DBG("not enough squares ", num_squares, " < ", upcxx::rank_n() * MIN_BLOCKS_PER_PROC, "\n"); break; } // there is a remainder - this is not a perfect division @@ -211,7 +211,7 @@ static int get_square_block_dim(int64_t num_grid_points) { continue; } // skip sizes that distribute the blocks in columns - if (d > 1 && _grid_size->x % (d * rank_n()) == 0) { + if (d > 1 && _grid_size->x % (d * upcxx::rank_n()) == 0) { DBG("dim ", d, " gives perfect division of all blocks into x axis - skip\n"); continue; } @@ -259,7 +259,7 @@ Tissue::Tissue() int64_t num_blocks = num_grid_points / _grid_blocks.block_size; - int64_t blocks_per_rank = ceil((double)num_blocks / rank_n()); + int64_t blocks_per_rank = ceil((double)num_blocks / upcxx::rank_n()); bool threeD = _grid_size->z > 1; SLOG("Dividing ", num_grid_points, " grid points into ", num_blocks, @@ -288,7 +288,7 @@ Tissue::Tissue() // FIXME: these blocks need to be stride distributed to better load balance grid_points->reserve(blocks_per_rank * _grid_blocks.block_size); for (int64_t i = 0; i < blocks_per_rank; i++) { - int64_t start_id = (i * rank_n() + rank_me()) * _grid_blocks.block_size; + int64_t start_id = (i * upcxx::rank_n() + upcxx::rank_me()) * _grid_blocks.block_size; if (start_id >= num_grid_points) break; for (auto id = start_id; id < start_id + _grid_blocks.block_size; id++) { assert(id < num_grid_points); @@ -322,7 +322,7 @@ Tissue::Tissue() #endif } } - barrier(); + upcxx::barrier(); } int Tissue::load_data_file(const string &fname, int num_grid_points, EpiCellType epicell_type) { @@ -353,11 +353,11 @@ int Tissue::load_data_file(const string &fname, int num_grid_points, EpiCellType intrank_t Tissue::get_rank_for_grid_point(int64_t grid_i) { int64_t block_i = grid_i / _grid_blocks.block_size; - return block_i % rank_n(); + return block_i % upcxx::rank_n(); } GridPoint *Tissue::get_local_grid_point(grid_points_t &grid_points, int64_t grid_i) { - int64_t block_i = grid_i / _grid_blocks.block_size / rank_n(); + int64_t block_i = grid_i / _grid_blocks.block_size / upcxx::rank_n(); int64_t i = grid_i % _grid_blocks.block_size + block_i * _grid_blocks.block_size; assert(i < grid_points->size()); GridPoint *grid_point = &(*grid_points)[i]; @@ -441,13 +441,13 @@ void Tissue::accumulate_chemokines(HASH_TABLE &chemokines_to_upd // accumulate updates for each target rank HASH_TABLE>> target_rank_updates; for (auto &[coords_1d, chemokines] : chemokines_to_update) { - progress(); + upcxx::progress(); target_rank_updates[get_rank_for_grid_point(coords_1d)].push_back({coords_1d, chemokines}); } future<> fut_chain = make_future<>(); // dispatch all updates to each target rank in turn for (auto &[target_rank, update_vector] : target_rank_updates) { - progress(); + upcxx::progress(); auto fut = rpc( target_rank, [](grid_points_t &grid_points, new_active_grid_points_t &new_active_grid_points, @@ -473,13 +473,13 @@ void Tissue::accumulate_virions(HASH_TABLE &virions_to_update, // accumulate updates for each target rank HASH_TABLE>> target_rank_updates; for (auto &[coords_1d, virions] : virions_to_update) { - progress(); + upcxx::progress(); target_rank_updates[get_rank_for_grid_point(coords_1d)].push_back({coords_1d, virions}); } future<> fut_chain = make_future<>(); // dispatch all updates to each target rank in turn for (auto &[target_rank, update_vector] : target_rank_updates) { - progress(); + upcxx::progress(); auto fut = rpc( target_rank, [](grid_points_t &grid_points, new_active_grid_points_t &new_active_grid_points, @@ -525,7 +525,7 @@ bool Tissue::try_add_new_tissue_tcell(int64_t grid_i) { if (grid_point->tcell) return false; if (grid_point->chemokine < _options->min_chemokine) return false; new_active_grid_points->insert({grid_point, true}); - string tcell_id = to_string(rank_me()) + "-" + to_string(*tcells_generated); + string tcell_id = to_string(upcxx::rank_me()) + "-" + to_string(*tcells_generated); (*tcells_generated)++; grid_point->tcell = new TCell(tcell_id); grid_point->tcell->moved = true; diff --git a/src/utils.cpp b/src/utils.cpp index 997fcec..26bec92 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -42,7 +42,7 @@ void dump_single_file(const string &fname, const string &out_str) { upcxx::barrier(); ad.destroy(); // wait until rank 0 has finished setting up the file - if (rank_me()) fileno = open(fname.c_str(), O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (upcxx::rank_me()) fileno = open(fname.c_str(), O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fileno == -1) WARN("Error trying to open file ", fname, ": ", strerror(errno), "\n"); auto bytes_written = pwrite(fileno, out_str.c_str(), sz, my_fpos); close(fileno);