Skip to content

Commit

Permalink
Merge pull request #11134 from jwillenbring/trilinos-13-4-superlu-dis…
Browse files Browse the repository at this point in the history 
…t-8-patch

Trilinos 13 4 superlu dist 8 patch

This was tested by the xSDK team and confirmed to fix the issue they were seeing.
  • Loading branch information
@jwillenbring
jwillenbring committed Nov 9, 2022
2 parents 0976cf2 + e275556 commit 403f2cd
Show file tree
Hide file tree
Showing 2 changed files with 181 additions and 7 deletions.
150 changes: 143 additions & 7 deletions packages/amesos2/src/Amesos2_Superludist_FunctionMap.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,14 @@ namespace Amesos2 {
* \brief Solve the system A*X=B or A'*X=B using the L and U factors
* of A.
*/
#if SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void gstrs(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n, type_map::LUstruct_t* lu_struct,
SLUD::D::dScalePermstruct_t* scale_perm_struct, SLUD::gridinfo_t* grid,
type_map::type* B, SLUD::int_t l_numrows, SLUD::int_t fst_global_row,
SLUD::int_t ldb, int nrhs, type_map::SOLVEstruct_t* solve_struct,
SLUD::SuperLUStat_t* stat, int* info)
#elif SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
static void gstrs(SLUD::int_t n, type_map::LUstruct_t* lu_struct,
SLUD::D::dScalePermstruct_t* scale_perm_struct, SLUD::gridinfo_t* grid,
type_map::type* B, SLUD::int_t l_numrows, SLUD::int_t fst_global_row,
Expand All @@ -148,8 +155,13 @@ namespace Amesos2 {
SLUD::SuperLUStat_t* stat, int* info)
#endif
{
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
SLUD::D::pdgstrs(options, n, lu_struct, scale_perm_struct, grid, B, l_numrows,
fst_global_row, ldb, nrhs, solve_struct, stat, info);
#else
SLUD::D::pdgstrs(n, lu_struct, scale_perm_struct, grid, B, l_numrows,
fst_global_row, ldb, nrhs, solve_struct, stat, info);
#endif
}

/**
Expand All @@ -159,18 +171,40 @@ namespace Amesos2 {
* useful if the matrix is small enough to fit in memory on a single
* processor.
*/
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void gstrs_Bglobal(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n, type_map::LUstruct_t* lu_struct,
SLUD::gridinfo_t* grid, type_map::type* B,
SLUD::int_t ldb, int nrhs,
SLUD::SuperLUStat_t* stat, int* info)

{
SLUD::D::pdgstrs_Bglobal(options, n, lu_struct, grid, B, ldb, nrhs, stat, info);
}
#else
static void gstrs_Bglobal(SLUD::int_t n, type_map::LUstruct_t* lu_struct,
SLUD::gridinfo_t* grid, type_map::type* B,
SLUD::int_t ldb, int nrhs,
SLUD::SuperLUStat_t* stat, int* info)

{
SLUD::D::pdgstrs_Bglobal(n, lu_struct, grid, B, ldb, nrhs, stat, info);
}
#endif

/**
* \brief Use iterative refined to improve the solution.
*/
#if SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void gsrfs(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n, SLUD::SuperMatrix* A, double anorm,
type_map::LUstruct_t* lu_struct,
SLUD::D::dScalePermstruct_t* scale_perm,
SLUD::gridinfo_t* grid, type_map::type* B, SLUD::int_t ldb,
type_map::type* X, SLUD::int_t ldx, int nrhs,
type_map::SOLVEstruct_t* solve_struct, double* berr,
SLUD::SuperLUStat_t* stat, int* info)
#elif SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
static void gsrfs(SLUD::int_t n, SLUD::SuperMatrix* A, double anorm,
type_map::LUstruct_t* lu_struct,
SLUD::D::dScalePermstruct_t* scale_perm,
Expand All @@ -188,8 +222,13 @@ namespace Amesos2 {
SLUD::SuperLUStat_t* stat, int* info)
#endif
{
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
SLUD::D::pdgsrfs(options, n, A, anorm, lu_struct, scale_perm, grid, B, ldb,
X, ldx, nrhs, solve_struct, berr, stat, info);
#else
SLUD::D::pdgsrfs(n, A, anorm, lu_struct, scale_perm, grid, B, ldb,
X, ldx, nrhs, solve_struct, berr, stat, info);
#endif
}

/**
Expand All @@ -199,6 +238,18 @@ namespace Amesos2 {
* all calling processors. This method is available in case the
* user requests such functionality.
*/
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void gsrfs_ABXglobal(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n, SLUD::SuperMatrix* A, double anorm,
type_map::LUstruct_t* lu_struct, SLUD::gridinfo_t* grid,
type_map::type* B, SLUD::int_t ldb, type_map::type* X,
SLUD::int_t ldx, int nrhs, double* berr,
SLUD::SuperLUStat_t* stat, int* info)
{
SLUD::D::pdgsrfs_ABXglobal(options, n, A, anorm, lu_struct, grid, B, ldb,
X, ldx, nrhs, berr, stat, info);
}
#else
static void gsrfs_ABXglobal(SLUD::int_t n, SLUD::SuperMatrix* A, double anorm,
type_map::LUstruct_t* lu_struct, SLUD::gridinfo_t* grid,
type_map::type* B, SLUD::int_t ldb, type_map::type* X,
Expand All @@ -208,6 +259,7 @@ namespace Amesos2 {
SLUD::D::pdgsrfs_ABXglobal(n, A, anorm, lu_struct, grid, B, ldb,
X, ldx, nrhs, berr, stat, info);
}
#endif

/**
* \brief Creates a SuperLU_DIST distributed CRS matrix using the
Expand Down Expand Up @@ -329,12 +381,22 @@ namespace Amesos2 {
/*
* This version suitable for A in NCPformat
*/
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void distribute(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::fact_t fact, SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::Glu_freeable_t* glu_freeable,
type_map::LUstruct_t* lu, SLUD::gridinfo_t* grid)
{
SLUD::D::ddistribute(options, n, A, glu_freeable, lu, grid);
}
#else
static void distribute(SLUD::fact_t fact, SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::Glu_freeable_t* glu_freeable,
type_map::LUstruct_t* lu, SLUD::gridinfo_t* grid)
{
SLUD::D::ddistribute(fact, n, A, glu_freeable, lu, grid);
}
#endif

/*
* This version suitable for A in NR_loc format.
Expand All @@ -343,7 +405,13 @@ namespace Amesos2 {
* SamePattern_SameRowPerm, otherwise dist_psymbtonum should be
* called.o
*/
#if SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void pdistribute(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::D::dScalePermstruct_t* scale_perm,
SLUD::Glu_freeable_t* glu_freeable, type_map::LUstruct_t* lu,
SLUD::gridinfo_t* grid)
#elif SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
static void pdistribute(SLUD::fact_t fact, SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::D::dScalePermstruct_t* scale_perm,
SLUD::Glu_freeable_t* glu_freeable, type_map::LUstruct_t* lu,
Expand All @@ -355,7 +423,11 @@ namespace Amesos2 {
SLUD::gridinfo_t* grid)
#endif
{
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
SLUD::D::pddistribute(options, n, A, scale_perm, glu_freeable, lu, grid);
#else
SLUD::D::pddistribute(fact, n, A, scale_perm, glu_freeable, lu, grid);
#endif
}

/*
Expand All @@ -366,7 +438,13 @@ namespace Amesos2 {
*
* This routine should always be called with fact == DOFACT
*/
#if SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void dist_psymbtonum(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n, SLUD::SuperMatrix* A,
SLUD::D::dScalePermstruct_t* scale_perm,
SLUD::Pslu_freeable_t* pslu_freeable,
type_map::LUstruct_t* lu, SLUD::gridinfo_t* grid)
#elif SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
static void dist_psymbtonum(SLUD::fact_t fact, SLUD::int_t n, SLUD::SuperMatrix* A,
SLUD::D::dScalePermstruct_t* scale_perm,
SLUD::Pslu_freeable_t* pslu_freeable,
Expand All @@ -378,7 +456,11 @@ namespace Amesos2 {
type_map::LUstruct_t* lu, SLUD::gridinfo_t* grid)
#endif
{
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
SLUD::D::ddist_psymbtonum(options, n, A, scale_perm, pslu_freeable, lu, grid);
#else
SLUD::D::ddist_psymbtonum(fact, n, A, scale_perm, pslu_freeable, lu, grid);
#endif
}

/*
Expand Down Expand Up @@ -462,7 +544,16 @@ namespace Amesos2 {
SLUD::Z::pzgstrf(options, m, n, anorm, LU, grid, stat, info);
}

#if SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void gstrs(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n, type_map::LUstruct_t* lu_struct,
SLUD::Z::zScalePermstruct_t* scale_perm_struct,
SLUD::gridinfo_t* grid, type_map::type* B,
SLUD::int_t l_numrows, SLUD::int_t fst_global_row,
SLUD::int_t ldb, int nrhs,
type_map::SOLVEstruct_t* solve_struct,
SLUD::SuperLUStat_t* stat, int* info)
#elif SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
static void gstrs(SLUD::int_t n, type_map::LUstruct_t* lu_struct,
SLUD::Z::zScalePermstruct_t* scale_perm_struct,
SLUD::gridinfo_t* grid, type_map::type* B,
Expand All @@ -479,16 +570,31 @@ namespace Amesos2 {
type_map::SOLVEstruct_t* solve_struct,
#endif
{
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
SLUD::Z::pzgstrs(options, n, lu_struct, scale_perm_struct, grid, B, l_numrows,
fst_global_row, ldb, nrhs, solve_struct, stat, info);
#else
SLUD::Z::pzgstrs(n, lu_struct, scale_perm_struct, grid, B, l_numrows,
fst_global_row, ldb, nrhs, solve_struct, stat, info);
#endif
}

#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void gstrs_Bglobal(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n, type_map::LUstruct_t* lu_struct,
SLUD::gridinfo_t* grid, type_map::type* B,
SLUD::int_t ldb, int nrhs, SLUD::SuperLUStat_t* stat, int* info)
{
SLUD::Z::pzgstrs_Bglobal(options,n, lu_struct, grid, B, ldb, nrhs, stat, info);
}
#else
static void gstrs_Bglobal(SLUD::int_t n, type_map::LUstruct_t* lu_struct,
SLUD::gridinfo_t* grid, type_map::type* B,
SLUD::int_t ldb, int nrhs, SLUD::SuperLUStat_t* stat, int* info)
{
SLUD::Z::pzgstrs_Bglobal(n, lu_struct, grid, B, ldb, nrhs, stat, info);
}
#endif

static void create_CompRowLoc_Matrix(SLUD::SuperMatrix* A, SLUD::int_t g_numrows,
SLUD::int_t g_numcols, SLUD::int_t l_nnz,
Expand Down Expand Up @@ -560,14 +666,30 @@ namespace Amesos2 {
*equed = AMESOS2_SLUD_GET_DIAG_SCALE(eq);
}

#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void distribute(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::Glu_freeable_t* glu_freeable,
type_map::LUstruct_t* lu, SLUD::gridinfo_t* grid)
{
SLUD::Z::zdistribute(options, n, A, glu_freeable, lu, grid);
}
#else
static void distribute(SLUD::fact_t fact, SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::Glu_freeable_t* glu_freeable,
type_map::LUstruct_t* lu, SLUD::gridinfo_t* grid)
{
SLUD::Z::zdistribute(fact, n, A, glu_freeable, lu, grid);
}
#endif

#if SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void pdistribute(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::Z::zScalePermstruct_t* scale_perm,
SLUD::Glu_freeable_t* glu_freeable, type_map::LUstruct_t* lu,
SLUD::gridinfo_t* grid)
#elif SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
static void pdistribute(SLUD::fact_t fact, SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::Z::zScalePermstruct_t* scale_perm,
SLUD::Glu_freeable_t* glu_freeable, type_map::LUstruct_t* lu,
Expand All @@ -579,10 +701,20 @@ namespace Amesos2 {
SLUD::gridinfo_t* grid)
#endif
{
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
SLUD::Z::pzdistribute(options, n, A, scale_perm, glu_freeable, lu, grid);
#else
SLUD::Z::pzdistribute(fact, n, A, scale_perm, glu_freeable, lu, grid);
#endif
}

#if SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
static void dist_psymbtonum(SLUD::amesos2_superlu_dist_options_t* options,
SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::Z::zScalePermstruct_t* scale_perm,
SLUD::Pslu_freeable_t* pslu_freeable, type_map::LUstruct_t* lu,
SLUD::gridinfo_t* grid)
#elif SUPERLU_DIST_MAJOR_VERSION > 6 || (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
static void dist_psymbtonum(SLUD::fact_t fact, SLUD::int_t n,
SLUD::SuperMatrix* A, SLUD::Z::zScalePermstruct_t* scale_perm,
SLUD::Pslu_freeable_t* pslu_freeable, type_map::LUstruct_t* lu,
Expand All @@ -594,7 +726,11 @@ namespace Amesos2 {
SLUD::gridinfo_t* grid)
#endif
{
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
SLUD::Z::zdist_psymbtonum(options, n, A, scale_perm, pslu_freeable, lu, grid);
#else
SLUD::Z::zdist_psymbtonum(fact, n, A, scale_perm, pslu_freeable, lu, grid);
#endif
}

static double plangs(char* norm, SLUD::SuperMatrix* A, SLUD::gridinfo_t* grid)
Expand Down
38 changes: 38 additions & 0 deletions packages/amesos2/src/Amesos2_Superludist_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,12 +391,23 @@ namespace Amesos2 {
#ifdef HAVE_AMESOS2_TIMERS
Teuchos::TimeMonitor symFactTime( this->timers_.symFactTime_ );
#endif

#if (SUPERLU_DIST_MAJOR_VERSION > 7)
info = SLUD::symbfact_dist(&(data_.options), (data_.grid.nprow) * (data_.grid.npcol),
data_.domains, &(data_.A), data_.perm_c.getRawPtr(),
data_.perm_r.getRawPtr(), data_.sizes,
data_.fstVtxSep, &(data_.pslu_freeable),
&(data_.grid.comm), &(data_.symb_comm),
&(data_.mem_usage));

#else
info = SLUD::symbfact_dist((data_.grid.nprow) * (data_.grid.npcol),
data_.domains, &(data_.A), data_.perm_c.getRawPtr(),
data_.perm_r.getRawPtr(), data_.sizes,
data_.fstVtxSep, &(data_.pslu_freeable),
&(data_.grid.comm), &(data_.symb_comm),
&(data_.mem_usage));
#endif
}
TEUCHOS_TEST_FOR_EXCEPTION( info > 0.0,
std::runtime_error,
Expand Down Expand Up @@ -446,17 +457,35 @@ namespace Amesos2 {
// matter that the glu_freeable member has never been
// initialized, because it is never accessed. It is a
// placeholder arg. The real work is done in data_.lu
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
data_.options.Fact = SLUD::SamePattern_SameRowPerm;
function_map::pdistribute(&(data_.options),
as<SLUD::int_t>(this->globalNumRows_), // aka "n"
&(data_.A), &(data_.scale_perm),
&(data_.glu_freeable), &(data_.lu),
&(data_.grid));
#else
function_map::pdistribute(SLUD::SamePattern_SameRowPerm,
as<SLUD::int_t>(this->globalNumRows_), // aka "n"
&(data_.A), &(data_.scale_perm),
&(data_.glu_freeable), &(data_.lu),
&(data_.grid));
#endif
} else {
#if (SUPERLU_DIST_MAJOR_VERSION > 7)
data_.options.Fact = SLUD::DOFACT;
function_map::dist_psymbtonum(&(data_.options),
as<SLUD::int_t>(this->globalNumRows_), // aka "n"
&(data_.A), &(data_.scale_perm),
&(data_.pslu_freeable), &(data_.lu),
&(data_.grid));
#else
function_map::dist_psymbtonum(SLUD::DOFACT,
as<SLUD::int_t>(this->globalNumRows_), // aka "n"
&(data_.A), &(data_.scale_perm),
&(data_.pslu_freeable), &(data_.lu),
&(data_.grid));
#endif
}

// Retrieve the normI of A (required by gstrf).
Expand Down Expand Up @@ -582,11 +611,20 @@ namespace Amesos2 {
#ifdef HAVE_AMESOS2_TIMERS
Teuchos::TimeMonitor solveTimer(this->timers_.solveTime_);
#endif

#if (SUPERLU_DIST_MAJOR_VERSION > 7)
function_map::gstrs(&(data_.options), as<SLUD::int_t>(this->globalNumRows_), &(data_.lu),
&(data_.scale_perm), &(data_.grid), bvals_.getRawPtr(),
as<SLUD::int_t>(local_len_rhs), as<SLUD::int_t>(first_global_row_b),
as<SLUD::int_t>(local_len_rhs), as<int>(nrhs),
&(data_.solve_struct), &(data_.stat), &ierr);
#else
function_map::gstrs(as<SLUD::int_t>(this->globalNumRows_), &(data_.lu),
&(data_.scale_perm), &(data_.grid), bvals_.getRawPtr(),
as<SLUD::int_t>(local_len_rhs), as<SLUD::int_t>(first_global_row_b),
as<SLUD::int_t>(local_len_rhs), as<int>(nrhs),
&(data_.solve_struct), &(data_.stat), &ierr);
#endif
} // end block for solve time

TEUCHOS_TEST_FOR_EXCEPTION( ierr < 0,
Expand Down

0 comments on commit 403f2cd

Please sign in to comment.