STK: Snapshot 02-23-23 20:19 from Sierra 5.11.7-495-g36224bc1

packages/stk/cmake/Dependencies.cmake

@@ -11,6 +11,7 @@ SET(SUBPACKAGES_DIRS_CLASSIFICATIONS_OPTREQS
   IO              stk_io               PT  OPTIONAL
   Search          stk_search           PT  OPTIONAL
   Transfer        stk_transfer         PT  OPTIONAL
+  Middle_mesh     stk_middle_mesh      PT  OPTIONAL
   Tools           stk_tools            PT  OPTIONAL
   Balance         stk_balance          PT  OPTIONAL
   Unit_test_utils stk_unit_test_utils  PT  OPTIONAL

packages/stk/stk_balance/stk_balance/internal/privateDeclarations.hpp

@@ -44,7 +44,6 @@
 #include <stk_mesh/base/CreateEdges.hpp>
 #include <stk_mesh/base/CreateFaces.hpp>
 #include <stk_mesh/base/SkinMesh.hpp>
-#include <stk_mesh/base/CoordinateSystems.hpp>
 #include <stk_mesh/baseImpl/elementGraph/BulkDataIdMapper.hpp>
 #include <stk_util/environment/Env.hpp>
 

packages/stk/stk_coupling/stk_coupling/SplitComms.cpp

@@ -30,8 +30,12 @@ namespace coupling
 namespace impl 
 {
 
+SplitCommsImpl::SplitCommsImpl()
+{}
+
 SplitCommsImpl::SplitCommsImpl(MPI_Comm parentComm, int localColor)
   : m_parentComm(parentComm), m_splitComm(MPI_COMM_NULL), m_localColor(localColor),
+    m_finalizationDestructor([this](){free_comms_from_destructor();}),
     m_isInitialized(true)
 {
   stk::util::set_coupling_version(parentComm);
@@ -43,11 +47,7 @@ SplitCommsImpl::SplitCommsImpl(MPI_Comm parentComm, int localColor)
 
 SplitCommsImpl::~SplitCommsImpl()
 {
-  int isMPIFinalized;
-  MPI_Finalized(&isMPIFinalized);
-  if (m_isInitialized && !isMPIFinalized && get_free_comms_in_destructor()) {
-    free_comms_from_destructor();
-  }
+  m_finalizationDestructor.destructor();
 }
 
 
@@ -215,6 +215,9 @@ PairwiseRanks SplitCommsImpl::get_pairwise_root_ranks(int otherColor) const
 
 void SplitCommsImpl::free_comms_from_destructor()
 {
+  if (!get_free_comms_in_destructor())
+    return;
+
   if (!m_isInitialized) {
     std::cerr << "Warning: Cannot free communicators of uninitialized SplitCommsImpl" << std::endl;
     return;
@@ -230,6 +233,14 @@ void SplitCommsImpl::free_comms_from_destructor()
 
 void SplitCommsImpl::free_comms_impl()
 {
+  int isMPIFinalized;
+  MPI_Finalized(&isMPIFinalized);
+  if (isMPIFinalized) {
+    std::cerr << "Attempting to free commuicators after MPI was finalized."
+              << "  The communicators will not be freed, and you have leaked memory" << std::endl;
+    return;
+  }
+
   MPI_Comm_free(&m_splitComm);
 
   for (auto& item : m_pairwiseComms) {

packages/stk/stk_coupling/stk_coupling/SplitComms.hpp

@@ -16,6 +16,7 @@
 #include <utility>
 #include <map>
 #include <memory>
+#include "stk_util/parallel/MPIFinalizationCallback.hpp"
 
 namespace stk
 {
@@ -33,7 +34,8 @@ namespace impl {
 class SplitCommsImpl
 {
   public:
-  SplitCommsImpl() = default;
+  SplitCommsImpl();
+
   SplitCommsImpl(MPI_Comm parentComm, int localColor);
 
   ~SplitCommsImpl();
@@ -82,6 +84,7 @@ STK_DEPRECATED
   std::map<int, MPI_Comm> m_pairwiseComms;
   std::vector<int> m_otherColors;
   std::map<int, PairwiseRanks> m_rootRanks;
+  MPIFinalizationCallback m_finalizationDestructor;
   bool m_isInitialized = false;
   bool m_freeCommsInDestructor = false;
   bool m_haveFreedComms = false;

packages/stk/stk_doc_tests/stk_io/howToCreateAndWriteNodesetOrSideset.cpp

@@ -44,7 +44,6 @@
 #include "stk_unit_test_utils/MeshFixture.hpp"
 #include "stk_io/WriteMesh.hpp"
 
-#include <stk_mesh/base/CoordinateSystems.hpp>
 #include <stk_mesh/base/Field.hpp>
 #include <stk_unit_test_utils/BuildMesh.hpp>
 

packages/stk/stk_doc_tests/stk_io/writeResults.cpp

@@ -46,7 +46,6 @@
 #include <ostream>                      // for basic_ostream::operator<<
 #include <stddef.h>                     // for size_t
 #include <stk_io/StkMeshIoBroker.hpp>   // for StkMeshIoBroker
-#include <stk_mesh/base/CoordinateSystems.hpp>
 #include <stk_mesh/base/Field.hpp>      // for Field
 #include <stk_mesh/base/GetEntities.hpp>  // for get_entities
 #include <stk_mesh/base/MetaData.hpp>   // for MetaData, put_field

packages/stk/stk_doc_tests/stk_mesh/howToGetFields.cpp

@@ -34,7 +34,6 @@
 
 #include <gtest/gtest.h>                // for AssertHelper, EXPECT_EQ, etc
 #include <stddef.h>                     // for size_t
-#include <stk_mesh/base/CoordinateSystems.hpp>  // for Cartesian3d, etc
 #include <stk_mesh/base/Field.hpp>      // for Field
 #include <stk_mesh/base/MetaData.hpp>   // for MetaData, entity_rank_names, etc
 #include "stk_mesh/base/FieldBase.hpp"  // for field_data, etc

packages/stk/stk_doc_tests/stk_mesh/howToIterateConnectivity.cpp

@@ -42,7 +42,6 @@
 #include <string>                       // for string
 #include "stk_io/DatabasePurpose.hpp"   // for DatabasePurpose::READ_MESH
 #include "stk_mesh/base/Bucket.hpp"     // for Bucket
-#include "stk_mesh/base/CoordinateSystems.hpp"  // for Cartesian
 #include "stk_mesh/base/Entity.hpp"     // for Entity
 #include "stk_mesh/base/Field.hpp"      // for Field
 #include "stk_mesh/base/FieldBase.hpp"  // for field_data, FieldBase

packages/stk/stk_doc_tests/stk_mesh/useAdvancedFields.cpp

@@ -35,7 +35,6 @@
 #include <gtest/gtest.h>                // for AssertHelper, EXPECT_EQ, etc
 #include <stk_mesh/base/BulkData.hpp>   // for BulkData
 #include <stk_mesh/base/MeshBuilder.hpp>
-#include <stk_mesh/base/CoordinateSystems.hpp>  // for Cartesian, etc
 #include <stk_mesh/base/FEMHelpers.hpp>  // for declare_element
 #include <stk_mesh/base/Field.hpp>      // for Field
 #include <stk_mesh/base/MetaData.hpp>   // for MetaData, put_field, etc
@@ -66,15 +65,18 @@ TEST(stkMeshHowTo, useAdvancedFields)
   stk::mesh::Part &tetPart = metaData.declare_part_with_topology("tetElementPart", stk::topology::TET_4);
   stk::mesh::Part &hexPart = metaData.declare_part_with_topology("hexElementPart", stk::topology::HEX_8);
 
-  double initialTensorValue[] = {1, 2, 3, 4, 5, 6, 7, 8, 9};
-  stk::mesh::put_field_on_mesh(tensorField, metaData.universal_part(), 9, initialTensorValue);
+  const int numTensorValues = 9;
+  const int numCopies = 2;
+  double initialTensorValue[] = { 1,  2,  3,  4,  5,  6,  7,  8,  9,
+                                 11, 12, 13, 14, 15, 16, 17, 18, 19};
+  stk::mesh::put_field_on_mesh(tensorField, metaData.universal_part(), numTensorValues, numCopies, initialTensorValue);
   stk::io::set_field_output_type(tensorField, stk::io::FieldOutputType::FULL_TENSOR_36);
 
-  double initialVectorValue[] = {1, 2, 3, 4, 5, 6, 7, 8};
-  const unsigned nodesPerTet = 4;
-  stk::mesh::put_field_on_mesh(variableSizeField, tetPart, nodesPerTet, initialVectorValue);
-  const unsigned nodesPerHex = 8;
-  stk::mesh::put_field_on_mesh(variableSizeField, hexPart, nodesPerHex, initialVectorValue);
+  const int numVectorValues = 3;
+  double initialVectorValue[] = {1, 2, 3, 11, 12, 13};
+  stk::mesh::put_field_on_mesh(variableSizeField, tetPart, numVectorValues, initialVectorValue);
+  stk::mesh::put_field_on_mesh(variableSizeField, hexPart, numVectorValues, numCopies, initialVectorValue);
+  stk::io::set_field_output_type(variableSizeField, stk::io::FieldOutputType::VECTOR_3D);
 
   metaData.commit();
   stk::mesh::BulkData& mesh = *bulkPtr;
@@ -87,36 +89,50 @@ TEST(stkMeshHowTo, useAdvancedFields)
   stk::mesh::Entity hexElem=stk::mesh::declare_element(mesh, hexPart, hexId, hexNodes);
   mesh.modification_end();
 
-  const unsigned tensor_scalars_per_hex = stk::mesh::field_scalars_per_entity(tensorField, hexElem);
-  const unsigned tensor_scalars_per_tet = stk::mesh::field_scalars_per_entity(tensorField, tetElem);
+  const int tensorScalarsPerTet = stk::mesh::field_scalars_per_entity(tensorField, tetElem);
+  const int tensorScalarsPerHex = stk::mesh::field_scalars_per_entity(tensorField, hexElem);
+  EXPECT_EQ(tensorScalarsPerTet, numTensorValues*numCopies);
+  EXPECT_EQ(tensorScalarsPerHex, numTensorValues*numCopies);
 
-  EXPECT_EQ(tensor_scalars_per_hex, tensor_scalars_per_tet);
-  const unsigned tensor_enum_size = stk::mesh::FullTensor36::Size;
-  EXPECT_EQ(tensor_scalars_per_hex, tensor_enum_size);
+  const int tensorExtent0PerTet = stk::mesh::field_extent0_per_entity(tensorField, tetElem);
+  const int tensorExtent0PerHex = stk::mesh::field_extent0_per_entity(tensorField, hexElem);
+  EXPECT_EQ(tensorExtent0PerTet, numTensorValues);
+  EXPECT_EQ(tensorExtent0PerHex, numTensorValues);
+
+  const int tensorExtent1PerTet = stk::mesh::field_extent1_per_entity(tensorField, tetElem);
+  const int tensorExtent1PerHex = stk::mesh::field_extent1_per_entity(tensorField, hexElem);
+  EXPECT_EQ(tensorExtent1PerTet, numCopies);
+  EXPECT_EQ(tensorExtent1PerHex, numCopies);
 
   double* tensorData = stk::mesh::field_data(tensorField, hexElem);
-  for(unsigned i=0; i<tensor_scalars_per_hex; i++)
-  {
+  for (int i = 0; i < tensorScalarsPerHex; ++i) {
     EXPECT_EQ(initialTensorValue[i], tensorData[i]);
   }
 
-  const unsigned scalars_per_tet = stk::mesh::field_scalars_per_entity(variableSizeField, tetElem);
-  EXPECT_EQ(nodesPerTet, scalars_per_tet);
+  const int vectorScalarsPerTet = stk::mesh::field_scalars_per_entity(variableSizeField, tetElem);
+  const int vectorScalarsPerHex = stk::mesh::field_scalars_per_entity(variableSizeField, hexElem);
+  EXPECT_EQ(vectorScalarsPerTet, numVectorValues);
+  EXPECT_EQ(vectorScalarsPerHex, numVectorValues*numCopies);
 
-  const unsigned scalars_per_hex = stk::mesh::field_scalars_per_entity(variableSizeField, hexElem);
-  EXPECT_EQ(nodesPerHex, scalars_per_hex);
+  const int vectorExtent0PerTet = stk::mesh::field_extent0_per_entity(variableSizeField, tetElem);
+  const int vectorExtent0PerHex = stk::mesh::field_extent0_per_entity(variableSizeField, hexElem);
+  EXPECT_EQ(vectorExtent0PerTet, numVectorValues);
+  EXPECT_EQ(vectorExtent0PerHex, numVectorValues);
 
-  double* vectorHexData = stk::mesh::field_data(variableSizeField, hexElem);
-  for(unsigned i=0; i<scalars_per_hex; i++)
-  {
-    EXPECT_EQ(initialVectorValue[i], vectorHexData[i]);
-  }
+  const int vectorExtent1PerTet = stk::mesh::field_extent1_per_entity(variableSizeField, tetElem);
+  const int vectorExtent1PerHex = stk::mesh::field_extent1_per_entity(variableSizeField, hexElem);
+  EXPECT_EQ(vectorExtent1PerTet, 1);
+  EXPECT_EQ(vectorExtent1PerHex, numCopies);
 
   double* vectorTetData = stk::mesh::field_data(variableSizeField, tetElem);
-  for(unsigned i=0; i<scalars_per_tet; i++)
-  {
+  for (int i = 0; i < vectorScalarsPerTet; ++i) {
     EXPECT_EQ(initialVectorValue[i], vectorTetData[i]);
   }
+
+  double* vectorHexData = stk::mesh::field_data(variableSizeField, hexElem);
+  for (int i = 0; i < vectorScalarsPerHex; ++i) {
+    EXPECT_EQ(initialVectorValue[i], vectorHexData[i]);
+  }
 }
 //ENDUseAdvancedFields
 

packages/stk/stk_doc_tests/stk_mesh/useSimpleFields.cpp

@@ -37,7 +37,6 @@
 #include <stddef.h>                     // for size_t
 #include <stk_mesh/base/BulkData.hpp>   // for BulkData
 #include <stk_mesh/base/MeshBuilder.hpp>
-#include <stk_mesh/base/CoordinateSystems.hpp>  // for Cartesian3d, etc
 #include <stk_mesh/base/FEMHelpers.hpp>  // for declare_element
 #include <stk_mesh/base/Field.hpp>      // for Field
 #include <stk_mesh/base/MetaData.hpp>   // for MetaData, entity_rank_names, etc

packages/stk/stk_expreval/stk_expreval/DeviceVariableMap.hpp

@@ -57,18 +57,25 @@ class DeviceVariableMap
     NGP_ThrowRequireMsg(parsedEval.get_num_variables() <= MAX_BOUND_VARIABLES,
                         "stk::expreval::DeviceVariableMap is not large enough to hold all variables");
 
-#ifndef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST
-    const auto & deviceNodes = parsedEval.m_deviceNodes;
-#else
-    const auto & deviceNodes = parsedEval.m_hostNodes;
-#endif
+    KOKKOS_IF_ON_DEVICE((
+      const auto & deviceNodes = parsedEval.m_deviceNodes;
+      for (unsigned nodeIndex = 0u; nodeIndex < deviceNodes.extent(0); ++nodeIndex) {
+        if (deviceNodes(nodeIndex).m_opcode == OPCODE_ASSIGN) {
+          m_deviceVariableMap[deviceNodes(nodeIndex).m_data.variable.variableIndex] = DeviceVariable(deviceNodes(nodeIndex).m_data.variable.variableType,
+                                                                                                    deviceNodes(nodeIndex).m_data.variable.variableSize);
+        }
+      }
+    ))
 
-    for (unsigned nodeIndex = 0u; nodeIndex < deviceNodes.extent(0); ++nodeIndex) {
-      if (deviceNodes(nodeIndex).m_opcode == OPCODE_ASSIGN) {
-        m_deviceVariableMap[deviceNodes(nodeIndex).m_data.variable.variableIndex] = DeviceVariable(deviceNodes(nodeIndex).m_data.variable.variableType,
-                                                                                                   deviceNodes(nodeIndex).m_data.variable.variableSize);
+    KOKKOS_IF_ON_HOST((
+      const auto & hostNodes = parsedEval.m_hostNodes;
+      for (unsigned nodeIndex = 0u; nodeIndex < hostNodes.extent(0); ++nodeIndex) {
+        if (hostNodes(nodeIndex).m_opcode == OPCODE_ASSIGN) {
+          m_deviceVariableMap[hostNodes(nodeIndex).m_data.variable.variableIndex] = DeviceVariable(hostNodes(nodeIndex).m_data.variable.variableType,
+                                                                                                   hostNodes(nodeIndex).m_data.variable.variableSize);
+        }
       }
-    }
+    ))
   }
 
   KOKKOS_INLINE_FUNCTION

packages/stk/stk_expreval/stk_expreval/Function.hpp

@@ -156,38 +156,41 @@ double fpart(double x)
 KOKKOS_INLINE_FUNCTION
 double real_rand()
 {
-#if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST)
-  return static_cast<double>(std::rand()) / (static_cast<double>(RAND_MAX) + 1.0);
-#else
-  NGP_ThrowErrorMsg("The rand function is not supported on GPUs");
-  return 0.0;
-#endif
+  KOKKOS_IF_ON_HOST((
+    return static_cast<double>(std::rand()) / (static_cast<double>(RAND_MAX) + 1.0);
+  ))
+  KOKKOS_IF_ON_DEVICE((
+    NGP_ThrowErrorMsg("The rand function is not supported on GPUs");
+    return 0.0;
+  ))
 }
 
 /// Sets x as the random number seed. Interface to the srand function provided by the
 /// ANSI C math library.
 KOKKOS_INLINE_FUNCTION
 double real_srand(double x)
 {
-#if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST)
-  std::srand(static_cast<int>(x));
-  return 0.0;
-#else
-  NGP_ThrowErrorMsg("The srand function is not supported on GPUs");
-  return 0.0;
-#endif
+  KOKKOS_IF_ON_HOST((
+    std::srand(static_cast<int>(x));
+    return 0.0;
+  ))
+  KOKKOS_IF_ON_DEVICE((
+    NGP_ThrowErrorMsg("The srand function is not supported on GPUs");
+    return 0.0;
+  ))
 }
 
 /// Return the current time
 KOKKOS_INLINE_FUNCTION
 double current_time()
 {
-#if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST)
-  return static_cast<double>(::time(nullptr));
-#else
-  NGP_ThrowErrorMsg("The time function is not supported on GPUs");
-  return 0.0;
-#endif
+  KOKKOS_IF_ON_HOST((
+    return static_cast<double>(::time(nullptr));
+  ))
+  KOKKOS_IF_ON_DEVICE((
+    NGP_ThrowErrorMsg("The time function is not supported on GPUs");
+    return 0.0;
+  ))
 }
 
 KOKKOS_INLINE_FUNCTION void hash_combine(std::size_t& seed, double v) {
@@ -201,11 +204,11 @@ extern int sRandomRangeLowValue;
 KOKKOS_INLINE_FUNCTION
 void random_seed(double x)
 {
-#if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST)
-  int y = std::hash<double>{}(x);
-  sRandomRangeHighValue =  y;
-  sRandomRangeLowValue  = ~y;
-#endif
+  KOKKOS_IF_ON_HOST((
+    int y = std::hash<double>{}(x);
+    sRandomRangeHighValue =  y;
+    sRandomRangeLowValue  = ~y;
+  ))
 }
 
 /// Non-platform specific (pseudo) random number generator.
@@ -225,29 +228,31 @@ double seeded_pseudo_random(std::size_t seed, int& low, int& high)
 KOKKOS_INLINE_FUNCTION
 double random0()
 {
-#if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST)
-  sRandomRangeHighValue = (sRandomRangeHighValue<<8) + (sRandomRangeHighValue>>8);
-  sRandomRangeHighValue += sRandomRangeLowValue;
-  sRandomRangeLowValue += sRandomRangeHighValue;
-  int val = std::abs(sRandomRangeHighValue);
-  return double(val) / double(RAND_MAX);
-#else
-  NGP_ThrowErrorMsg("The random function is not supported on GPUs");
-  return 0.0;
-#endif
+  KOKKOS_IF_ON_HOST((
+    sRandomRangeHighValue = (sRandomRangeHighValue<<8) + (sRandomRangeHighValue>>8);
+    sRandomRangeHighValue += sRandomRangeLowValue;
+    sRandomRangeLowValue += sRandomRangeHighValue;
+    int val = std::abs(sRandomRangeHighValue);
+    return double(val) / double(RAND_MAX);
+  ))
+  KOKKOS_IF_ON_DEVICE((
+    NGP_ThrowErrorMsg("The random function is not supported on GPUs");
+    return 0.0;
+  ))
 }
 
 /// Non-platform specific (pseudo) random number generator.
 KOKKOS_INLINE_FUNCTION
 double random1(double seed)
 {
-#if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST)
-  random_seed(seed);
-  return random0();
-#else
-  NGP_ThrowErrorMsg("The random function is not supported on GPUs");
-  return 0.0;
-#endif
+  KOKKOS_IF_ON_HOST((
+    random_seed(seed);
+    return random0();
+  ))
+  KOKKOS_IF_ON_DEVICE((
+    NGP_ThrowErrorMsg("The random function is not supported on GPUs");
+    return 0.0;
+  ))
 }
 
 /// Non-platform specific (pseudo) random number generator that