Use RAJA plugin capability

src/care/Accessor.h

@@ -3,7 +3,7 @@
 #define CHAI_Accessor__HPP
 
 #include "care/config.h"
-#include "care/RAJAPlugin.h"
+#include "care/PluginData.h"
 
 #include <algorithm>
 #include <cstdlib>
@@ -93,17 +93,17 @@ class RaceConditionAccessor : public NoOpAccessor<T> {
 
    CARE_HOST_DEVICE RaceConditionAccessor<T>(RaceConditionAccessor<T> const & other ) : m_shallow_copy_of_cpu_data(other.m_shallow_copy_of_cpu_data), m_deep_copy_of_previous_state_of_cpu_data(other.m_deep_copy_of_previous_state_of_cpu_data), m_accesses(other.m_accesses), m_size_in_bytes(other.m_size_in_bytes), m_name(other.m_name) {
 #ifndef CARE_GPUCC
-      if (RAJAPlugin::isParallelContext()) {
+      if (PluginData::isParallelContext()) {
          auto data = m_shallow_copy_of_cpu_data;
-         if (!RAJAPlugin::post_parallel_forall_action_registered((void *)data)) {
+         if (!PluginData::post_parallel_forall_action_registered((void *)data)) {
             auto len = m_size_in_bytes / sizeof(T);
             m_deep_copy_of_previous_state_of_cpu_data = new typename std::remove_const<T>::type[len];
             std::copy_n(data, len, m_deep_copy_of_previous_state_of_cpu_data);
             auto prev_data = m_deep_copy_of_previous_state_of_cpu_data;
             m_accesses = new std::unordered_map<int, std::set<int>> {};
             auto accesses = m_accesses;
             const char * name = m_name;
-            RAJAPlugin::register_post_parallel_forall_action((void *)data, std::bind(detectRaceCondition<T>, data, prev_data, accesses, len, name, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));
+            PluginData::register_post_parallel_forall_action((void *)data, std::bind(detectRaceCondition<T>, data, prev_data, accesses, len, name, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));
          }
       }
 #endif
@@ -115,8 +115,8 @@ class RaceConditionAccessor : public NoOpAccessor<T> {
    const
    {
 #ifndef CARE_GPUCC
-      if (m_accesses && RAJAPlugin::isParallelContext()) {
-         (*m_accesses)[i].insert(RAJAPlugin::s_threadID);
+      if (m_accesses && PluginData::isParallelContext()) {
+         (*m_accesses)[i].insert(PluginData::s_threadID);
       }
 #endif
    }
@@ -153,12 +153,12 @@ class RaceConditionAccessorWithCallback : public RaceConditionAccessor<T> {
    const
    {
       m_callback(i);
-      if (RaceConditionAccessor<T>::m_accesses && RAJAPlugin::isParallelContext()) {
-         printf("inserting %i into (%p) accesses[%i] for pointer %p\n", RAJAPlugin::s_threadID, (void *) RaceConditionAccessor<T>::m_accesses, i, RaceConditionAccessor<T>::m_shallow_copy_of_cpu_data);
+      if (RaceConditionAccessor<T>::m_accesses && PluginData::isParallelContext()) {
+         printf("inserting %i into (%p) accesses[%i] for pointer %p\n", PluginData::s_threadID, (void *) RaceConditionAccessor<T>::m_accesses, i, RaceConditionAccessor<T>::m_shallow_copy_of_cpu_data);
          printf("size of accesses before insert %lu\n", RaceConditionAccessor<T>::m_accesses->size());
       }
       RaceConditionAccessor<T>::operator[](i);
-      if (RaceConditionAccessor<T>::m_accesses && RAJAPlugin::isParallelContext()) {
+      if (RaceConditionAccessor<T>::m_accesses && PluginData::isParallelContext()) {
          printf("size of accesses after insert %lu\n", RaceConditionAccessor<T>::m_accesses->size());
          printf("size of accesses[%i] after insert %lu\n", i, (*RaceConditionAccessor<T>::m_accesses)[i].size());
       }

src/care/CHAICallback.cpp

@@ -8,7 +8,7 @@
 // CARE headers
 #include "care/CHAICallback.h"
 #ifndef CARE_DISABLE_RAJAPLUGIN
-#include "care/RAJAPlugin.h"
+#include "care/PluginData.h"
 #endif
 
 // Other library headers
@@ -301,7 +301,7 @@ namespace care {
    void CHAICallback::deregisterRecord(const chai::PointerRecord* record) {
 #ifndef CARE_DISABLE_RAJAPLUGIN
       if (s_log_data > 0) {
-         RAJAPlugin::removeActivePointer(record);
+         PluginData::removeActivePointer(record);
       }
 #endif
       NameMap& s_names = getNameMap();
@@ -433,16 +433,16 @@ namespace care {
                         fprintf(s_log_file,
                                 "[CARE] [CHAI] %s: Moved %lu bytes to space %i at %s:%i\n",
                                 name.c_str(), size, (int) space,
-                                RAJAPlugin::getCurrentLoopFileName().c_str(),
-                                RAJAPlugin::getCurrentLoopLineNumber());
+                                PluginData::getFileName(),
+                                PluginData::getLineNumber());
                      }
                      else {
                         fprintf(s_log_file,
                                 "[CARE] [CHAI] %s (%p): Moved %lu bytes to space %i (%p) at %s:%i\n",
                                 name.c_str(), m_record, size, (int) space,
                                 m_record->m_pointers[space],
-                                RAJAPlugin::getCurrentLoopFileName().c_str(),
-                                RAJAPlugin::getCurrentLoopLineNumber());
+                                PluginData::getFileName(),
+                                PluginData::getLineNumber());
                      }
 #endif
                   }
@@ -456,20 +456,20 @@ namespace care {
                         fprintf(s_log_file,
                                 "[CARE] [CHAI] %s: Captured %lu bytes to space %i at %s:%i\n",
                                 name.c_str(), size, (int) space,
-                                RAJAPlugin::getCurrentLoopFileName().c_str(),
-                                RAJAPlugin::getCurrentLoopLineNumber());
+                                PluginData::getFileName(),
+                                PluginData::getLineNumber());
                      }
                      else {
                         fprintf(s_log_file,
                                 "[CARE] [CHAI] %s (%p): Captured %lu bytes to space %i (%p) at %s:%i\n",
                                 name.c_str(), m_record, size, (int) space,
                                 m_record->m_pointers[space],
-                                RAJAPlugin::getCurrentLoopFileName().c_str(),
-                                RAJAPlugin::getCurrentLoopLineNumber());
+                                PluginData::getFileName(),
+                                PluginData::getLineNumber());
                      }
                   }
                   if (s_log_data > 0) {
-                     RAJAPlugin::addActivePointer(m_record);
+                     PluginData::addActivePointer(m_record);
                   }
 #endif
                   break;
@@ -522,7 +522,7 @@ namespace care {
                case chai::ACTION_CAPTURED:
 #ifndef CARE_DISABLE_RAJAPLUGIN
                   if (s_log_data > 0) {
-                     RAJAPlugin::addActivePointer(m_record);
+                     PluginData::addActivePointer(m_record);
                   }
 #endif
                   break;

src/care/CMakeLists.txt

@@ -51,8 +51,10 @@ set(care_headers
     numeric.h
     PointerTypes.h
     policies.h
-    care.h
-    RAJAPlugin.h
+    care.h 
+    PluginData.h
+    DebugPlugin.h
+    ProfilePlugin.h
     scan.h
     scan_impl.h
     Setup.h
@@ -65,7 +67,9 @@ set(care_sources
     CHAICallback.cpp
     ExecutionSpace.cpp
     LoopFuser.cpp
-    RAJAPlugin.cpp
+    PluginData.cpp
+    DebugPlugin.cpp
+    ProfilePlugin.cpp
     scan.cpp
     )
 

src/care/DebugPlugin.cpp

@@ -0,0 +1,139 @@
+#include "care/DebugPlugin.h"
+#include "chai/ExecutionSpaces.hpp"
+#include "care/CHAICallback.h"
+#include "care/PluginData.h"
+
+// Std library headers
+#if defined(CARE_DEBUG) && !defined(_WIN32)
+#include <execinfo.h>
+#endif // defined(CARE_DEBUG) && !defined(_WIN32)
+
+#include <unordered_set>
+
+namespace care{
+   void DebugPlugin::registerPlugin() {
+      static RAJA::util::PluginRegistry::add<care::DebugPlugin> L ("Debug plugin", "CARE plugin for debugging");
+   }
+
+   void DebugPlugin::preLaunch(const RAJA::util::PluginContext& p) {
+#if !defined(CHAI_DISABLE_RM)
+      // Prepare to record CHAI data
+      if (CHAICallback::isActive()) {
+         PluginData::clearActivePointers();
+
+#if defined(CARE_GPUCC) && defined(CARE_DEBUG)
+         GPUWatchpoint::setOrCheckWatchpoint<int>();
+#endif // defined(CARE_GPUCC) && defined(CARE_DEBUG)
+      }
+#endif // !defined(CHAI_DISABLE_RM)
+   }
+
+
+   void DebugPlugin::postLaunch(const RAJA::util::PluginContext& p) {
+#if !defined(CHAI_DISABLE_RM)
+	   chai::ExecutionSpace space;
+
+	   switch (p.platform) {
+         case RAJA::Platform::host:
+         	space = chai::CPU; break;
+#if defined(CHAI_ENABLE_CUDA)
+    	   case RAJA::Platform::cuda:
+      	   space = chai::GPU; break;
+#endif
+#if defined(CHAI_ENABLE_HIP)
+         case RAJA::Platform::hip:
+            space = chai::GPU; break;
+#endif
+         default:
+            space = chai::NONE;
+      }
+
+      if (CHAICallback::isActive()) {			
+         writeLoopData(space, PluginData::getFileName(), PluginData::getLineNumber());
+
+#if defined(CARE_GPUCC) && defined(CARE_DEBUG)
+         GPUWatchpoint::setOrCheckWatchpoint<int>();
+#endif // defined(CARE_GPUCC) && defined(CARE_DEBUG)
+      }
+
+      if (PluginData::isParallelContext()) {
+         for (auto const & it : PluginData::get_post_parallel_forall_actions()) {
+             it.second(space, PluginData::getFileName(), PluginData::getLineNumber());
+          }
+          PluginData::clear_post_parallel_forall_actions();
+          PluginData::s_threadID = -1;
+      }
+#endif // !defined(CHAI_DISABLE_RM)
+   }
+
+   void DebugPlugin::writeLoopData(chai::ExecutionSpace space, const char * fileName, int lineNumber) {
+      if (CHAICallback::loggingIsEnabled()) {
+         const int s_log_data = CHAICallback::getLogData();
+
+         if (s_log_data == (int) space ||
+             s_log_data == (int) chai::ExecutionSpace::NUM_EXECUTION_SPACES) {
+            // Get the output file
+            FILE* callback_output_file = CHAICallback::getLogFile();
+
+            // Write the loop header
+            int numArrays = 0;
+            std::unordered_set<const chai::PointerRecord*> usedRecords;
+
+            for (const chai::PointerRecord* record : PluginData::getActivePointers()) {
+               if (usedRecords.count(record) > 0) {
+                  continue;
+               }
+               else {
+                  usedRecords.emplace(record);
+                  ++numArrays;
+               }
+            }
+
+            fprintf(callback_output_file, "[CARE] AFTER LOOP EXECUTION %s:%i (%i arrays)\n", fileName, lineNumber, numArrays);
+
+#if defined(CARE_DEBUG) && !defined(_WIN32)
+            // Write the stack trace
+            const int stackDepth = 16;
+            void *stackArray[stackDepth];
+            size_t stackSize = backtrace(stackArray, stackDepth);
+            char **stackStrings = backtrace_symbols(stackArray, stackSize);
+
+            // Skip the first two contexts
+            for (size_t i = 2 ; i < stackSize ; ++i) {
+               fprintf(callback_output_file, "[CARE] [STACK] %lu: %s\n", i, stackStrings[i]);
+            }
+
+            if (CHAICallback::isDiffFriendly()) {
+               // For diff-friendly output, keep the number of lines constant
+               for (size_t i = stackSize ; i < stackDepth ; ++i) {
+                  fprintf(callback_output_file, "[CARE] [STACK] %lu: <empty>\n", i);
+               }
+            }
+
+            free(stackStrings);
+#endif // defined(CARE_DEBUG) && !defined(_WIN32)
+
+            // Flush to the output file
+            fflush(callback_output_file);
+
+#ifdef CARE_DEBUG
+            // Write the arrays captured in the loop
+            usedRecords.clear();
+
+            for (const chai::PointerRecord* record : PluginData::getActivePointers()) {
+               if (record && usedRecords.find(record) == usedRecords.end()) {
+                  usedRecords.emplace(record);
+                  CHAICallback::writeArray(record, space);
+                  fflush(callback_output_file);
+               }
+            }
+#else // CARE_DEBUG
+            // Write message that -logchaidata is not supported
+            fprintf(callback_output_file, "[CARE] [CHAI] -logchaidata is not supported in this build\n");
+            fflush(callback_output_file);
+#endif // CARE_DEBUG
+         }
+      }
+   }
+}
+

src/care/DebugPlugin.h

@@ -0,0 +1,33 @@
+#ifndef _CARE_DebugPlugin_H_
+#define _CARE_DebugPlugin_H_
+
+#include "RAJA/util/PluginStrategy.hpp"
+#include "chai/ExecutionSpaces.hpp"
+
+namespace care{
+
+   class DebugPlugin : public RAJA::util::PluginStrategy
+   {
+      public:
+         DebugPlugin() = default;
+
+         static void registerPlugin();
+
+         void preLaunch(const RAJA::util::PluginContext& p) override;
+
+         void postLaunch(const RAJA::util::PluginContext& p) override;
+
+         /////////////////////////////////////////////////////////////////////////////////
+         ///
+         /// @brief Writes out debugging information after a loop is executed.
+         ///
+         /// @arg[in] space The execution space
+         /// @arg[in] fileName The file where the loop macro was called
+         /// @arg[in] lineNumber The line number where the loop macro was called
+         ///
+         /////////////////////////////////////////////////////////////////////////////////
+         static void writeLoopData(chai::ExecutionSpace space, const char * fileName, int lineNumber);
+   };
+}
+
+#endif

src/care/DefaultMacros.h

@@ -534,7 +534,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #ifdef CARE_ENABLE_RACE_DETECTION
-#define CARE_SET_THREAD_ID(INDEX) care::RAJAPlugin::s_threadID = INDEX ;
+#define CARE_SET_THREAD_ID(INDEX) care::DebugPlugin::s_threadID = INDEX ;
 #else
 #define CARE_SET_THREAD_ID(INDEX)
 #endif

src/care/PluginData.cpp

@@ -0,0 +1,59 @@
+#include "PluginData.h"
+
+namespace care{
+   const char * PluginData::s_file_name = "N/A";
+   int PluginData::s_line_number = -1;
+   bool PluginData::s_parallel_context = false;
+   std::unordered_map<void *, std::function<void(chai::ExecutionSpace, const char *, int)>> PluginData::s_post_parallel_forall_actions 
+      = std::unordered_map<void *, std::function<void(chai::ExecutionSpace, const char *, int)>>{};     
+   std::vector<const chai::PointerRecord*> PluginData::s_active_pointers_in_loop = std::vector<const chai::PointerRecord*>{};
+   int PluginData::s_threadID = -1;
+
+   void PluginData::setFileName(const char * name) {PluginData::s_file_name = name;}
+
+   void PluginData::setLineNumber(int num) {PluginData::s_line_number = num;}
+
+   const char * PluginData::getFileName() {return s_file_name;}
+
+   int PluginData::getLineNumber() {return s_line_number;}
+
+   void PluginData::setParallelContext(bool isParallel) {
+      s_parallel_context = isParallel;
+   }
+
+   bool PluginData::isParallelContext(){
+      return s_parallel_context;
+   }
+
+   std::unordered_map<void *, std::function<void(chai::ExecutionSpace, const char *, int)>> PluginData::get_post_parallel_forall_actions() {
+      return s_post_parallel_forall_actions;
+   }
+
+   void PluginData::register_post_parallel_forall_action(void * key, std::function<void(chai::ExecutionSpace, const char *, int)> action) { 
+      s_post_parallel_forall_actions[key] = action;
+   }
+
+   bool PluginData::post_parallel_forall_action_registered(void * key) {
+      bool registered = s_post_parallel_forall_actions.count(key) > 0;
+      return registered;
+   }
+
+   void PluginData::clear_post_parallel_forall_actions(){s_post_parallel_forall_actions.clear();}
+
+   std::vector<const chai::PointerRecord*> PluginData::getActivePointers() {return s_active_pointers_in_loop;}
+
+   void PluginData::addActivePointer(const chai::PointerRecord* record) {
+      s_active_pointers_in_loop.emplace_back(record);
+   }
+
+   void PluginData::removeActivePointer(const chai::PointerRecord* record) {
+      for (size_t i = 0; i < s_active_pointers_in_loop.size(); ++i) {
+         if (s_active_pointers_in_loop[i] == record) {
+            s_active_pointers_in_loop[i] = nullptr;
+         }
+      }
+   }
+
+   void PluginData::clearActivePointers() {s_active_pointers_in_loop.clear();}
+
+}