Use RAJA plugin capability

src/care/DebugPlugin.cpp

@@ -0,0 +1,169 @@
+#include "care/DebugPlugin.h"
+
+#include "chai/ArrayManager.hpp"
+
+#include "chai/ExecutionSpaces.hpp"
+
+#include "care/CHAICallback.h"
+
+#include "care/RAJAPlugin.h"
+
+/* CUDA profiling macros */
+#if defined(__CUDACC__) && CARE_HAVE_NVTOOLSEXT
+#include "nvToolsExt.h"
+#endif
+
+// Std library headers
+#if defined(CARE_DEBUG) && !defined(_WIN32)
+#include <execinfo.h>
+#endif // defined(CARE_DEBUG) && !defined(_WIN32)
+
+#include <unordered_set>
+
+using namespace care;
+
+namespace chai{
+
+	std::string s_current_loop_file_name = "N/A";
+   int s_current_loop_line_number = -1;
+   std::vector<const chai::PointerRecord*> s_active_pointers_in_loop = std::vector<const chai::PointerRecord*>{};
+
+DebugPlugin::DebugPlugin() {}
+
+   /////////////////////////////////////////////////////////////////////////////////
+   ///
+   /// @brief Set up to be done before executing a RAJA loop.
+   ///
+   /// @arg[in] space The execution space
+   /// @arg[in] name The name of the loop
+   ///
+   /////////////////////////////////////////////////////////////////////////////////
+
+void 
+DebugPlugin::preLaunch(const RAJA::util::PluginContext& p) {
+#if !defined(CHAI_DISABLE_RM)
+      // Prepare to record CHAI data
+      if (CHAICallback::isActive()) {
+         s_current_loop_file_name = fileName;
+         s_current_loop_line_number = lineNumber;
+         s_active_pointers_in_loop.clear();
+
+#if defined(CARE_GPUCC) && defined(CARE_DEBUG)
+         GPUWatchpoint::setOrCheckWatchpoint<int>();
+#endif // defined(CARE_GPUCC) && defined(CARE_DEBUG)
+      }
+#endif // !defined(CHAI_DISABLE_RM)
+
+   }
+
+
+
+void 
+DebugPlugin::postLaunch(const RAJA::util::PluginContext& p) {
+#if !defined(CHAI_DISABLE_RM)
+
+		chai::ExecutionSpace space;
+
+		switch (p.platform) {
+    		case RAJA::Platform::host:
+      		space = chai::CPU; break;
+#if defined(CHAI_ENABLE_CUDA)
+    		case RAJA::Platform::cuda:
+      		space = chai::GPU; break;
+#endif
+#if defined(CHAI_ENABLE_HIP)
+    		case RAJA::Platform::hip:
+      		space = chai::GPU; break;
+#endif
+    		default:
+      		space = chai::NONE;
+  			}
+
+		if (CHAICallback::isActive()) {			
+			writeLoopData(space, fileName, lineNumber);
+
+         // Clear out the captured arrays
+         s_active_pointers_in_loop.clear();
+
+#if defined(CARE_GPUCC) && defined(CARE_DEBUG)
+         GPUWatchpoint::setOrCheckWatchpoint<int>();
+#endif // defined(CARE_GPUCC) && defined(CARE_DEBUG)
+      }
+
+#endif // !defined(CHAI_DISABLE_RM)
+   }
+
+void 
+DebugPlugin::writeLoopData(chai::ExecutionSpace space, const char * fileName, int lineNumber) {
+      if (CHAICallback::loggingIsEnabled()) {
+         const int s_log_data = CHAICallback::getLogData();
+
+         if (s_log_data == (int) space ||
+             s_log_data == (int) chai::ExecutionSpace::NUM_EXECUTION_SPACES) {
+            // Get the output file
+            FILE* callback_output_file = CHAICallback::getLogFile();
+
+            // Write the loop header
+            int numArrays = 0;
+            std::unordered_set<const chai::PointerRecord*> usedRecords;
+
+            for (const chai::PointerRecord* record : s_active_pointers_in_loop) {
+               if (usedRecords.count(record) > 0) {
+                  continue;
+               }
+               else {
+                  usedRecords.emplace(record);
+                  ++numArrays;
+               }
+            }
+
+            fprintf(callback_output_file, "[CARE] AFTER LOOP EXECUTION %s:%i (%i arrays)\n", fileName, lineNumber, numArrays);
+
+#if defined(CARE_DEBUG) && !defined(_WIN32)
+            // Write the stack trace
+            const int stackDepth = 16;
+            void *stackArray[stackDepth];
+            size_t stackSize = backtrace(stackArray, stackDepth);
+            char **stackStrings = backtrace_symbols(stackArray, stackSize);
+
+            // Skip the first two contexts
+            for (size_t i = 2 ; i < stackSize ; ++i) {
+               fprintf(callback_output_file, "[CARE] [STACK] %lu: %s\n", i, stackStrings[i]);
+            }
+
+            if (CHAICallback::isDiffFriendly()) {
+               // For diff-friendly output, keep the number of lines constant
+               for (size_t i = stackSize ; i < stackDepth ; ++i) {
+                  fprintf(callback_output_file, "[CARE] [STACK] %lu: <empty>\n", i);
+               }
+            }
+
+            free(stackStrings);
+#endif // defined(CARE_DEBUG) && !defined(_WIN32)
+
+            // Flush to the output file
+            fflush(callback_output_file);
+
+#ifdef CARE_DEBUG
+            // Write the arrays captured in the loop
+            usedRecords.clear();
+
+            for (const chai::PointerRecord* record : s_active_pointers_in_loop) {
+               if (record && usedRecords.find(record) == usedRecords.end()) {
+                  usedRecords.emplace(record);
+                  CHAICallback::writeArray(record, space);
+                  fflush(callback_output_file);
+               }
+            }
+#else // CARE_DEBUG
+            // Write message that -logchaidata is not supported
+            fprintf(callback_output_file, "[CARE] [CHAI] -logchaidata is not supported in this build\n");
+            fflush(callback_output_file);
+#endif // CARE_DEBUG
+         }
+      }
+   }
+}
+
+static RAJA::util::PluginRegistry::add<chai::DebugPlugin> L ("Debug plugin", "test");
+

src/care/DebugPlugin.h

@@ -0,0 +1,45 @@
+#ifndef _CARE_DebugPlugin_H_
+#define _CARE_DebugPlugin_H_
+
+#include "RAJA/util/PluginStrategy.hpp"
+#include "chai/ExecutionSpaces.hpp"
+#include "care/config.h"
+#include "care/CHAICallback.h"
+#include "care/RAJAPlugin.h"
+
+using namespace care;
+
+namespace care{
+
+class RAJAPlugin;
+
+}
+
+namespace chai{
+
+	class DebugPlugin : public RAJA::util::PluginStrategy
+	{
+		public:
+			DebugPlugin();
+
+			char * fileName;
+
+			int lineNumber;
+
+			void preLaunch(const RAJA::util::PluginContext& p) override;
+
+			void postLaunch(const RAJA::util::PluginContext& p) override;
+
+			void writeLoopData(chai::ExecutionSpace space, const char * fileName, int lineNumber);
+
+		private:
+
+			std::string s_current_loop_file_name;
+
+         int s_current_loop_line_number;
+
+			std::vector<const chai::PointerRecord*> s_active_pointers_in_loop;
+	};
+}
+
+#endif

src/care/ProfilePlugin.cpp

@@ -0,0 +1,86 @@
+#include "care/ProfilePlugin.h"
+
+#include "chai/ArrayManager.hpp"
+
+#include "chai/ExecutionSpaces.hpp"
+
+#include "care/CHAICallback.h"
+
+#include "care/RAJAPlugin.h"
+
+/* CUDA profiling macros */
+#if defined(__CUDACC__) && CARE_HAVE_NVTOOLSEXT
+#include "nvToolsExt.h"
+#endif
+
+// Std library headers
+#if defined(CARE_DEBUG) && !defined(_WIN32)
+#include <execinfo.h>
+#endif // defined(CARE_DEBUG) && !defined(_WIN32)
+
+#include <unordered_set>
+
+using namespace care;
+
+namespace chai{
+
+	bool s_profile_host_loops = true;
+	static uint32_t s_colors[7];
+   static int s_num_colors;
+   static unsigned int s_current_color;
+
+ProfilePlugin::ProfilePlugin() {}
+
+   /////////////////////////////////////////////////////////////////////////////////
+   ///
+   /// @brief Set up to be done before executing a RAJA loop.
+   ///
+   /// @arg[in] space The execution space
+   /// @arg[in] name The name of the loop
+   ///
+   /////////////////////////////////////////////////////////////////////////////////
+
+void 
+ProfilePlugin::preLaunch(const RAJA::util::PluginContext& p) {
+#if CARE_HAVE_NVTOOLSEXT
+#if defined(CARE_GPUCC)
+      // Profile the host loops
+      if (s_profile_host_loops) {
+         if (p.platform == RAJA::Platform::host) {
+            std::string name = fileName + std::to_string(lineNumber);
+
+            int color_id = s_current_color + 1;
+            color_id = color_id % s_num_colors;
+
+            // TODO: Add error checking
+            nvtxEventAttributes_t eventAttrib = { 0 };
+            eventAttrib.version = NVTX_VERSION;
+            eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
+            eventAttrib.colorType = NVTX_COLOR_ARGB;
+            eventAttrib.color = s_colors[color_id];
+            eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
+            eventAttrib.message.ascii = name.c_str();
+            nvtxRangePushEx(&eventAttrib);
+         }
+      }
+#endif // CARE_HAVE_NVTOOLSEXT
+#endif // defined(CARE_GPUCC)
+   }
+
+void 
+ProfilePlugin::postLaunch(const RAJA::util::PluginContext& p) {
+#if defined(CARE_GPUCC)
+      if (s_profile_host_loops) {
+         if (p.platform == RAJA::Platform::host) {
+            // TODO: Add error checking
+            nvtxRangePop();
+         }
+      }
+#endif
+
+}
+}
+
+static RAJA::util::PluginRegistry::add<chai::ProfilePlugin> L ("Care profile plugin", "test");
+
+

src/care/ProfilePlugin.h

@@ -0,0 +1,46 @@
+#ifndef _CARE_ProfilePlugin_H_
+#define _CARE_ProfilePlugin_H_
+
+#include "RAJA/util/PluginStrategy.hpp"
+#include "chai/ExecutionSpaces.hpp"
+#include "care/config.h"
+#include "care/CHAICallback.h"
+#include "care/RAJAPlugin.h"
+
+using namespace care;
+
+namespace care{
+
+class RAJAPlugin;
+
+}
+
+namespace chai{
+
+	class ProfilePlugin : public RAJA::util::PluginStrategy
+	{
+		public:
+			ProfilePlugin();
+
+			char * fileName;
+
+			int lineNumber;
+
+			void preLaunch(const RAJA::util::PluginContext& p) override;
+
+			void postLaunch(const RAJA::util::PluginContext& p) override;
+
+		private:
+
+			bool s_profile_host_loops;
+
+			unsigned int s_current_color;
+
+			int s_num_colors;
+
+			uint32_t s_colors[7];
+
+	};
+}
+
+#endif

src/care/RAJAPlugin.cpp

@@ -25,6 +25,8 @@
 
 #include <unordered_set>
 
+using namespace std;
+
 namespace care {
    bool RAJAPlugin::s_update_chai_execution_space = true;
    bool RAJAPlugin::s_debug_chai_data = true;
@@ -43,6 +45,7 @@ namespace care {
    std::vector<const chai::PointerRecord*> RAJAPlugin::s_active_pointers_in_loop = std::vector<const chai::PointerRecord*>{};
    std::unordered_map<void *, std::function<void(chai::ExecutionSpace, const char *, int)>> RAJAPlugin::s_post_parallel_forall_actions = std::unordered_map<void *, std::function<void(chai::ExecutionSpace, const char *, int)>>{};
    int RAJAPlugin::s_threadID = -1;
+
    /////////////////////////////////////////////////////////////////////////////////
    ///
    /// @brief Set up to be done before executing a RAJA loop.