Add config to dump heap on GPU OOM (#891)

Signed-off-by: Jason Lowe <jlowe@nvidia.com>

docs/configs.md

@@ -32,6 +32,7 @@ Name | Description | Default Value
 <a name="memory.gpu.allocFraction"></a>spark.rapids.memory.gpu.allocFraction|The fraction of total GPU memory that should be initially allocated for pooled memory. Extra memory will be allocated as needed, but it may result in more fragmentation. This must be less than or equal to the maximum limit configured via spark.rapids.memory.gpu.maxAllocFraction.|0.9
 <a name="memory.gpu.debug"></a>spark.rapids.memory.gpu.debug|Provides a log of GPU memory allocations and frees. If set to STDOUT or STDERR the logging will go there. Setting it to NONE disables logging. All other values are reserved for possible future expansion and in the mean time will disable logging.|NONE
 <a name="memory.gpu.maxAllocFraction"></a>spark.rapids.memory.gpu.maxAllocFraction|The fraction of total GPU memory that limits the maximum size of the RMM pool. The value must be greater than or equal to the setting for spark.rapids.memory.gpu.allocFraction. Note that this limit will be reduced by the reserve memory configured in spark.rapids.memory.gpu.reserve.|1.0
+<a name="memory.gpu.oomDumpDir"></a>spark.rapids.memory.gpu.oomDumpDir|The path to a local directory where a heap dump will be created if the GPU encounters an unrecoverable out-of-memory (OOM) error. The filename will be of the form: "gpu-oom-<pid>.hprof" where <pid> is the process ID.|None
 <a name="memory.gpu.pool"></a>spark.rapids.memory.gpu.pool|Select the RMM pooling allocator to use. Valid values are "DEFAULT", "ARENA", and "NONE". With "DEFAULT", `rmm::mr::pool_memory_resource` is used; with "ARENA", `rmm::mr::arena_memory_resource` is used. If set to "NONE", pooling is disabled and RMM just passes through to CUDA memory allocation directly.|ARENA
 <a name="memory.gpu.pooling.enabled"></a>spark.rapids.memory.gpu.pooling.enabled|Should RMM act as a pooling allocator for GPU memory, or should it just pass through to CUDA memory allocation directly. DEPRECATED: please use spark.rapids.memory.gpu.pool instead.|true
 <a name="memory.gpu.reserve"></a>spark.rapids.memory.gpu.reserve|The amount of GPU memory that should remain unallocated by RMM and left for system use such as memory needed for kernels, kernel launches or JIT compilation.|1073741824

sql-plugin/src/main/scala/com/nvidia/spark/rapids/DeviceMemoryEventHandler.scala

@@ -16,18 +16,23 @@
 
 package com.nvidia.spark.rapids
 
+import java.io.File
+import java.lang.management.ManagementFactory
+
 import ai.rapids.cudf.{NvtxColor, NvtxRange, RmmEventHandler}
+import com.sun.management.HotSpotDiagnosticMXBean
 
-import org.apache.spark.TaskContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.rapids.execution.TrampolineUtil
 
 /**
  * RMM event handler to trigger spilling from the device memory store.
  * @param store device memory store that will be triggered to spill
+ * @param oomDumpDir local directory to create heap dumps on GPU OOM
  */
-class DeviceMemoryEventHandler(store: RapidsDeviceMemoryStore)
-    extends RmmEventHandler with Logging {
+class DeviceMemoryEventHandler(
+    store: RapidsDeviceMemoryStore,
+    oomDumpDir: Option[String]) extends RmmEventHandler with Logging {
 
   /**
    * Handles RMM allocation failures by spilling buffers from device memory.
@@ -44,6 +49,7 @@ class DeviceMemoryEventHandler(store: RapidsDeviceMemoryStore)
         if (storeSize == 0) {
           logWarning("Device store exhausted, unable to satisfy "
               + s"allocation of $allocSize bytes")
+          oomDumpDir.foreach(heapDump)
           return false
         }
         val targetSize = Math.max(storeSize - allocSize, 0)
@@ -71,4 +77,19 @@ class DeviceMemoryEventHandler(store: RapidsDeviceMemoryStore)
 
   override def onDeallocThreshold(totalAllocated: Long): Unit = {
   }
+
+  private def heapDump(dumpDir: String): Unit = {
+    val dumpPath = getDumpPath(dumpDir)
+    logWarning(s"Dumping heap to $dumpPath")
+    val server = ManagementFactory.getPlatformMBeanServer
+    val mxBean = ManagementFactory.newPlatformMXBeanProxy(server,
+      "com.sun.management:type=HotSpotDiagnostic", classOf[HotSpotDiagnosticMXBean])
+    mxBean.dumpHeap(dumpPath, false)
+  }
+
+  private def getDumpPath(dumpDir: String): String = {
+    // pid is typically before the '@' character in the name
+    val pid = ManagementFactory.getRuntimeMXBean.getName.split('@').head
+    new File(dumpDir, s"gpu-oom-$pid.hprof").toString
+  }
 }

sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsBufferCatalog.scala

@@ -137,7 +137,7 @@ object RapidsBufferCatalog extends Logging with Arm {
     hostStorage.setSpillStore(diskStorage)
 
     logInfo("Installing GPU memory handler for spill")
-    memoryEventHandler = new DeviceMemoryEventHandler(deviceStorage)
+    memoryEventHandler = new DeviceMemoryEventHandler(deviceStorage, rapidsConf.gpuOomDumpDir)
     Rmm.setEventHandler(memoryEventHandler)
   }
 

sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala

@@ -282,6 +282,13 @@ object RapidsConf {
     .stringConf
     .createWithDefault("NONE")
 
+  val GPU_OOM_DUMP_DIR = conf("spark.rapids.memory.gpu.oomDumpDir")
+    .doc("The path to a local directory where a heap dump will be created if the GPU " +
+      "encounters an unrecoverable out-of-memory (OOM) error. The filename will be of the " +
+      "form: \"gpu-oom-<pid>.hprof\" where <pid> is the process ID.")
+    .stringConf
+    .createOptional
+
   private val RMM_ALLOC_MAX_FRACTION_KEY = "spark.rapids.memory.gpu.maxAllocFraction"
   private val RMM_ALLOC_RESERVE_KEY = "spark.rapids.memory.gpu.reserve"
 
@@ -872,6 +879,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging {
 
   lazy val rmmDebugLocation: String = get(RMM_DEBUG)
 
+  lazy val gpuOomDumpDir: Option[String] = get(GPU_OOM_DUMP_DIR)
+
   lazy val isUvmEnabled: Boolean = get(UVM_ENABLED)
 
   lazy val isPooledMemEnabled: Boolean = get(POOLED_MEM)