diff --git a/docs/configs.md b/docs/configs.md
index 63a39347e65..ee8de4d498f 100644
--- a/docs/configs.md
+++ b/docs/configs.md
@@ -32,6 +32,7 @@ Name | Description | Default Value
 <a name="memory.gpu.allocFraction"></a>spark.rapids.memory.gpu.allocFraction|The fraction of total GPU memory that should be initially allocated for pooled memory. Extra memory will be allocated as needed, but it may result in more fragmentation. This must be less than or equal to the maximum limit configured via spark.rapids.memory.gpu.maxAllocFraction.|0.9
 <a name="memory.gpu.debug"></a>spark.rapids.memory.gpu.debug|Provides a log of GPU memory allocations and frees. If set to STDOUT or STDERR the logging will go there. Setting it to NONE disables logging. All other values are reserved for possible future expansion and in the mean time will disable logging.|NONE
 <a name="memory.gpu.maxAllocFraction"></a>spark.rapids.memory.gpu.maxAllocFraction|The fraction of total GPU memory that limits the maximum size of the RMM pool. The value must be greater than or equal to the setting for spark.rapids.memory.gpu.allocFraction. Note that this limit will be reduced by the reserve memory configured in spark.rapids.memory.gpu.reserve.|1.0
+<a name="memory.gpu.oomDumpDir"></a>spark.rapids.memory.gpu.oomDumpDir|The path to a local directory where a heap dump will be created if the GPU encounters an unrecoverable out-of-memory (OOM) error. The filename will be of the form: "gpu-oom-<pid>.hprof" where <pid> is the process ID.|None
 <a name="memory.gpu.pool"></a>spark.rapids.memory.gpu.pool|Select the RMM pooling allocator to use. Valid values are "DEFAULT", "ARENA", and "NONE". With "DEFAULT", `rmm::mr::pool_memory_resource` is used; with "ARENA", `rmm::mr::arena_memory_resource` is used. If set to "NONE", pooling is disabled and RMM just passes through to CUDA memory allocation directly.|ARENA
 <a name="memory.gpu.pooling.enabled"></a>spark.rapids.memory.gpu.pooling.enabled|Should RMM act as a pooling allocator for GPU memory, or should it just pass through to CUDA memory allocation directly. DEPRECATED: please use spark.rapids.memory.gpu.pool instead.|true
 <a name="memory.gpu.reserve"></a>spark.rapids.memory.gpu.reserve|The amount of GPU memory that should remain unallocated by RMM and left for system use such as memory needed for kernels, kernel launches or JIT compilation.|1073741824
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DeviceMemoryEventHandler.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DeviceMemoryEventHandler.scala
index f50daa60ac2..7467a2fd5fb 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DeviceMemoryEventHandler.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DeviceMemoryEventHandler.scala
@@ -16,18 +16,23 @@
 
 package com.nvidia.spark.rapids
 
+import java.io.File
+import java.lang.management.ManagementFactory
+
 import ai.rapids.cudf.{NvtxColor, NvtxRange, RmmEventHandler}
+import com.sun.management.HotSpotDiagnosticMXBean
 
-import org.apache.spark.TaskContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.rapids.execution.TrampolineUtil
 
 /**
  * RMM event handler to trigger spilling from the device memory store.
  * @param store device memory store that will be triggered to spill
+ * @param oomDumpDir local directory to create heap dumps on GPU OOM
  */
-class DeviceMemoryEventHandler(store: RapidsDeviceMemoryStore)
-    extends RmmEventHandler with Logging {
+class DeviceMemoryEventHandler(
+    store: RapidsDeviceMemoryStore,
+    oomDumpDir: Option[String]) extends RmmEventHandler with Logging {
 
   /**
    * Handles RMM allocation failures by spilling buffers from device memory.
@@ -44,6 +49,7 @@ class DeviceMemoryEventHandler(store: RapidsDeviceMemoryStore)
         if (storeSize == 0) {
           logWarning("Device store exhausted, unable to satisfy "
               + s"allocation of $allocSize bytes")
+          oomDumpDir.foreach(heapDump)
           return false
         }
         val targetSize = Math.max(storeSize - allocSize, 0)
@@ -71,4 +77,19 @@ class DeviceMemoryEventHandler(store: RapidsDeviceMemoryStore)
 
   override def onDeallocThreshold(totalAllocated: Long): Unit = {
   }
+
+  private def heapDump(dumpDir: String): Unit = {
+    val dumpPath = getDumpPath(dumpDir)
+    logWarning(s"Dumping heap to $dumpPath")
+    val server = ManagementFactory.getPlatformMBeanServer
+    val mxBean = ManagementFactory.newPlatformMXBeanProxy(server,
+      "com.sun.management:type=HotSpotDiagnostic", classOf[HotSpotDiagnosticMXBean])
+    mxBean.dumpHeap(dumpPath, false)
+  }
+
+  private def getDumpPath(dumpDir: String): String = {
+    // pid is typically before the '@' character in the name
+    val pid = ManagementFactory.getRuntimeMXBean.getName.split('@').head
+    new File(dumpDir, s"gpu-oom-$pid.hprof").toString
+  }
 }
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsBufferCatalog.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsBufferCatalog.scala
index 12b7aca9baa..675884d1037 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsBufferCatalog.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsBufferCatalog.scala
@@ -137,7 +137,7 @@ object RapidsBufferCatalog extends Logging with Arm {
     hostStorage.setSpillStore(diskStorage)
 
     logInfo("Installing GPU memory handler for spill")
-    memoryEventHandler = new DeviceMemoryEventHandler(deviceStorage)
+    memoryEventHandler = new DeviceMemoryEventHandler(deviceStorage, rapidsConf.gpuOomDumpDir)
     Rmm.setEventHandler(memoryEventHandler)
   }
 
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
index 5f1ae8848d1..29a2c3a8d3b 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
@@ -282,6 +282,13 @@ object RapidsConf {
     .stringConf
     .createWithDefault("NONE")
 
+  val GPU_OOM_DUMP_DIR = conf("spark.rapids.memory.gpu.oomDumpDir")
+    .doc("The path to a local directory where a heap dump will be created if the GPU " +
+      "encounters an unrecoverable out-of-memory (OOM) error. The filename will be of the " +
+      "form: \"gpu-oom-<pid>.hprof\" where <pid> is the process ID.")
+    .stringConf
+    .createOptional
+
   private val RMM_ALLOC_MAX_FRACTION_KEY = "spark.rapids.memory.gpu.maxAllocFraction"
   private val RMM_ALLOC_RESERVE_KEY = "spark.rapids.memory.gpu.reserve"
 
@@ -872,6 +879,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging {
 
   lazy val rmmDebugLocation: String = get(RMM_DEBUG)
 
+  lazy val gpuOomDumpDir: Option[String] = get(GPU_OOM_DUMP_DIR)
+
   lazy val isUvmEnabled: Boolean = get(UVM_ENABLED)
 
   lazy val isPooledMemEnabled: Boolean = get(POOLED_MEM)