From 178c14ee0faf03debd39fa9b4bc011fed9ac1c18 Mon Sep 17 00:00:00 2001 From: Alessandro Bellina Date: Thu, 3 Dec 2020 00:53:23 -0600 Subject: [PATCH] At BatchedTableCompressor.finish synchronize to allow for "right-size" copies to finish before it is potentially spilled (#1248) Signed-off-by: Alessandro Bellina --- .../com/nvidia/spark/rapids/TableCompressionCodec.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TableCompressionCodec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TableCompressionCodec.scala index 1785d31fe3c..56a74366f84 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TableCompressionCodec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TableCompressionCodec.scala @@ -193,6 +193,12 @@ abstract class BatchedTableCompressor(maxBatchMemorySize: Long, stream: Cuda.Str val compressedTables = results.toArray results.clear() + + // Ensure we synchronize on the CUDA stream, because `CompressedTable` instances + // could be copied to host during a spill before we are done. + // TODO: A better way to do this would be via CUDA events, synchronizing on the event + // instead of the whole stream + stream.sync() compressedTables }