Skip to content

Commit

Permalink
Fix NullPointerException on null partition insert
Browse files Browse the repository at this point in the history
Closes NVIDIA#1735

Signed-off-by: Gera Shegalov <gera@apache.org>
  • Loading branch information
gerashegalov committed Feb 17, 2021
1 parent 39e20a0 commit fa0ec99
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 8 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, NVIDIA CORPORATION.
* Copyright (c) 2019-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -25,7 +25,7 @@ import org.apache.spark.internal.io.FileCommitProtocol
import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession}
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTablePartition}
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName
import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.getPartitionPathString
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
Expand Down Expand Up @@ -204,12 +204,7 @@ case class GpuInsertIntoHadoopFsRelationCommand(
committer: FileCommitProtocol): Unit = {
val staticPartitionPrefix = if (staticPartitions.nonEmpty) {
"/" + partitionColumns.flatMap { p =>
staticPartitions.get(p.name) match {
case Some(value) =>
Some(escapePathName(p.name) + "=" + escapePathName(value))
case None =>
None
}
staticPartitions.get(p.name).map(getPartitionPathString(p.name, _))
}.mkString("/")
} else {
""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright (c) 2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.nvidia.spark.rapids

import org.scalatest.BeforeAndAfterEach

class InsertPartitionSuite extends SparkQueryCompareTestSuite with BeforeAndAfterEach {
var tableNr = 0

override def afterEach(): Unit = {
List(1, 2).foreach { tnr =>
SparkSessionHolder.sparkSession.sql(s"DROP TABLE t$tnr")
}
}

testSparkResultsAreEqual(
testName ="Insert null-value partition ",
spark => {
tableNr += 1
spark.sql(s"""CREATE TABLE t${tableNr}(i STRING, c STRING)
|USING PARQUET PARTITIONED BY (c)""".stripMargin)
spark.sql(s"""INSERT OVERWRITE t${tableNr} PARTITION (c=null)
|VALUES ('1')""".stripMargin)})(
_.sparkSession.sql(s"SELECT * FROM t$tableNr"))
}

0 comments on commit fa0ec99

Please sign in to comment.