Fix NullPointerException on null partition insert

integration_tests/pom.xml

@@ -29,31 +29,31 @@
     <version>0.4.0-SNAPSHOT</version>
 
     <properties>
-        <spark.test.version>3.0.0</spark.test.version>
+        <spark.test.version>${spark300.version}</spark.test.version>
     </properties>
     <profiles>
         <profile>
             <id>spark301dbtests</id>
             <properties>
-                <spark.test.version>3.0.1-databricks</spark.test.version>
+                <spark.test.version>${spark301db.version}</spark.test.version>
             </properties>
         </profile>
         <profile>
             <id>spark301tests</id>
             <properties>
-                <spark.test.version>3.0.1</spark.test.version>
+                <spark.test.version>${spark301.version}</spark.test.version>
             </properties>
         </profile>
         <profile>
             <id>spark302tests</id>
             <properties>
-                <spark.test.version>3.0.2-SNAPSHOT</spark.test.version>
+                <spark.test.version>${spark302.version}</spark.test.version>
             </properties>
         </profile>
         <profile>
             <id>spark311tests</id>
             <properties>
-                <spark.test.version>3.1.1-SNAPSHOT</spark.test.version>
+                <spark.test.version>${spark311.version}</spark.test.version>
             </properties>
         </profile>
     </profiles>

pom.xml

@@ -139,13 +139,23 @@
             <modules>
                 <module>tests-spark310+</module>
             </modules>
+            <repositories>
+                <!--
+                TODO remove once Apache Spark 3.1.1 is released.
+                Last 3.1.1-SNAPSHOT is before SPARK-34223
+                -->
+                <repository>
+                    <id>spark311rc</id>
+                    <url>https://repository.apache.org/content/repositories/orgapachespark-1365/</url>
+                </repository>
+            </repositories>
         </profile>
     </profiles>
 
     <properties>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
-        <spark.version>3.0.0</spark.version>
+        <spark.version>${spark300.version}</spark.version>
         <cuda.version>cuda10-1</cuda.version>
         <cudf.version>0.18-SNAPSHOT</cudf.version>
         <scala.binary.version>2.12</scala.binary.version>
@@ -174,8 +184,9 @@
          If you update a dependendy version so it is no longer a SNAPSHOT
          please update the snapshot-shims profile as well so it is accurate -->
         <spark301.version>3.0.1</spark301.version>
+        <spark301db.version>3.0.1-databricks</spark301db.version>
         <spark302.version>3.0.2-SNAPSHOT</spark302.version>
-        <spark311.version>3.1.1-SNAPSHOT</spark311.version>
+        <spark311.version>3.1.1</spark311.version>
         <mockito.version>3.6.0</mockito.version>
         <scala.plugin.version>4.3.0</scala.plugin.version>
         <maven.jar.plugin.version>3.2.0</maven.jar.plugin.version>

sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@ import org.apache.spark.internal.io.FileCommitProtocol
 import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession}
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTablePartition}
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.getPartitionPathString
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
@@ -204,12 +204,7 @@ case class GpuInsertIntoHadoopFsRelationCommand(
       committer: FileCommitProtocol): Unit = {
     val staticPartitionPrefix = if (staticPartitions.nonEmpty) {
       "/" + partitionColumns.flatMap { p =>
-        staticPartitions.get(p.name) match {
-          case Some(value) =>
-            Some(escapePathName(p.name) + "=" + escapePathName(value))
-          case None =>
-            None
-        }
+        staticPartitions.get(p.name).map(getPartitionPathString(p.name, _))
       }.mkString("/")
     } else {
       ""

tests-spark310+/pom.xml

@@ -29,7 +29,7 @@
     <version>0.4.0-SNAPSHOT</version>
 
     <properties>
-        <spark.test.version>3.1.0-SNAPSHOT</spark.test.version>
+        <spark.test.version>${spark311.version}</spark.test.version>
     </properties>
 
     <dependencies>

tests-spark310+/src/test/scala/com/nvidia/spark/rapids/InsertPartition311Suite.scala

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.nvidia.spark.rapids
+
+import org.scalatest.BeforeAndAfterEach
+
+class InsertPartition311Suite extends SparkQueryCompareTestSuite with BeforeAndAfterEach {
+  var tableNr = 0
+
+  override def afterEach(): Unit = {
+    List(1, 2).foreach { tnr =>
+      SparkSessionHolder.sparkSession.sql(s"DROP TABLE IF EXISTS t$tnr")
+    }
+  }
+
+  testSparkResultsAreEqual(
+    testName ="Insert null-value partition ",
+    spark => {
+      tableNr += 1
+      spark.sql(s"""CREATE TABLE t${tableNr}(i STRING, c STRING)
+                   |USING PARQUET PARTITIONED BY (c)""".stripMargin)
+      spark.sql(s"""INSERT OVERWRITE t${tableNr} PARTITION (c=null)
+                   |VALUES ('1')""".stripMargin)})(
+    _.sparkSession.sql(s"SELECT * FROM t$tableNr"))
+}

tests/pom.xml

@@ -31,31 +31,31 @@
     <version>0.4.0-SNAPSHOT</version>
 
     <properties>
-        <spark.test.version>3.0.0</spark.test.version>
+        <spark.test.version>${spark300.version}</spark.test.version>
     </properties>
     <profiles>
         <profile>
             <id>spark301dbtests</id>
             <properties>
-                <spark.test.version>3.0.1-databricks</spark.test.version>
+                <spark.test.version>${spark301db.version}</spark.test.version>
             </properties>
         </profile>
         <profile>
             <id>spark301tests</id>
             <properties>
-                <spark.test.version>3.0.1</spark.test.version>
+                <spark.test.version>${spark301.version}</spark.test.version>
             </properties>
         </profile>
         <profile>
             <id>spark302tests</id>
             <properties>
-                <spark.test.version>3.0.2-SNAPSHOT</spark.test.version>
+                <spark.test.version>${spark302.version}</spark.test.version>
             </properties>
         </profile>
         <profile>
             <id>spark311tests</id>
             <properties>
-                <spark.test.version>3.1.1-SNAPSHOT</spark.test.version>
+                <spark.test.version>${spark311.version}</spark.test.version>
             </properties>
         </profile>
     </profiles>
@@ -77,7 +77,7 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.binary.version}</artifactId>
-	    <version>${spark.test.version}</version>
+	        <version>${spark.test.version}</version>
         </dependency>
         <dependency>
             <groupId>org.scalatest</groupId>