Skip to content

Commit

Permalink
[SPARK-4676][SQL] JavaSchemaRDD.schema may throw NullType MatchError …
Browse files Browse the repository at this point in the history
…if sql has null

val jsc = new org.apache.spark.api.java.JavaSparkContext(sc)
val jhc = new org.apache.spark.sql.hive.api.java.JavaHiveContext(jsc)
val nrdd = jhc.hql("select null from spark_test.for_test")
println(nrdd.schema)
Then the error is thrown as follows:
scala.MatchError: NullType (of class org.apache.spark.sql.catalyst.types.NullType$)
at org.apache.spark.sql.types.util.DataTypeConversions$.asJavaDataType(DataTypeConversions.scala:43)

Author: YanTangZhai <hakeemzhai@tencent.com>
Author: yantangzhai <tyz0303@163.com>
Author: Michael Armbrust <michael@databricks.com>

Closes apache#3538 from YanTangZhai/MatchNullType and squashes the following commits:

e052dff [yantangzhai] [SPARK-4676] [SQL] JavaSchemaRDD.schema may throw NullType MatchError if sql has null
4b4bb34 [yantangzhai] [SPARK-4676] [SQL] JavaSchemaRDD.schema may throw NullType MatchError if sql has null
896c7b7 [yantangzhai] fix NullType MatchError in JavaSchemaRDD when sql has null
6e643f8 [YanTangZhai] Merge pull request alteryx#11 from apache/master
e249846 [YanTangZhai] Merge pull request alteryx#10 from apache/master
d26d982 [YanTangZhai] Merge pull request alteryx#9 from apache/master
76d4027 [YanTangZhai] Merge pull request alteryx#8 from apache/master
03b62b0 [YanTangZhai] Merge pull request alteryx#7 from apache/master
8a00106 [YanTangZhai] Merge pull request alteryx#6 from apache/master
cbcba66 [YanTangZhai] Merge pull request #3 from apache/master
cdef539 [YanTangZhai] Merge pull request #1 from apache/master
  • Loading branch information
YanTangZhai authored and marmbrus committed Dec 2, 2014
1 parent 69b6fed commit 1066427
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@ public abstract class DataType {
*/
public static final ShortType ShortType = new ShortType();

/**
* Gets the NullType object.
*/
public static final NullType NullType = new NullType();

/**
* Creates an ArrayType by specifying the data type of elements ({@code elementType}).
* The field of {@code containsNull} is set to {@code true}.
Expand Down
27 changes: 27 additions & 0 deletions sql/core/src/main/java/org/apache/spark/sql/api/java/NullType.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.spark.sql.api.java;

/**
* The data type representing null and NULL values.
*
* {@code NullType} is represented by the singleton object {@link DataType#NullType}.
*/
public class NullType extends DataType {
protected NullType() {}
}
10 changes: 10 additions & 0 deletions sql/core/src/main/scala/org/apache/spark/sql/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,16 @@ package object sql {
@DeveloperApi
val ShortType = catalyst.types.ShortType

/**
* :: DeveloperApi ::
*
* The data type representing `NULL` values.
*
* @group dataType
*/
@DeveloperApi
val NullType = catalyst.types.NullType

/**
* :: DeveloperApi ::
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ protected[sql] object DataTypeConversions {
case IntegerType => JDataType.IntegerType
case LongType => JDataType.LongType
case ShortType => JDataType.ShortType
case NullType => JDataType.NullType

case arrayType: ArrayType => JDataType.createArrayType(
asJavaDataType(arrayType.elementType), arrayType.containsNull)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,22 @@ class JavaSQLSuite extends FunSuite {
javaSqlCtx.sql("SELECT * FROM people").collect()
}

test("schema with null from JavaBeans") {
val person = new PersonBean
person.setName("Michael")
person.setAge(29)

val rdd = javaCtx.parallelize(person :: Nil)
val schemaRDD = javaSqlCtx.applySchema(rdd, classOf[PersonBean])

schemaRDD.registerTempTable("people")
val nullRDD = javaSqlCtx.sql("SELECT null FROM people")
val structFields = nullRDD.schema.getFields()
assert(structFields.size == 1)
assert(structFields(0).getDataType().isInstanceOf[NullType])
assert(nullRDD.collect.head.row === Seq(null))
}

test("all types in JavaBeans") {
val bean = new AllTypesBean
bean.setStringField("")
Expand Down

0 comments on commit 1066427

Please sign in to comment.