Merge pull request #6 from marmbrus/joinWork

Minor changes to get more tests passing.
marmbrus · Jan 6, 2014 · 66adceb · 66adceb
2 parents 61b266f + a43d41c
commit 66adceb
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 5 deletions.
diff --git a/src/main/scala/catalyst/execution/MetastoreCatalog.scala b/src/main/scala/catalyst/execution/MetastoreCatalog.scala
@@ -77,12 +77,15 @@ class HiveMetastoreCatalog(hiveConf: HiveConf) extends Catalog {
 }
 
 object HiveMetatoreTypes {
+  val VARCHAR = "(?i)VARCHAR\\((\\d+)\\)".r
   def toDataType(metastoreType: String): DataType =
     metastoreType match {
       case "string" => StringType
       case "int" => IntegerType
       case "double" => DoubleType
       case "bigint" => LongType
+      case "binary" => BinaryType
+      case VARCHAR(_) => StringType
     }
 }
 

diff --git a/src/main/scala/catalyst/execution/hiveOperators.scala b/src/main/scala/catalyst/execution/hiveOperators.scala
@@ -68,6 +68,7 @@ case class HiveTableScan(attributes: Seq[Attribute], relation: MetastoreRelation
       buildRow(values.map {
         case "NULL" => null
         case "null" => null
+        case varchar: org.apache.hadoop.hive.common.`type`.HiveVarchar => varchar.getValue
         case other => other
       })
     }

diff --git a/src/main/scala/catalyst/frontend/Hive.scala b/src/main/scala/catalyst/frontend/Hive.scala
@@ -615,6 +615,7 @@ object HiveQl {
     /* Casts */
     case Token("TOK_FUNCTION", Token("TOK_STRING", Nil) :: arg :: Nil) => Cast(nodeToExpr(arg), StringType)
     case Token("TOK_FUNCTION", Token("TOK_INT", Nil) :: arg :: Nil) => Cast(nodeToExpr(arg), IntegerType)
+    case Token("TOK_FUNCTION", Token("TOK_BIGINT", Nil) :: arg :: Nil) => Cast(nodeToExpr(arg), LongType)
     case Token("TOK_FUNCTION", Token("TOK_FLOAT", Nil) :: arg :: Nil) => Cast(nodeToExpr(arg), FloatType)
     case Token("TOK_FUNCTION", Token("TOK_DOUBLE", Nil) :: arg :: Nil) => Cast(nodeToExpr(arg), DoubleType)
     case Token("TOK_FUNCTION", Token("TOK_SMALLINT", Nil) :: arg :: Nil) => Cast(nodeToExpr(arg), ShortType)
@@ -628,6 +629,7 @@ object HiveQl {
     case Token("*", left :: right:: Nil) => Multiply(nodeToExpr(left), nodeToExpr(right))
     case Token("/", left :: right:: Nil) => Divide(nodeToExpr(left), nodeToExpr(right))
     case Token("DIV", left :: right:: Nil) => Divide(nodeToExpr(left), nodeToExpr(right))
+    case Token("%", left :: right:: Nil) => Remainder(nodeToExpr(left), nodeToExpr(right))
 
     /* Comparisons */
     case Token("=", left :: right:: Nil) => Equals(nodeToExpr(left), nodeToExpr(right))

diff --git a/src/test/scala/catalyst/execution/HiveComparisionTest.scala b/src/test/scala/catalyst/execution/HiveComparisionTest.scala
@@ -58,7 +58,7 @@ abstract class HiveComaparisionTest extends FunSuite with BeforeAndAfterAll with
     str.replaceAll("file:\\/.*\\/", "<PATH>")
   }
 
-  val installHooksCommand = "SET.*hooks".r
+  val installHooksCommand = "(?i)SET.*hooks".r
   def createQueryTest(testCaseName: String, sql: String) = {
     test(testCaseName) {
       logger.error(

diff --git a/src/test/scala/catalyst/execution/HiveCompatability.scala b/src/test/scala/catalyst/execution/HiveCompatability.scala
@@ -188,12 +188,12 @@ class HiveCompatability extends HiveQueryFileTest {
     "literal_string",
     "load_file_with_space_in_the_name",
     "mapjoin_subquery2",
+    "mapreduce3",
     "mergejoins",
     "mergejoins_mixed",
     "misc_json",
     "multi_join_union",
     "multigroupby_singlemr",
-    "no_hooks",
     "noalias_subq1",
     "nomore_ambiguous_table_col",
     "notable_alias1",
@@ -301,6 +301,7 @@ class HiveCompatability extends HiveQueryFileTest {
     "udf_minute",
     "udf_modulo",
     "udf_month",
+    "udf_negative",
     "udf_not",
     "udf_notequal",
     "udf_nvl",
@@ -374,6 +375,9 @@ class HiveCompatability extends HiveQueryFileTest {
     "union7",
     "union8",
     "union9",
-    "union_script"
+    "union_script",
+    "varchar_2",
+    "varchar_join1",
+    "varchar_union1"
   )
 }
diff --git a/src/test/scala/catalyst/execution/HiveQueryFileTest.scala b/src/test/scala/catalyst/execution/HiveQueryFileTest.scala
@@ -25,8 +25,9 @@ abstract class HiveQueryFileTest extends HiveComaparisionTest {
 
   val runAll = !(System.getProperty("shark.hive.alltests") == null)
 
+  val whiteListProperty = "shark.hive.whitelist"
   // Allow the whiteList to be overridden by a system property
-  val realWhiteList = Option(System.getProperty("shark.hive.whitelist")).map(_.split(",").toSeq).getOrElse(whiteList)
+  val realWhiteList = Option(System.getProperty(whiteListProperty)).map(_.split(",").toSeq).getOrElse(whiteList)
 
   // Go through all the test cases and add them to scala test.
   testCases.foreach {
@@ -38,7 +39,10 @@ abstract class HiveQueryFileTest extends HiveComaparisionTest {
         val queriesString = fileToString(testCaseFile)
         createQueryTest(testCaseName, queriesString)
       } else {
-        ignore(testCaseName) {}
+        // Only output warnings for the built in whitelist as this clutters the output when the user
+        // trying to execute a single test from the commandline.
+        if(System.getProperty(whiteListProperty) == null)
+          ignore(testCaseName) {}
       }
   }
 }