Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fall back to CPU for RoundCeil and RoundFloor expressions #5798

Merged
merged 2 commits into from
Jun 9, 2022
Merged

Fall back to CPU for RoundCeil and RoundFloor expressions #5798

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8c178e6
Fall back to CPU for RoundCeil and RoundFloor expressions
nartal1 Jun 9, 2022
82038c0
use fallback assert api
nartal1 Jun 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions integration_tests/src/main/python/arithmetic_ops_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,17 +404,18 @@ def test_hypot(data_gen):
'hypot(a, b)',
))

@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn)
@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale + [DecimalGen(30, 15)], ids=idfn)
def test_floor(data_gen):
assert_gpu_and_cpu_are_equal_collect(
lambda spark : unary_op_df(spark, data_gen).selectExpr('floor(a)'))

# This test should be enabled to run on GPU as part of https://github.com/NVIDIA/spark-rapids/issues/5797
@pytest.mark.skipif(is_before_spark_330(), reason='scale parameter in Floor function is not supported before Spark 3.3.0')
@allow_non_gpu('ProjectExec')
@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn)
def test_floor_scale_zero(data_gen):
assert_gpu_and_cpu_are_equal_collect(
lambda spark : unary_op_df(spark, data_gen).selectExpr('floor(a, 0)'),
conf={'spark.rapids.sql.castFloatToDecimal.enabled':'true'})
assert_gpu_fallback_collect(
lambda spark : unary_op_df(spark, data_gen).selectExpr('floor(a, 0)'), 'RoundFloor')

@pytest.mark.skipif(is_before_spark_330(), reason='scale parameter in Floor function is not supported before Spark 3.3.0')
@allow_non_gpu('ProjectExec')
Expand All @@ -423,17 +424,18 @@ def test_floor_scale_nonzero(data_gen):
assert_gpu_fallback_collect(
lambda spark : unary_op_df(spark, data_gen).selectExpr('floor(a, -1)'), 'RoundFloor')

@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn)
@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale + [DecimalGen(30, 15)], ids=idfn)
def test_ceil(data_gen):
assert_gpu_and_cpu_are_equal_collect(
lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a)'))

# This test should be enabled to run on GPU as part of https://github.com/NVIDIA/spark-rapids/issues/5797
@pytest.mark.skipif(is_before_spark_330(), reason='scale parameter in Ceil function is not supported before Spark 3.3.0')
@allow_non_gpu('ProjectExec')
@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn)
def test_ceil_scale_zero(data_gen):
assert_gpu_and_cpu_are_equal_collect(
lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a, 0)'),
conf={'spark.rapids.sql.castFloatToDecimal.enabled':'true'})
assert_gpu_fallback_collect(
lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a, 0)'), 'RoundCeil')

@pytest.mark.parametrize('data_gen', [_decimal_gen_36_neg5, _decimal_gen_38_neg10], ids=idfn)
def test_floor_ceil_overflow(data_gen):
Expand Down
31 changes: 0 additions & 31 deletions ...c/main/311until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala
View file
Open in desktop

This file was deleted.

2 changes: 2 additions & 0 deletions sql-plugin/src/main/330+/scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ trait Spark33XShims extends Spark321PlusShims with Spark320PlusNonDBShims {
("scale", TypeSig.lit(TypeEnum.INT), TypeSig.lit(TypeEnum.INT))),
(ceil, conf, p, r) => new BinaryExprMeta[RoundCeil](ceil, conf, p, r) {
override def tagExprForGpu(): Unit = {
willNotWorkOnGpu("RoundCeil is currently not supported on GPU")
ceil.child.dataType match {
case dt: DecimalType =>
val precision = GpuFloorCeil.unboundedOutputPrecision(dt)
Expand Down Expand Up @@ -118,6 +119,7 @@ trait Spark33XShims extends Spark321PlusShims with Spark320PlusNonDBShims {
("scale", TypeSig.lit(TypeEnum.INT), TypeSig.lit(TypeEnum.INT))),
(floor, conf, p, r) => new BinaryExprMeta[RoundFloor](floor, conf, p, r) {
override def tagExprForGpu(): Unit = {
willNotWorkOnGpu("RoundFloor is currently not supported on GPU")
floor.child.dataType match {
case dt: DecimalType =>
val precision = GpuFloorCeil.unboundedOutputPrecision(dt)
Expand Down
35 changes: 0 additions & 35 deletions sql-plugin/src/main/330+/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala
View file
Open in desktop

This file was deleted.

13 changes: 10 additions & 3 deletions sql-plugin/src/main/scala/org/apache/spark/sql/rapids/mathExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import ai.rapids.cudf.ast.BinaryOperator
import com.nvidia.spark.rapids._

import org.apache.spark.sql.catalyst.expressions.{Expression, ImplicitCastInputTypes}
import org.apache.spark.sql.rapids.shims.RapidsFloorCeilUtils
import org.apache.spark.sql.types._

abstract class CudfUnaryMathExpression(name: String) extends GpuUnaryMathExpression(name)
Expand Down Expand Up @@ -166,7 +165,11 @@ object GpuFloorCeil {
}

case class GpuCeil(child: Expression) extends CudfUnaryMathExpression("CEIL") {
override def dataType: DataType = RapidsFloorCeilUtils.outputDataType(child.dataType)
override def dataType: DataType = child.dataType match {
case dt: DecimalType =>
DecimalType.bounded(GpuFloorCeil.unboundedOutputPrecision(dt), 0)
case _ => LongType
}

override def hasSideEffects: Boolean = true

Expand Down Expand Up @@ -238,7 +241,11 @@ case class GpuExpm1(child: Expression) extends CudfUnaryMathExpression("EXPM1")
}

case class GpuFloor(child: Expression) extends CudfUnaryMathExpression("FLOOR") {
override def dataType: DataType = RapidsFloorCeilUtils.outputDataType(child.dataType)
override def dataType: DataType = child.dataType match {
case dt: DecimalType =>
DecimalType.bounded(GpuFloorCeil.unboundedOutputPrecision(dt), 0)
case _ => LongType
}

override def hasSideEffects: Boolean = true

Expand Down