Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix order of operations when using mkString in typeConversionInfo #3113

Merged
merged 4 commits into from
Aug 9, 2021
Merged

Fix order of operations when using mkString in typeConversionInfo #3113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
77c6d1a
Fix order of operations when using mkString in typeConversionInfo
abellina Aug 2, 2021
d0df884
Apply code review comments
abellina Aug 3, 2021
7d11db2
Remove unnecessary parenthesis
abellina Aug 3, 2021
ab0f4fb
Merge branch 'branch-21.10' of github.com:NVIDIA/spark-rapids into mi…
abellina Aug 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 16 additions & 14 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,10 +189,9 @@ abstract class RapidsMeta[INPUT <: BASE, BASE, OUTPUT <: BASE](
/**
* Call this method to record information about type conversions via DataTypeMeta.
*/
final def addConvertedDataType(name: String, typeMeta: DataTypeMeta): Unit = {
val reason = s"Converted DataType of $name from ${typeMeta.wrapped.get} to " +
s"${typeMeta.dataType.get}, because ${typeMeta.reasonForConversion}"
typeConversionReasons.get.add(reason)
final def addConvertedDataType(expression: Expression, typeMeta: DataTypeMeta): Unit = {
typeConversionReasons.get.add(
s"$expression: ${typeMeta.reasonForConversion}")
}

/**
Expand Down Expand Up @@ -311,23 +310,23 @@ abstract class RapidsMeta[INPUT <: BASE, BASE, OUTPUT <: BASE](
(cannotRunOnGpuBecauseOfSparkPlan || shouldThisBeRemoved) => "could " + replaceMessage
case Some(v) if v.isEmpty => "will " + replaceMessage
case Some(v) =>
noReplacementPossibleMessage(v mkString "; ")
noReplacementPossibleMessage(v.mkString("; "))
}

private def willBeRemovedInfo: String = shouldBeRemovedReasons match {
case None => ""
case Some(v) if v.isEmpty => ""
case Some(v) =>
val reasons = v mkString "; "
val reasons = v.mkString("; ")
s" but is going to be removed because $reasons"
}

private def typeConversionInfo: String = typeConversionReasons match {
case None => ""
case Some(v) if v.isEmpty => ""
case Some(v) =>
" The data type of following expressions will be converted in GPU runtime:\n" +
v mkString "; "
"The data type of following expressions will be converted in GPU runtime: " +
Copy link
Collaborator

@razajafri razajafri Aug 5, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: string interpolator can be used here?

v.mkString("; ")
}

/**
Expand Down Expand Up @@ -385,13 +384,14 @@ abstract class RapidsMeta[INPUT <: BASE, BASE, OUTPUT <: BASE](
}

strBuilder.append(willWorkOnGpuInfo).
append(willBeRemovedInfo).
append("\n")
append(willBeRemovedInfo)

typeConversionInfo match {
case info if info.isEmpty =>
case info => strBuilder.append(info).append("\n")
case info => strBuilder.append(". ").append(info)
}

strBuilder.append("\n")
}
printChildren(strBuilder, depth, all)
}
Expand Down Expand Up @@ -685,7 +685,7 @@ abstract class SparkPlanMeta[INPUT <: SparkPlan](plan: INPUT,
"The length of outputTypeMetas doesn't match to the length of plan's output")
wrapped.output.zip(typeMetas).map {
case (ar, meta) if meta.typeConverted =>
addConvertedDataType(ar.name, meta)
addConvertedDataType(ar, meta)
AttributeReference(ar.name, meta.dataType.get, ar.nullable, ar.metadata)(
ar.exprId, ar.qualifier)
case (ar, _) =>
Expand Down Expand Up @@ -828,8 +828,10 @@ class DataTypeMeta(
* Returns the reason for conversion if exists
*/
def reasonForConversion: String = {
val reasonMsg = (if (typeConverted) reason else None).getOrElse("")
s"Converted ${wrapped.get} to ${dataType.get}, because $reasonMsg"
val reasonMsg = (if (typeConverted) reason else None)
.map(r => s", because $r").getOrElse("")
s"Converted ${wrapped.getOrElse("N/A")} to " +
s"${dataType.getOrElse("N/A")}" + reasonMsg
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: same here we can use $reasonMsg inside the string

}
}

Expand Down
4 changes: 2 additions & 2 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ final class TypeSig private(
meta.willNotWorkOnGpu(s"$name only supports $dt if it is a literal value")
}
if (typeMeta.typeConverted) {
meta.addConvertedDataType(expr.getClass.getSimpleName, typeMeta)
meta.addConvertedDataType(expr, typeMeta)
}
}
}
Expand Down Expand Up @@ -569,7 +569,7 @@ case class ContextChecks(
s"produces an unsupported type $dt")
}
if (meta.typeMeta.typeConverted) {
meta.addConvertedDataType(expr.prettyName, meta.typeMeta)
meta.addConvertedDataType(expr, meta.typeMeta)
}
case None =>
if (!meta.ignoreUnsetDataTypes) {
Expand Down