Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for \h, \H, \v, \V, and \R character classes #5477

Merged
merged 6 commits into from
May 17, 2022
Merged

Add support for \h, \H, \v, \V, and \R character classes #5477

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3235748
Add support for \h, \H, \v, \V, and \R
anthony-chang May 12, 2022
cd840d3
Add comments with source of character class list
anthony-chang May 12, 2022
93d4c5d
Merge branch 'branch-22.06' of github.com:anthony-chang/spark-rapids …
anthony-chang May 12, 2022
1cf2e13
Merge branch 'branch-22.06' of github.com:anthony-chang/spark-rapids …
anthony-chang May 12, 2022
315f2b2
Consolidate tests
anthony-chang May 12, 2022
a8b67c3
Move new character class expansion to parse stage from rewrite stage,…
anthony-chang May 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions integration_tests/src/main/python/string_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -867,6 +867,40 @@ def test_regexp_whitespace():
),
conf=_regexp_conf)

def test_regexp_horizontal_vertical_whitespace():
gen = mk_str_gen(
'''\xA0\u1680\u180e[abcd]\t\n{1,3} [0-9]\n {1,3}\x0b\t[abcd]\r\f[0-9]{0,10}
[\u2001-\u200a]{1,3}\u202f\u205f\u3000\x85\u2028\u2029
''')
assert_gpu_and_cpu_are_equal_collect(
lambda spark: unary_op_df(spark, gen).selectExpr(
'rlike(a, "\\h{2}")',
'rlike(a, "\\v{3}")',
'rlike(a, "[abcd]+\\h+[0-9]+")',
'rlike(a, "[abcd]+\\v+[0-9]+")',
'rlike(a, "\\H{3}")',
anthony-chang marked this conversation as resolved.
Show resolved Hide resolved
'rlike(a, "\\V{3}")',
'rlike(a, "[abcd]+\\h+\\V{2,3}")',
'regexp_extract(a, "([a-d]+)([0-9\\v]+)([a-d]+)", 2)',
'regexp_extract(a, "([a-d]+)(\\H+)([0-9]+)", 2)',
'regexp_extract(a, "([a-d]+)(\\V+)([0-9]+)", 3)',
'regexp_replace(a, "(\\v+)", "@")',
'regexp_replace(a, "(\\H+)", "#")',
),
conf=_regexp_conf)

def test_regexp_linebreak():
gen = mk_str_gen(
'[abc]{1,3}\u000D\u000A[def]{1,3}[\u000A\u000B\u000C\u000D\u0085\u2028\u2029]{0,5}[123]')
assert_gpu_and_cpu_are_equal_collect(
lambda spark: unary_op_df(spark, gen).selectExpr(
'rlike(a, "\\R")',
'rlike(a, "\\R{2}")',
'regexp_extract(a, "([a-d]+)\\R([a-d]+)", 1)',
'regexp_replace(a, "\\R", "")',
),
conf=_regexp_conf)

def test_rlike():
gen = mk_str_gen('[abcd]{1,3}')
assert_gpu_and_cpu_are_equal_collect(
Expand Down
24 changes: 23 additions & 1 deletion sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ class RegexParser(pattern: String) {
// string anchors
consumeExpected(ch)
RegexEscaped(ch)
case 's' | 'S' | 'd' | 'D' | 'w' | 'W' =>
case 's' | 'S' | 'd' | 'D' | 'w' | 'W' | 'h' | 'H' | 'v' | 'V' | 'R' =>
// meta sequences
consumeExpected(ch)
RegexEscaped(ch)
Expand Down Expand Up @@ -675,6 +675,28 @@ class CudfRegexTranspiler(mode: RegexMode) {
RegexChar(' '), RegexChar('\u000b'))
chars ++= Seq('n', 't', 'r', 'f').map(RegexEscaped)
RegexCharacterClass(negated = ch.isUpper, characters = chars)
case 'h' | 'H' =>
anthony-chang marked this conversation as resolved.
Show resolved Hide resolved
val chars: ListBuffer[RegexCharacterClassComponent] = ListBuffer(
RegexChar(' '), RegexChar('\u00A0'), RegexChar('\u1680'), RegexChar('\u180e'),
RegexChar('\u202f'), RegexChar('\u205f'), RegexChar('\u3000')
)
chars += RegexEscaped('t')
chars += RegexCharacterRange('\u2000', '\u200a')
RegexCharacterClass(negated = ch.isUpper, characters = chars)
case 'v' | 'V' =>
val chars: ListBuffer[RegexCharacterClassComponent] = ListBuffer(
RegexChar('\u000B'), RegexChar('\u0085'), RegexChar('\u2028'), RegexChar('\u2029')
)
chars ++= Seq('n', 'f', 'r').map(RegexEscaped)
RegexCharacterClass(negated = ch.isUpper, characters = chars)
case 'R' =>
// Unicode linebreak sequence
val l = RegexSequence(ListBuffer(RegexChar('\u000D'), RegexChar('\u000A')))
val r = RegexCharacterClass(false, ListBuffer[RegexCharacterClassComponent](
RegexChar('\u000A'), RegexChar('\u000B'), RegexChar('\u000C'), RegexChar('\u000D'),
RegexChar('\u0085'), RegexChar('\u2028'), RegexChar('\u2029')
))
RegexChoice(l, r)
case _ =>
regex
}
Expand Down
18 changes: 18 additions & 0 deletions tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionTranspilerSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,24 @@ class RegularExpressionTranspilerSuite extends FunSuite with Arm {
Seq("\u001eTEST"))
}

test("horizonal whitespace - find") {
anthony-chang marked this conversation as resolved.
Show resolved Hide resolved
val patterns = Seq("\\h", "\\H")
val inputs = Seq("abcd", "te\r\nst", raw"\xA0", "\u2025f \u0042", "\t\u2001", "\u2001- a\u200a")
assertCpuGpuMatchesRegexpFind(patterns, inputs)
}

test("vertical whitespace - find") {
val patterns = Seq("\\v", "\\V")
val inputs = Seq("abcd", "te \tst", "\n", "\r\n", raw"\x85", "\u2028 \u2029")
assertCpuGpuMatchesRegexpFind(patterns, inputs)
}

test("linebreak matcher - find") {
val inputs = Seq("abcd", "te\r\nst", "1\u000D\u000A2", "\u000C\u000D",
"\u000A\u000D", "\u0085\u2029", "\u000C test \u2028")
assertCpuGpuMatchesRegexpFind(Seq("\\R"), inputs)
}

test("match literal $ - find") {
assertCpuGpuMatchesRegexpFind(
Seq("\\$", "\\$[0-9]"),
Expand Down