Skip to content

Commit

Permalink
[SPARK-37219][SQL][FOLLOWUP] Make AS OF syntax more flexible
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?
Make `AS OF` syntax more flexible

### Why are the changes needed?
`((FOR SYSTEM_VERSION) | VERSION)` -> `FOR? (SYSTEM_VERSION | VERSION) AS OF`

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
Existing tests

Closes #34621 from huaxingao/asof_followup.

Authored-by: Huaxin Gao <huaxin_gao@apple.com>
Signed-off-by: Huaxin Gao <huaxin_gao@apple.com>
  • Loading branch information
huaxingao committed Nov 18, 2021
1 parent 9d19c27 commit f361ad8
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -600,8 +600,8 @@ fromClause
;

temporalClause
: ((FOR SYSTEM_VERSION) | VERSION) AS OF version=(INTEGER_VALUE | STRING)
| ((FOR SYSTEM_TIME) | TIMESTAMP) AS OF timestamp=STRING
: FOR? (SYSTEM_VERSION | VERSION) AS OF version=(INTEGER_VALUE | STRING)
| FOR? (SYSTEM_TIME | TIMESTAMP) AS OF timestamp=STRING
;

aggregationClause
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2440,6 +2440,14 @@ class DDLParserSuite extends AnalysisTest {
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

comparePlans(
parsePlan("SELECT * FROM a.b.c FOR VERSION AS OF 'Snapshot123456789'"),
Project(Seq(UnresolvedStar(None)),
UnresolvedRelation(
Seq("a", "b", "c"),
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

timeTravel = TimeTravelSpec.create(None, Some("123456789"))
comparePlans(
parsePlan("SELECT * FROM a.b.c FOR SYSTEM_VERSION AS OF 123456789"),
Expand All @@ -2449,6 +2457,14 @@ class DDLParserSuite extends AnalysisTest {
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

comparePlans(
parsePlan("SELECT * FROM a.b.c SYSTEM_VERSION AS OF 123456789"),
Project(Seq(UnresolvedStar(None)),
UnresolvedRelation(
Seq("a", "b", "c"),
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

timeTravel = TimeTravelSpec.create(Some("2019-01-29 00:37:58"), None)
comparePlans(
parsePlan("SELECT * FROM a.b.c TIMESTAMP AS OF '2019-01-29 00:37:58'"),
Expand All @@ -2457,6 +2473,15 @@ class DDLParserSuite extends AnalysisTest {
Seq("a", "b", "c"),
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

comparePlans(
parsePlan("SELECT * FROM a.b.c FOR TIMESTAMP AS OF '2019-01-29 00:37:58'"),
Project(Seq(UnresolvedStar(None)),
UnresolvedRelation(
Seq("a", "b", "c"),
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

comparePlans(
parsePlan("SELECT * FROM a.b.c FOR SYSTEM_TIME AS OF '2019-01-29 00:37:58'"),
Project(Seq(UnresolvedStar(None)),
Expand All @@ -2465,6 +2490,14 @@ class DDLParserSuite extends AnalysisTest {
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

comparePlans(
parsePlan("SELECT * FROM a.b.c SYSTEM_TIME AS OF '2019-01-29 00:37:58'"),
Project(Seq(UnresolvedStar(None)),
UnresolvedRelation(
Seq("a", "b", "c"),
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

timeTravel = TimeTravelSpec.create(Some("2019-01-29"), None)
comparePlans(
parsePlan("SELECT * FROM a.b.c TIMESTAMP AS OF '2019-01-29'"),
Expand All @@ -2473,6 +2506,15 @@ class DDLParserSuite extends AnalysisTest {
Seq("a", "b", "c"),
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

comparePlans(
parsePlan("SELECT * FROM a.b.c FOR TIMESTAMP AS OF '2019-01-29'"),
Project(Seq(UnresolvedStar(None)),
UnresolvedRelation(
Seq("a", "b", "c"),
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

comparePlans(
parsePlan("SELECT * FROM a.b.c FOR SYSTEM_TIME AS OF '2019-01-29'"),
Project(Seq(UnresolvedStar(None)),
Expand All @@ -2481,6 +2523,14 @@ class DDLParserSuite extends AnalysisTest {
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

comparePlans(
parsePlan("SELECT * FROM a.b.c SYSTEM_TIME AS OF '2019-01-29'"),
Project(Seq(UnresolvedStar(None)),
UnresolvedRelation(
Seq("a", "b", "c"),
new CaseInsensitiveStringMap(properties),
timeTravelSpec = timeTravel)))

val e1 = intercept[DateTimeException] {
parsePlan("SELECT * FROM a.b.c TIMESTAMP AS OF '2019-01-11111'")
}.getMessage
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2910,10 +2910,19 @@ class DataSourceV2SQLSuite
=== Array(Row(1), Row(2)))
assert(sql("SELECT * FROM t VERSION AS OF 2345678910").collect
=== Array(Row(3), Row(4)))
assert(sql("SELECT * FROM t FOR VERSION AS OF 'Snapshot123456789'").collect
=== Array(Row(1), Row(2)))
assert(sql("SELECT * FROM t FOR VERSION AS OF 2345678910").collect
=== Array(Row(3), Row(4)))

assert(sql("SELECT * FROM t FOR SYSTEM_VERSION AS OF 'Snapshot123456789'").collect
=== Array(Row(1), Row(2)))
assert(sql("SELECT * FROM t FOR SYSTEM_VERSION AS OF 2345678910").collect
=== Array(Row(3), Row(4)))
assert(sql("SELECT * FROM t SYSTEM_VERSION AS OF 'Snapshot123456789'").collect
=== Array(Row(1), Row(2)))
assert(sql("SELECT * FROM t SYSTEM_VERSION AS OF 2345678910").collect
=== Array(Row(3), Row(4)))
}

val ts1 = DateTimeUtils.stringToTimestampAnsi(
Expand All @@ -2938,10 +2947,19 @@ class DataSourceV2SQLSuite
=== Array(Row(5), Row(6)))
assert(sql("SELECT * FROM t TIMESTAMP AS OF '2021-01-29 00:37:58'").collect
=== Array(Row(7), Row(8)))
assert(sql("SELECT * FROM t FOR TIMESTAMP AS OF '2019-01-29 00:37:58'").collect
=== Array(Row(5), Row(6)))
assert(sql("SELECT * FROM t FOR TIMESTAMP AS OF '2021-01-29 00:37:58'").collect
=== Array(Row(7), Row(8)))

assert(sql("SELECT * FROM t FOR SYSTEM_TIME AS OF '2019-01-29 00:37:58'").collect
=== Array(Row(5), Row(6)))
assert(sql("SELECT * FROM t FOR SYSTEM_TIME AS OF '2021-01-29 00:37:58'").collect
=== Array(Row(7), Row(8)))
assert(sql("SELECT * FROM t SYSTEM_TIME AS OF '2019-01-29 00:37:58'").collect
=== Array(Row(5), Row(6)))
assert(sql("SELECT * FROM t SYSTEM_TIME AS OF '2021-01-29 00:37:58'").collect
=== Array(Row(7), Row(8)))
}
}

Expand Down

0 comments on commit f361ad8

Please sign in to comment.