Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-31257][SPARK-33561][SQL] Unify create table syntax #28026

Closed
wants to merge 11 commits into from
Closed

[SPARK-31257][SPARK-33561][SQL] Unify create table syntax #28026

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
37505b1
Add base tests to PlanResolutionSuite.
rdblue Oct 16, 2020
03b002b
Update parser and statements to unify CREATE.
rdblue Oct 17, 2020
745d5e8
Fix Scala 2.13 compile errors.
rdblue Oct 19, 2020
df6533e
Fix test failures.
rdblue Oct 19, 2020
e88dabb
Fix 2.13 compile error.
rdblue Oct 19, 2020
8ca5942
address review comments (#8)
cloud-fan Nov 23, 2020
a3947d9
Remove unused import.
rdblue Nov 23, 2020
1558478
Fix test imports.
rdblue Nov 24, 2020
fd60b4f
Fix more imports.
rdblue Nov 24, 2020
a471f33
Remove more unused imports.
rdblue Nov 24, 2020
25ec746
fix mistakes (#9)
cloud-fan Nov 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 13 additions & 16 deletions sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,28 +119,17 @@ statement
(RESTRICT | CASCADE)? #dropNamespace
| SHOW (DATABASES | NAMESPACES) ((FROM | IN) multipartIdentifier)?
(LIKE? pattern=STRING)? #showNamespaces
| createTableHeader ('(' colTypeList ')')? tableProvider
| createTableHeader ('(' colTypeList ')')? tableProvider?
rdblue marked this conversation as resolved.
Show resolved Hide resolved
createTableClauses
(AS? query)? #createTable
| createTableHeader ('(' columns=colTypeList ')')?
(commentSpec |
(PARTITIONED BY '(' partitionColumns=colTypeList ')' |
PARTITIONED BY partitionColumnNames=identifierList) |
bucketSpec |
skewSpec |
rowFormat |
createFileFormat |
locationSpec |
(TBLPROPERTIES tableProps=tablePropertyList))*
(AS? query)? #createHiveTable
| CREATE TABLE (IF NOT EXISTS)? target=tableIdentifier
LIKE source=tableIdentifier
(tableProvider |
rowFormat |
createFileFormat |
locationSpec |
(TBLPROPERTIES tableProps=tablePropertyList))* #createTableLike
| replaceTableHeader ('(' colTypeList ')')? tableProvider
| replaceTableHeader ('(' colTypeList ')')? tableProvider?
createTableClauses
(AS? query)? #replaceTable
| ANALYZE TABLE multipartIdentifier partitionSpec? COMPUTE STATISTICS
Expand Down Expand Up @@ -393,8 +382,11 @@ tableProvider

createTableClauses
:((OPTIONS options=tablePropertyList) |
(PARTITIONED BY partitioning=transformList) |
(PARTITIONED BY partitioning=partitionFieldList) |
cloud-fan marked this conversation as resolved.
Show resolved Hide resolved
skewSpec |
bucketSpec |
rowFormat |
createFileFormat |
locationSpec |
commentSpec |
(TBLPROPERTIES tableProps=tablePropertyList))*
Expand Down Expand Up @@ -741,8 +733,13 @@ namedExpressionSeq
: namedExpression (',' namedExpression)*
;

transformList
: '(' transforms+=transform (',' transforms+=transform)* ')'
partitionFieldList
: '(' fields+=partitionField (',' fields+=partitionField)* ')'
;

partitionField
rdblue marked this conversation as resolved.
Show resolved Hide resolved
: transform #partitionTransform
| colType #partitionColumn
;

transform
Expand Down
10 changes: 10 additions & 0 deletions sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ public interface TableCatalog extends CatalogPlugin {
*/
String PROP_LOCATION = "location";

/**
* A reserved property to specify a table was created with EXTERNAL.
*/
String PROP_EXTERNAL = "external";
rdblue marked this conversation as resolved.
Show resolved Hide resolved

/**
* A reserved property to specify the description of the table.
*/
Expand All @@ -61,6 +66,11 @@ public interface TableCatalog extends CatalogPlugin {
*/
String PROP_OWNER = "owner";

/**
* A prefix used to pass OPTIONS in table properties
*/
String OPTION_PREFIX = "option.";

/**
* List the tables in a namespace from the catalog.
* <p>
Expand Down
16 changes: 8 additions & 8 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
RenameTable(catalog.asTableCatalog, oldName.asIdentifier, newNameParts.asIdentifier)

case c @ CreateTableStatement(
NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _, _) =>
assertNoNullTypeInSchema(c.tableSchema)
assertNoCharTypeInSchema(c.tableSchema)
CreateV2Table(
Expand All @@ -152,11 +152,11 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
c.tableSchema,
// convert the bucket spec and add it as a transform
c.partitioning ++ c.bucketSpec.map(_.asTransform),
convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
convertTableProperties(c),
ignoreIfExists = c.ifNotExists)

case c @ CreateTableAsSelectStatement(
NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _, _, _) =>
if (c.asSelect.resolved) {
assertNoNullTypeInSchema(c.asSelect.schema)
}
Expand All @@ -166,12 +166,12 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
// convert the bucket spec and add it as a transform
c.partitioning ++ c.bucketSpec.map(_.asTransform),
c.asSelect,
convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
convertTableProperties(c),
writeOptions = c.writeOptions,
ignoreIfExists = c.ifNotExists)

case c @ ReplaceTableStatement(
NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
assertNoNullTypeInSchema(c.tableSchema)
assertNoCharTypeInSchema(c.tableSchema)
ReplaceTable(
Expand All @@ -180,11 +180,11 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
c.tableSchema,
// convert the bucket spec and add it as a transform
c.partitioning ++ c.bucketSpec.map(_.asTransform),
convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
convertTableProperties(c),
orCreate = c.orCreate)

case c @ ReplaceTableAsSelectStatement(
NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _, _) =>
if (c.asSelect.resolved) {
assertNoNullTypeInSchema(c.asSelect.schema)
}
Expand All @@ -194,7 +194,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
// convert the bucket spec and add it as a transform
c.partitioning ++ c.bucketSpec.map(_.asTransform),
c.asSelect,
convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
convertTableProperties(c),
writeOptions = c.writeOptions,
orCreate = c.orCreate)

Expand Down
Loading