Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

AWS: Retain Glue Catalog column comment after updating Iceberg table #10276

Merged
merged 5 commits into from
Jul 5, 2024
Merged

AWS: Retain Glue Catalog column comment after updating Iceberg table #10276

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
aa12a67
Retain Glue Catalog column comment
lawofcycles May 29, 2024
ccfc518
merge handling existing column comment to creating column set
lawofcycles Jun 8, 2024
5fac03e
apply spotless
lawofcycles Jun 8, 2024
8df1fc2
Add newline after else block for improved readability
lawofcycles Jun 27, 2024
5c809ac
Merge branch 'main' into keep-glue-column-desc
lawofcycles Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
merge handling existing column comment to creating column set
  • Loading branch information
@lawofcycles
lawofcycles committed Jun 8, 2024
commit ccfc5187d923af63ee90f491d80b6b3eb3e017d3
56 changes: 27 additions & 29 deletions aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.iceberg.aws.glue;

import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
Expand Down Expand Up @@ -262,23 +263,15 @@ static void setTableInputInformation(
Optional.ofNullable(existingTable.description()).ifPresent(tableInputBuilder::description);
}

List<Column> columns = toColumns(metadata);
Map<String, String> existingColumnMap = null;
if (existingTable != null) {
List<Column> existingColumns = existingTable.storageDescriptor().columns();
Map<String, String> existingColumnMap =
existingColumns.stream().collect(Collectors.toMap(Column::name, Column::comment));

columns =
columns.stream()
.map(
newColumn -> {
String existingComment = existingColumnMap.get(newColumn.name());
return existingComment != null && newColumn.comment() == null
? newColumn.toBuilder().comment(existingComment).build()
: newColumn;
})
.collect(Collectors.toList());
existingColumnMap =
existingColumns.stream().filter(column -> column.comment() != null).collect(Collectors.toMap(Column::name, Column::comment));
} else {
existingColumnMap = Collections.emptyMap();
}
List<Column> columns = toColumns(metadata, existingColumnMap);
amogh-jahagirdar marked this conversation as resolved.
Show resolved Hide resolved

tableInputBuilder.storageDescriptor(
storageDescriptor.location(metadata.location()).columns(columns).build());
Expand Down Expand Up @@ -343,18 +336,18 @@ private static String toTypeString(Type type) {
}
}

private static List<Column> toColumns(TableMetadata metadata) {
private static List<Column> toColumns(TableMetadata metadata, Map<String, String> existingColumnMap) {
List<Column> columns = Lists.newArrayList();
Set<String> addedNames = Sets.newHashSet();

for (NestedField field : metadata.schema().columns()) {
addColumnWithDedupe(columns, addedNames, field, true /* is current */);
addColumnWithDedupe(columns, addedNames, field, true /* is current */, existingColumnMap);
}

for (Schema schema : metadata.schemas()) {
if (schema.schemaId() != metadata.currentSchemaId()) {
for (NestedField field : schema.columns()) {
addColumnWithDedupe(columns, addedNames, field, false /* is not current */);
addColumnWithDedupe(columns, addedNames, field, false /* is not current */,existingColumnMap);
}
}
}
Expand All @@ -363,19 +356,24 @@ private static List<Column> toColumns(TableMetadata metadata) {
}

private static void addColumnWithDedupe(
List<Column> columns, Set<String> dedupe, NestedField field, boolean isCurrent) {
List<Column> columns, Set<String> dedupe, NestedField field, boolean isCurrent, Map<String, String> existingColumnMap) {
if (!dedupe.contains(field.name())) {
columns.add(
Column.builder()
.name(field.name())
.type(toTypeString(field.type()))
.comment(field.doc())
.parameters(
ImmutableMap.of(
ICEBERG_FIELD_ID, Integer.toString(field.fieldId()),
ICEBERG_FIELD_OPTIONAL, Boolean.toString(field.isOptional()),
ICEBERG_FIELD_CURRENT, Boolean.toString(isCurrent)))
.build());
Column.Builder builder = Column.builder()
.name(field.name())
.type(toTypeString(field.type()))
.parameters(
ImmutableMap.of(
ICEBERG_FIELD_ID, Integer.toString(field.fieldId()),
ICEBERG_FIELD_OPTIONAL, Boolean.toString(field.isOptional()),
ICEBERG_FIELD_CURRENT, Boolean.toString(isCurrent)));

if (field.doc() != null && !field.doc().isEmpty()) {
builder.comment(field.doc());
} else if (existingColumnMap != null && existingColumnMap.containsKey(field.name())) {
builder.comment(existingColumnMap.get(field.name()));
}

columns.add(builder.build());
dedupe.add(field.name());
}
}
Expand Down
Loading