-
Notifications
You must be signed in to change notification settings - Fork 395
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[#6361] feat(paimon):Support specifying primary keys during create paimon table by flink #6362
base: main
Are you sure you want to change the base?
Changes from all commits
df27ed9
b560a85
18afe70
d0ea80f
14d1299
7c94309
bfe1532
c86e7e7
e1ab8dc
ef3a906
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -27,6 +27,7 @@ | |
import java.util.List; | ||
import java.util.Map; | ||
import java.util.Objects; | ||
import java.util.Optional; | ||
import java.util.stream.Collectors; | ||
import java.util.stream.Stream; | ||
import org.apache.commons.compress.utils.Lists; | ||
|
@@ -40,6 +41,7 @@ | |
import org.apache.flink.table.catalog.CatalogTable; | ||
import org.apache.flink.table.catalog.ObjectPath; | ||
import org.apache.flink.table.catalog.ResolvedCatalogBaseTable; | ||
import org.apache.flink.table.catalog.UniqueConstraint; | ||
import org.apache.flink.table.catalog.exceptions.CatalogException; | ||
import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException; | ||
import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException; | ||
|
@@ -75,7 +77,11 @@ | |
import org.apache.gravitino.rel.Column; | ||
import org.apache.gravitino.rel.Table; | ||
import org.apache.gravitino.rel.TableChange; | ||
import org.apache.gravitino.rel.expressions.distributions.Distributions; | ||
import org.apache.gravitino.rel.expressions.sorts.SortOrder; | ||
import org.apache.gravitino.rel.expressions.transforms.Transform; | ||
import org.apache.gravitino.rel.indexes.Index; | ||
import org.apache.gravitino.rel.indexes.Indexes; | ||
|
||
/** | ||
* The BaseCatalog that provides a default implementation for all methods in the {@link | ||
|
@@ -276,8 +282,21 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig | |
propertiesConverter.toGravitinoTableProperties(table.getOptions()); | ||
Transform[] partitions = | ||
partitionConverter.toGravitinoPartitions(((CatalogTable) table).getPartitionKeys()); | ||
|
||
try { | ||
catalog().asTableCatalog().createTable(identifier, columns, comment, properties, partitions); | ||
|
||
Index[] indices = getGrivatinoIndeics(resolvedTable); | ||
catalog() | ||
.asTableCatalog() | ||
.createTable( | ||
identifier, | ||
columns, | ||
comment, | ||
properties, | ||
partitions, | ||
Distributions.NONE, | ||
new SortOrder[0], | ||
indices); | ||
} catch (NoSuchSchemaException e) { | ||
throw new DatabaseNotExistException(catalogName(), tablePath.getDatabaseName(), e); | ||
} catch (TableAlreadyExistsException e) { | ||
|
@@ -289,6 +308,20 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig | |
} | ||
} | ||
|
||
private static Index[] getGrivatinoIndeics(ResolvedCatalogBaseTable<?> resolvedTable) { | ||
Optional<UniqueConstraint> primaryKey = resolvedTable.getResolvedSchema().getPrimaryKey(); | ||
List<String> primaryColumns = primaryKey.map(UniqueConstraint::getColumns).orElse(null); | ||
if (primaryColumns == null) { | ||
return new Index[0]; | ||
} | ||
String[][] primaryField = | ||
primaryColumns.stream() | ||
.map(primaryColumn -> new String[] {primaryColumn}) | ||
.toArray(String[][]::new); | ||
Index primary = Indexes.primary("primary", primaryField); | ||
return new Index[] {primary}; | ||
} | ||
|
||
/** | ||
* The method only is used to change the comments. To alter columns, use the other alterTable API | ||
* and provide a list of TableChanges. | ||
|
@@ -521,12 +554,30 @@ protected CatalogBaseTable toFlinkTable(Table table) { | |
.column(column.name(), column.nullable() ? flinkType.nullable() : flinkType.notNull()) | ||
.withComment(column.comment()); | ||
} | ||
handleFlinkPrimaryKey(table, builder); | ||
Map<String, String> flinkTableProperties = | ||
propertiesConverter.toFlinkTableProperties(table.properties()); | ||
List<String> partitionKeys = partitionConverter.toFlinkPartitionKeys(table.partitioning()); | ||
return CatalogTable.of(builder.build(), table.comment(), partitionKeys, flinkTableProperties); | ||
} | ||
|
||
private static void handleFlinkPrimaryKey( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. could you use |
||
Table table, org.apache.flink.table.api.Schema.Builder builder) { | ||
List<Index> primaryKeyList = | ||
Arrays.stream(table.index()) | ||
.filter(index -> index.type() == Index.IndexType.PRIMARY_KEY) | ||
.collect(Collectors.toList()); | ||
if (primaryKeyList.isEmpty()) { | ||
return; | ||
} | ||
Preconditions.checkArgument( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. could you add the check for the size of filednames too? |
||
primaryKeyList.size() == 1, "More than one primary key is not supported."); | ||
builder.primaryKey( | ||
Arrays.stream(primaryKeyList.get(0).fieldNames()) | ||
.map(fieldNames -> fieldNames[0]) | ||
.collect(Collectors.toList())); | ||
} | ||
|
||
private Column toGravitinoColumn(org.apache.flink.table.catalog.Column column) { | ||
return Column.of( | ||
column.getName(), | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
use
getGrivatinoIndices
?