Skip to content

Commit

Permalink
fix(python): reuse table state in write engine (#2453)
Browse files Browse the repository at this point in the history
# Description
Instead of reusing the table state, it was being instantiated every time
you call write with the rust engine.

- #2448
  • Loading branch information
ion-elgreco authored Apr 25, 2024
1 parent f55ddc6 commit 6a7c684
Show file tree
Hide file tree
Showing 4 changed files with 10 additions and 4 deletions.
2 changes: 1 addition & 1 deletion python/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "deltalake-python"
version = "0.17.1"
version = "0.17.2"
authors = ["Qingping Hou <dave2008713@gmail.com>", "Will Jones <willjones127@gmail.com>"]
homepage = "https://github.com/delta-io/delta-rs"
license = "Apache-2.0"
Expand Down
1 change: 1 addition & 0 deletions python/deltalake/_internal.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ def write_to_deltalake(
data: pyarrow.RecordBatchReader,
partition_by: Optional[List[str]],
mode: str,
table: Optional[RawDeltaTable],
schema_mode: Optional[str],
predicate: Optional[str],
name: Optional[str],
Expand Down
1 change: 1 addition & 0 deletions python/deltalake/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ def write_deltalake(
data=data,
partition_by=partition_by,
mode=mode,
table=table._table if table is not None else None,
schema_mode=schema_mode,
predicate=predicate,
name=name,
Expand Down
10 changes: 7 additions & 3 deletions python/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1410,6 +1410,7 @@ fn write_to_deltalake(
table_uri: String,
data: PyArrowType<ArrowArrayStreamReader>,
mode: String,
table: Option<&RawDeltaTable>,
schema_mode: Option<String>,
partition_by: Option<Vec<String>>,
predicate: Option<String>,
Expand All @@ -1425,11 +1426,14 @@ fn write_to_deltalake(
let save_mode = mode.parse().map_err(PythonError::from)?;

let options = storage_options.clone().unwrap_or_default();
let table = rt()
.block_on(DeltaOps::try_from_uri_with_storage_options(
let table = if let Some(table) = table {
DeltaOps(table._table.clone())
} else {
rt().block_on(DeltaOps::try_from_uri_with_storage_options(
&table_uri, options,
))
.map_err(PythonError::from)?;
.map_err(PythonError::from)?
};

let mut builder = table.write(batches).with_save_mode(save_mode);
if let Some(schema_mode) = schema_mode {
Expand Down

0 comments on commit 6a7c684

Please sign in to comment.