Skip to content
/ beam Public
forked from apache/beam

Commit

Permalink
Add SDK capability to detect if the SDK Fn Harness data channel is busy.
Browse files Browse the repository at this point in the history
  • Loading branch information
priyansndesai authored and robertwb committed May 29, 2024
1 parent 49a4290 commit 6842136
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,11 @@ message ProcessBundleProgressResponse {
// as the MonitoringInfo could be reconstructed fully by overwriting its
// payload field with the bytes specified here.
map<string, bytes> monitoring_data = 5;
// Indicates that the SDK is still busy consuming the data that as already
// been received on the data channel. If this is set, a runner may abstain
// from sending further data on the data channel until this field becomes
// unset. This field is currently used during shuffle reads on large elements.
optional bool consuming_received_data = 6;

reserved 1, 2, 4;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1672,6 +1672,11 @@ message StandardProtocols {
// during bundle processing. This is disabled by default and enabled with the
// `enable_data_sampling` experiment.
DATA_SAMPLING = 8 [(beam_urn) = "beam:protocol:data_sampling:v1"];

// Indicates whether the SDK sets the consuming_received_data field on
// the ProcessBundleProgressResponse messages.
SDK_CONSUMING_RECEIVED_DATA = 9
[(beam_urn) = "beam:protocol:sdk_consuming_received_data:v1"];
}
}

Expand Down

0 comments on commit 6842136

Please sign in to comment.