From ea6b281735193063add6b9a1cff094a97a380686 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Mon, 23 Jan 2023 10:25:58 -0800 Subject: [PATCH 01/17] Eager workflow dispatch --- api/historyservice/v1/request_response.pb.go | 808 ++++++++++-------- common/metrics/metric_defs.go | 1 + go.mod | 6 +- go.sum | 12 +- proto/api | 2 +- .../historyservice/v1/request_response.proto | 2 + service/frontend/workflow_handler.go | 3 +- service/history/api/create_workflow_util.go | 52 +- .../signal_with_start_workflow.go | 3 +- service/history/api/startworkflow/api.go | 192 ++++- service/history/handler.go | 6 +- service/history/historyEngine.go | 2 +- service/history/historyEngine_test.go | 36 + service/history/workflow/mutable_state.go | 2 +- .../history/workflow/mutable_state_impl.go | 12 +- .../history/workflow/mutable_state_mock.go | 9 +- service/history/workflow/retry.go | 4 +- .../workflow/workflow_task_state_machine.go | 34 +- .../history/workflowTaskHandlerCallbacks.go | 4 +- 19 files changed, 740 insertions(+), 450 deletions(-) diff --git a/api/historyservice/v1/request_response.pb.go b/api/historyservice/v1/request_response.pb.go index cf2f377d348..acda00a13e6 100644 --- a/api/historyservice/v1/request_response.pb.go +++ b/api/historyservice/v1/request_response.pb.go @@ -182,6 +182,8 @@ func (m *StartWorkflowExecutionRequest) GetFirstWorkflowTaskBackoff() *time.Dura type StartWorkflowExecutionResponse struct { RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` Clock *v15.VectorClock `protobuf:"bytes,2,opt,name=clock,proto3" json:"clock,omitempty"` + // Set if request_eager_execution is set on the start request + EagerWorkflowTask *v1.PollWorkflowTaskQueueResponse `protobuf:"bytes,3,opt,name=eager_workflow_task,json=eagerWorkflowTask,proto3" json:"eager_workflow_task,omitempty"` } func (m *StartWorkflowExecutionResponse) Reset() { *m = StartWorkflowExecutionResponse{} } @@ -230,6 +232,13 @@ func (m *StartWorkflowExecutionResponse) GetClock() *v15.VectorClock { return nil } +func (m *StartWorkflowExecutionResponse) GetEagerWorkflowTask() *v1.PollWorkflowTaskQueueResponse { + if m != nil { + return m.EagerWorkflowTask + } + return nil +} + type GetMutableStateRequest struct { NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` Execution *v14.WorkflowExecution `protobuf:"bytes,2,opt,name=execution,proto3" json:"execution,omitempty"` @@ -5567,287 +5576,289 @@ func init() { } var fileDescriptor_b8c78c1d460a3711 = []byte{ - // 4477 bytes of a gzipped FileDescriptorProto + // 4499 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3c, 0x4b, 0x6c, 0x1c, 0xc9, 0x75, 0x6a, 0xce, 0x0c, 0x39, 0x7c, 0x24, 0xe7, 0xd3, 0xfc, 0x0d, 0x49, 0x69, 0x44, 0xf5, 0x8a, - 0x12, 0x57, 0xbb, 0x1a, 0xea, 0x63, 0x7b, 0x65, 0xc5, 0xbb, 0x1b, 0x89, 0xfa, 0x51, 0x90, 0x64, - 0x6e, 0x93, 0x2b, 0x2d, 0xd6, 0x5e, 0xf7, 0x36, 0xbb, 0x8b, 0x64, 0x87, 0x33, 0xdd, 0xa3, 0xae, - 0x1e, 0x92, 0xb3, 0x39, 0xe4, 0x63, 0x24, 0x88, 0x9d, 0x20, 0x59, 0x20, 0x17, 0x23, 0x70, 0x80, - 0x20, 0x40, 0x10, 0x5f, 0x82, 0x1c, 0x72, 0x08, 0x7c, 0xc8, 0x25, 0x01, 0x82, 0x20, 0xa7, 0x45, - 0x2e, 0x31, 0x12, 0x20, 0xce, 0x6a, 0x0f, 0x71, 0x90, 0x1c, 0x7c, 0x0c, 0x82, 0x1c, 0x82, 0xfa, - 0xf5, 0xf4, 0x6f, 0x3e, 0x4d, 0x4a, 0xd1, 0xda, 0xd9, 0x1b, 0xa7, 0xea, 0xbd, 0x57, 0xf5, 0xfe, - 0x55, 0xaf, 0x5e, 0x13, 0xbe, 0xe6, 0xa1, 0x46, 0xd3, 0x71, 0xf5, 0xfa, 0x0a, 0x46, 0xee, 0x3e, - 0x72, 0x57, 0xf4, 0xa6, 0xb5, 0xb2, 0x6b, 0x61, 0xcf, 0x71, 0xdb, 0x64, 0xc4, 0x32, 0xd0, 0xca, - 0xfe, 0xe5, 0x15, 0x17, 0x3d, 0x6d, 0x21, 0xec, 0x69, 0x2e, 0xc2, 0x4d, 0xc7, 0xc6, 0xa8, 0xd6, - 0x74, 0x1d, 0xcf, 0x91, 0x97, 0x04, 0x76, 0x8d, 0x61, 0xd7, 0xf4, 0xa6, 0x55, 0x0b, 0x63, 0xd7, - 0xf6, 0x2f, 0xcf, 0x57, 0x77, 0x1c, 0x67, 0xa7, 0x8e, 0x56, 0x28, 0xd2, 0x56, 0x6b, 0x7b, 0xc5, - 0x6c, 0xb9, 0xba, 0x67, 0x39, 0x36, 0x23, 0x33, 0x7f, 0x3a, 0x3a, 0xef, 0x59, 0x0d, 0x84, 0x3d, - 0xbd, 0xd1, 0xe4, 0x00, 0x67, 0x4c, 0xd4, 0x44, 0xb6, 0x89, 0x6c, 0xc3, 0x42, 0x78, 0x65, 0xc7, - 0xd9, 0x71, 0xe8, 0x38, 0xfd, 0x8b, 0x83, 0x9c, 0xf5, 0x19, 0x21, 0x1c, 0x18, 0x4e, 0xa3, 0xe1, - 0xd8, 0x64, 0xe7, 0x0d, 0x84, 0xb1, 0xbe, 0xc3, 0x37, 0x3c, 0xbf, 0x14, 0x82, 0xe2, 0x3b, 0x8d, - 0x83, 0x9d, 0x0f, 0x81, 0x79, 0x3a, 0xde, 0x7b, 0xda, 0x42, 0x2d, 0x14, 0x07, 0x0c, 0xaf, 0x8a, - 0xec, 0x56, 0x03, 0x13, 0xa0, 0x03, 0xc7, 0xdd, 0xdb, 0xae, 0x3b, 0x07, 0x1c, 0xea, 0x5c, 0x08, - 0x4a, 0x4c, 0xc6, 0xa9, 0xbd, 0x12, 0x82, 0x7b, 0xda, 0x42, 0x49, 0x7b, 0x0b, 0xb3, 0xb0, 0xad, - 0x5b, 0xf5, 0x96, 0x9b, 0xb0, 0xb3, 0x0b, 0x49, 0x8a, 0x35, 0xea, 0x8e, 0xb1, 0x17, 0x87, 0x7d, - 0xbd, 0x87, 0x11, 0xc4, 0xa1, 0x5f, 0x4d, 0x82, 0xf6, 0x59, 0x67, 0x92, 0xe7, 0xa0, 0xaf, 0xf5, - 0x04, 0x8d, 0x48, 0xe9, 0x7c, 0x4f, 0x60, 0xa2, 0x04, 0x0e, 0x78, 0x31, 0x09, 0xb0, 0xbb, 0x54, - 0x6b, 0x49, 0xe0, 0xb6, 0xde, 0x40, 0xb8, 0xa9, 0x1b, 0x09, 0x92, 0xbb, 0x94, 0x04, 0xef, 0xa2, - 0x66, 0xdd, 0x32, 0xa8, 0xd1, 0xc6, 0x31, 0xae, 0x26, 0x61, 0x34, 0x91, 0x8b, 0x2d, 0xec, 0x21, - 0x9b, 0xad, 0x81, 0x0e, 0x91, 0xd1, 0x22, 0xe8, 0x98, 0x23, 0xbd, 0x3d, 0x00, 0x92, 0x60, 0x4a, - 0x6b, 0xb4, 0x3c, 0x7d, 0xab, 0x8e, 0x34, 0xec, 0xe9, 0x9e, 0x58, 0xf5, 0x2b, 0x89, 0x56, 0xd5, - 0xd7, 0x69, 0xe7, 0xaf, 0x27, 0x2d, 0xac, 0x9b, 0x0d, 0xcb, 0xee, 0x8b, 0xab, 0xfc, 0xf6, 0x30, - 0x9c, 0xda, 0xf0, 0x74, 0xd7, 0x7b, 0xc2, 0x97, 0xbb, 0x2d, 0xd8, 0x52, 0x19, 0x82, 0x7c, 0x06, - 0xc6, 0x7d, 0xd9, 0x6a, 0x96, 0x59, 0x91, 0x16, 0xa5, 0xe5, 0x51, 0x75, 0xcc, 0x1f, 0x5b, 0x33, - 0x65, 0x03, 0x26, 0x30, 0xa1, 0xa1, 0xf1, 0x45, 0x2a, 0x43, 0x8b, 0xd2, 0xf2, 0xd8, 0x95, 0xb7, - 0x7c, 0x45, 0xd1, 0x30, 0x12, 0x61, 0xa8, 0xb6, 0x7f, 0xb9, 0xd6, 0x73, 0x65, 0x75, 0x9c, 0x12, - 0x15, 0xfb, 0xd8, 0x85, 0xe9, 0xa6, 0xee, 0x22, 0xdb, 0xd3, 0x7c, 0xc9, 0x6b, 0x96, 0xbd, 0xed, - 0x54, 0x32, 0x74, 0xb1, 0x2f, 0xd5, 0x92, 0x42, 0x97, 0x6f, 0x91, 0xfb, 0x97, 0x6b, 0xeb, 0x14, - 0xdb, 0x5f, 0x65, 0xcd, 0xde, 0x76, 0xd4, 0xc9, 0x66, 0x7c, 0x50, 0xae, 0xc0, 0x88, 0xee, 0x11, - 0x6a, 0x5e, 0x25, 0xbb, 0x28, 0x2d, 0xe7, 0x54, 0xf1, 0x53, 0x6e, 0x80, 0xe2, 0x6b, 0xb0, 0xb3, - 0x0b, 0x74, 0xd8, 0xb4, 0x58, 0xf8, 0xd3, 0x48, 0x9c, 0xab, 0xe4, 0xe8, 0x86, 0xe6, 0x6b, 0x2c, - 0x08, 0xd6, 0x44, 0x10, 0xac, 0x6d, 0x8a, 0x20, 0x78, 0x33, 0xfb, 0xf1, 0x8f, 0x4f, 0x4b, 0xea, - 0xe9, 0x83, 0x28, 0xe7, 0xb7, 0x7d, 0x4a, 0x04, 0x56, 0xde, 0x85, 0x39, 0xc3, 0xb1, 0x3d, 0xcb, - 0x6e, 0x21, 0x4d, 0xc7, 0x9a, 0x8d, 0x0e, 0x34, 0xcb, 0xb6, 0x3c, 0x4b, 0xf7, 0x1c, 0xb7, 0x32, - 0xbc, 0x28, 0x2d, 0x17, 0xae, 0x5c, 0x0c, 0xcb, 0x98, 0x7a, 0x17, 0x61, 0x76, 0x95, 0xe3, 0xdd, - 0xc0, 0x8f, 0xd0, 0xc1, 0x9a, 0x40, 0x52, 0x67, 0x8c, 0xc4, 0x71, 0xf9, 0x21, 0x94, 0xc5, 0x8c, - 0xa9, 0xf1, 0x10, 0x54, 0x19, 0xa1, 0x7c, 0x2c, 0x86, 0x57, 0xe0, 0x93, 0x64, 0x8d, 0x3b, 0xec, - 0x4f, 0xb5, 0xe4, 0xa3, 0xf2, 0x11, 0xf9, 0x31, 0xcc, 0xd4, 0x75, 0xec, 0x69, 0x86, 0xd3, 0x68, - 0xd6, 0x11, 0x95, 0x8c, 0x8b, 0x70, 0xab, 0xee, 0x55, 0xf2, 0x49, 0x34, 0x79, 0x88, 0xa1, 0x3a, - 0x6a, 0xd7, 0x1d, 0xdd, 0xc4, 0xea, 0x14, 0xc1, 0x5f, 0xf5, 0xd1, 0x55, 0x8a, 0x2d, 0x7f, 0x0b, - 0x16, 0xb6, 0x2d, 0x17, 0x7b, 0x9a, 0xaf, 0x05, 0x12, 0x45, 0xb4, 0x2d, 0xdd, 0xd8, 0x73, 0xb6, - 0xb7, 0x2b, 0xa3, 0x94, 0xf8, 0x5c, 0x4c, 0xf0, 0xb7, 0x78, 0x76, 0xba, 0x99, 0xfd, 0x1e, 0x91, - 0x7b, 0x85, 0xd2, 0x10, 0x66, 0xb7, 0xa9, 0xe3, 0xbd, 0x9b, 0x8c, 0x80, 0x72, 0x08, 0xd5, 0x6e, - 0x26, 0xc9, 0xbc, 0x46, 0x9e, 0x86, 0x61, 0xb7, 0x65, 0x77, 0xfc, 0x20, 0xe7, 0xb6, 0xec, 0x35, - 0x53, 0x7e, 0x1b, 0x72, 0x34, 0x14, 0x73, 0xcb, 0x7f, 0x35, 0xd1, 0x18, 0x29, 0x04, 0xe1, 0xf2, - 0x31, 0x32, 0x3c, 0xc7, 0x5d, 0x25, 0x3f, 0x55, 0x86, 0xa7, 0xfc, 0x87, 0x04, 0x33, 0x77, 0x91, - 0xf7, 0x90, 0x85, 0x85, 0x0d, 0x12, 0x15, 0x52, 0x38, 0xe0, 0x5d, 0x18, 0xf5, 0xcd, 0x31, 0xbe, - 0x85, 0xb0, 0x88, 0xe3, 0xbc, 0x75, 0x70, 0xe5, 0xab, 0x30, 0x83, 0x0e, 0x9b, 0xc8, 0xf0, 0x90, - 0xa9, 0xd9, 0xe8, 0xd0, 0xd3, 0xd0, 0x3e, 0xf1, 0x38, 0xcb, 0xa4, 0x5e, 0x96, 0x51, 0x27, 0xc5, - 0xec, 0x23, 0x74, 0xe8, 0xdd, 0x26, 0x73, 0x6b, 0xa6, 0x7c, 0x09, 0xa6, 0x8c, 0x96, 0x4b, 0x5d, - 0x73, 0xcb, 0xd5, 0x6d, 0x63, 0x57, 0xf3, 0x9c, 0x3d, 0x64, 0x53, 0xe7, 0x19, 0x57, 0x65, 0x3e, - 0x77, 0x93, 0x4e, 0x6d, 0x92, 0x19, 0xe5, 0xc7, 0x79, 0x98, 0x8d, 0x71, 0xcb, 0x25, 0x1c, 0xe2, - 0x45, 0x3a, 0x06, 0x2f, 0x6b, 0x30, 0xd1, 0x31, 0x93, 0x76, 0x13, 0x71, 0xc1, 0x9c, 0xed, 0x47, - 0x6c, 0xb3, 0xdd, 0x44, 0xea, 0xf8, 0x41, 0xe0, 0x97, 0xac, 0xc0, 0x44, 0x92, 0x34, 0xc6, 0xec, - 0x80, 0x14, 0xbe, 0x0a, 0x73, 0x4d, 0x17, 0xed, 0x5b, 0x4e, 0x0b, 0x6b, 0x34, 0x70, 0x21, 0xb3, - 0x03, 0x9f, 0xa5, 0xf0, 0x33, 0x02, 0x60, 0x83, 0xcd, 0x0b, 0xd4, 0x8b, 0x30, 0x49, 0xdd, 0x85, - 0xd9, 0xb6, 0x8f, 0x94, 0xa3, 0x48, 0x25, 0x32, 0x75, 0x87, 0xcc, 0x08, 0xf0, 0x55, 0x00, 0x6a, - 0xf6, 0xf4, 0x08, 0x43, 0xe3, 0x40, 0x8c, 0x2b, 0xff, 0x84, 0x43, 0x18, 0x23, 0x16, 0xfe, 0x0e, - 0xf9, 0xa1, 0x8e, 0x7a, 0xe2, 0x4f, 0x79, 0x1d, 0xca, 0xd8, 0xb3, 0x8c, 0xbd, 0xb6, 0x16, 0xa0, - 0x35, 0x92, 0x82, 0x56, 0x91, 0xa1, 0xfb, 0x03, 0xf2, 0x2f, 0xc3, 0x6b, 0x31, 0x8a, 0x1a, 0x36, - 0x76, 0x91, 0xd9, 0xaa, 0x23, 0xcd, 0x73, 0x98, 0x54, 0x68, 0x88, 0x74, 0x5a, 0x5e, 0x65, 0x6c, - 0x30, 0x67, 0x5d, 0x8a, 0x2c, 0xb3, 0xc1, 0x09, 0x6e, 0x3a, 0x54, 0x88, 0x9b, 0x8c, 0x5a, 0x57, - 0x1b, 0x9c, 0xe8, 0x66, 0x83, 0xf2, 0x37, 0xa0, 0xe0, 0x9b, 0x07, 0xcd, 0xc2, 0x95, 0x22, 0x8d, - 0xa8, 0xc9, 0x89, 0xc4, 0x0f, 0xac, 0x31, 0x93, 0x63, 0xd6, 0xeb, 0x9b, 0x1a, 0xfd, 0x29, 0x3f, - 0x81, 0x62, 0x88, 0x78, 0x0b, 0x57, 0x4a, 0x94, 0x7a, 0xad, 0x4b, 0xbc, 0x4e, 0x24, 0xdb, 0xc2, - 0x6a, 0x21, 0x48, 0xb7, 0x85, 0xe5, 0x0f, 0xa0, 0xbc, 0x4f, 0x8e, 0x14, 0x8e, 0xad, 0xb1, 0xf3, - 0x9c, 0x85, 0x70, 0xa5, 0x4c, 0x45, 0x79, 0xa9, 0xd6, 0xe3, 0xf0, 0xce, 0xc2, 0x0e, 0x45, 0xbc, - 0x27, 0xf0, 0xd4, 0xd2, 0x7e, 0x64, 0x44, 0x7e, 0x0b, 0x4e, 0x5a, 0xc4, 0x7c, 0xa3, 0x6a, 0x44, - 0x36, 0x71, 0x54, 0xb3, 0x22, 0x2f, 0x4a, 0xcb, 0x79, 0xb5, 0x62, 0xe1, 0x8d, 0xb0, 0x56, 0x6e, - 0xb3, 0x79, 0xf9, 0x4b, 0x30, 0x1b, 0xb3, 0x64, 0xef, 0x90, 0xc6, 0xcb, 0x49, 0x16, 0x40, 0xc2, - 0xd6, 0xbc, 0x79, 0x48, 0xa2, 0xe7, 0x55, 0x98, 0xe1, 0x08, 0x7e, 0x4e, 0xe5, 0x41, 0x76, 0x8a, - 0xc6, 0xba, 0x49, 0x3a, 0xdb, 0x71, 0x72, 0x12, 0x72, 0xef, 0x67, 0xf3, 0xf9, 0xd2, 0xe8, 0xfd, - 0x6c, 0x7e, 0xb4, 0x04, 0xf7, 0xb3, 0x79, 0x28, 0x8d, 0xdd, 0xcf, 0xe6, 0xc7, 0x4b, 0x13, 0xf7, - 0xb3, 0xf9, 0x42, 0xa9, 0xa8, 0xfc, 0xa7, 0x04, 0xb3, 0xeb, 0x4e, 0xbd, 0xfe, 0xff, 0x24, 0xa0, - 0xfe, 0x41, 0x1e, 0x2a, 0x71, 0x76, 0xbf, 0x88, 0xa8, 0x5f, 0x44, 0xd4, 0xe7, 0x1e, 0x51, 0xc7, - 0xbb, 0x46, 0xd4, 0xc4, 0xd8, 0x54, 0x78, 0x6e, 0xb1, 0xe9, 0x67, 0x33, 0x60, 0xf7, 0x88, 0x88, - 0xe5, 0xa3, 0x44, 0x44, 0x39, 0x5d, 0x44, 0x9c, 0x28, 0x15, 0x94, 0xef, 0x48, 0xb0, 0xa0, 0x22, - 0x8c, 0xbc, 0x48, 0xd0, 0x7e, 0x09, 0xf1, 0x50, 0xa9, 0xc2, 0xc9, 0xe4, 0xad, 0xb0, 0x58, 0xa5, - 0xfc, 0x20, 0x03, 0x8b, 0x2a, 0x32, 0x1c, 0xd7, 0x0c, 0x9e, 0xcf, 0xb9, 0x77, 0xa7, 0xd8, 0xf0, - 0x7b, 0x20, 0xc7, 0x6f, 0x6a, 0xe9, 0x77, 0x5e, 0x8e, 0x5d, 0xd1, 0xe4, 0xd7, 0x41, 0x16, 0x2e, - 0x68, 0x46, 0xc3, 0x57, 0xc9, 0x9f, 0x11, 0x91, 0x65, 0x16, 0x46, 0xa8, 0xef, 0xfa, 0x11, 0x6b, - 0x98, 0xfc, 0x5c, 0x33, 0xe5, 0x53, 0x00, 0xe2, 0x4a, 0xce, 0x03, 0xd3, 0xa8, 0x3a, 0xca, 0x47, - 0xd6, 0x4c, 0xf9, 0x43, 0x18, 0x6f, 0x3a, 0xf5, 0xba, 0x7f, 0xa3, 0x66, 0x31, 0xe9, 0xcd, 0xbe, - 0x37, 0x6a, 0x92, 0x04, 0x82, 0x92, 0x0b, 0x2a, 0x5a, 0x1d, 0x23, 0x24, 0x85, 0x10, 0xfd, 0x2b, - 0xcb, 0xc8, 0x11, 0xaf, 0x2c, 0x7f, 0x94, 0x87, 0x33, 0x3d, 0x54, 0xc5, 0x93, 0x4f, 0x2c, 0x67, - 0x48, 0x47, 0xce, 0x19, 0x3d, 0xf3, 0xc1, 0x50, 0xcf, 0x7c, 0x90, 0x4e, 0x69, 0xcb, 0x50, 0xea, - 0x92, 0x6f, 0x0a, 0x38, 0x4c, 0x37, 0x96, 0xc6, 0x72, 0xf1, 0x34, 0x16, 0x28, 0x27, 0x0c, 0x87, - 0xcb, 0x09, 0xd7, 0xa0, 0xc2, 0xe3, 0x7b, 0xa0, 0x98, 0xc0, 0x4f, 0x5a, 0x23, 0xf4, 0xa4, 0x35, - 0xc3, 0xe6, 0x3b, 0x05, 0x02, 0x7e, 0xce, 0x7a, 0x0a, 0xb3, 0x9e, 0xab, 0xdb, 0xd8, 0x22, 0xcb, - 0x86, 0x2e, 0xc3, 0xfc, 0x86, 0xfd, 0xd5, 0x7e, 0x01, 0x77, 0x53, 0xa0, 0x07, 0x95, 0x47, 0x6b, - 0x22, 0xd3, 0x5e, 0xd2, 0x94, 0xbc, 0x03, 0xa7, 0x12, 0x6a, 0x1f, 0x81, 0x54, 0x37, 0x9a, 0x22, - 0xd5, 0xcd, 0xc7, 0xfc, 0xaa, 0x93, 0xf5, 0xce, 0xc0, 0x78, 0x28, 0xe1, 0x8c, 0xd1, 0x84, 0x33, - 0xb6, 0x15, 0xc8, 0x34, 0x77, 0xa1, 0xd0, 0x51, 0x27, 0xad, 0xb9, 0x8c, 0x0f, 0x58, 0x73, 0x99, - 0xf0, 0xf1, 0x68, 0x85, 0x65, 0x15, 0xc6, 0x85, 0xa6, 0x29, 0x99, 0x89, 0x01, 0xc9, 0x8c, 0x71, - 0x2c, 0x4a, 0xc4, 0x81, 0x91, 0xa7, 0x2d, 0xc4, 0xb3, 0x5d, 0x66, 0x79, 0xec, 0xca, 0xbb, 0xb5, - 0x81, 0xca, 0xe8, 0xb5, 0xbe, 0xde, 0x53, 0x7b, 0x87, 0xd1, 0xbd, 0x6d, 0x7b, 0x6e, 0x5b, 0x15, - 0xab, 0x74, 0x5c, 0xb7, 0x78, 0x34, 0xd7, 0x9d, 0xff, 0x10, 0xc6, 0x83, 0x94, 0xe5, 0x12, 0x64, - 0xf6, 0x50, 0x9b, 0xc7, 0x51, 0xf2, 0xa7, 0x7c, 0x1d, 0x72, 0xfb, 0x7a, 0xbd, 0xd5, 0xe5, 0x88, - 0x47, 0x2b, 0xd9, 0x41, 0x6f, 0x25, 0xd4, 0xda, 0x2a, 0x43, 0xb9, 0x3e, 0x74, 0x4d, 0x62, 0xf9, - 0x27, 0x10, 0xcd, 0x6f, 0x18, 0x9e, 0xb5, 0x6f, 0x79, 0xed, 0x2f, 0xa2, 0x79, 0xda, 0x68, 0x1e, - 0x94, 0xdc, 0x0b, 0x8c, 0xe6, 0x7f, 0x93, 0x15, 0xd1, 0x3c, 0x51, 0x55, 0x3c, 0x9a, 0x3f, 0x82, - 0x62, 0x44, 0x5c, 0x3c, 0x9e, 0x2f, 0x85, 0x79, 0x09, 0x04, 0x1a, 0x76, 0x80, 0x6b, 0x53, 0x11, - 0xaa, 0x85, 0xb0, 0x48, 0x63, 0xfe, 0x37, 0x74, 0x14, 0xff, 0x0b, 0x04, 0xd8, 0x4c, 0x38, 0xc0, - 0x22, 0xa8, 0x8a, 0x33, 0x2c, 0x1f, 0xd2, 0x22, 0x71, 0x23, 0x3b, 0xe0, 0x82, 0x0b, 0x9c, 0xce, - 0x0d, 0x46, 0x66, 0x23, 0x14, 0x45, 0x1e, 0x42, 0x79, 0x17, 0xe9, 0xae, 0xb7, 0x85, 0x74, 0x4f, - 0x33, 0x91, 0xa7, 0x5b, 0x75, 0xcc, 0xab, 0xc0, 0xfd, 0x2b, 0x9d, 0x25, 0x1f, 0xf5, 0x16, 0xc3, - 0x8c, 0xa7, 0xcc, 0xe1, 0x23, 0xa7, 0xcc, 0x8b, 0x01, 0xc7, 0xf1, 0x1d, 0x8a, 0xda, 0xc8, 0x68, - 0xc7, 0x1b, 0x1e, 0x89, 0x89, 0x8e, 0x15, 0xe5, 0x8f, 0x68, 0x45, 0x3f, 0x94, 0xe0, 0x15, 0x66, - 0x2c, 0xa1, 0xb0, 0xc6, 0x0b, 0xb9, 0xa9, 0x7c, 0xde, 0x81, 0x12, 0x2f, 0x1f, 0xa3, 0xc8, 0xbb, - 0xc2, 0xad, 0xbe, 0x7e, 0x33, 0xc0, 0x16, 0xd4, 0xa2, 0xa0, 0xce, 0x07, 0x94, 0x5f, 0x1f, 0x82, - 0xb3, 0xbd, 0x11, 0xb9, 0x13, 0xe0, 0xce, 0xf1, 0x40, 0xbc, 0xa6, 0x70, 0x2f, 0xb8, 0xf7, 0xbc, - 0x02, 0x3f, 0xb9, 0x0b, 0x86, 0x3d, 0x0f, 0x41, 0x41, 0xe7, 0x8e, 0x49, 0x93, 0x2e, 0xae, 0x0c, - 0xd1, 0x5c, 0xf3, 0xd6, 0x51, 0x83, 0x08, 0x5f, 0x68, 0x42, 0x0f, 0x4c, 0x61, 0xe5, 0xcf, 0x25, - 0x12, 0xb1, 0x63, 0x42, 0xb8, 0xa3, 0x5b, 0xf5, 0x54, 0xda, 0xdb, 0x85, 0xc2, 0x36, 0xc5, 0x89, - 0xe8, 0xee, 0xc6, 0x51, 0x74, 0x17, 0x5a, 0x5d, 0x9d, 0xd8, 0x0e, 0xfe, 0x54, 0x5e, 0x21, 0x71, - 0xab, 0x2b, 0x0a, 0xbf, 0x56, 0xfc, 0x50, 0x02, 0x25, 0x1e, 0xdd, 0xee, 0x09, 0xcf, 0x4b, 0xc1, - 0x58, 0x33, 0xe8, 0xeb, 0x61, 0xde, 0x56, 0x07, 0xe0, 0xad, 0xdf, 0x16, 0x02, 0xe1, 0x40, 0x30, - 0xb8, 0x4e, 0x5c, 0xaa, 0x07, 0x1e, 0x37, 0x90, 0x57, 0xa1, 0x64, 0xe8, 0xb6, 0x81, 0xfc, 0x2c, - 0x83, 0xd8, 0xfe, 0xf3, 0x6a, 0x91, 0x8d, 0xab, 0x62, 0x38, 0xe8, 0xa5, 0x41, 0x9a, 0x2f, 0xc9, - 0x4b, 0x7b, 0x6d, 0x21, 0xee, 0xa5, 0xe7, 0x7c, 0x27, 0xed, 0x82, 0xc7, 0x35, 0x1e, 0x30, 0xe4, - 0x20, 0xe0, 0xff, 0xbd, 0x21, 0x77, 0x5d, 0xbd, 0xbb, 0x21, 0x27, 0xa1, 0x70, 0xb6, 0xfe, 0x82, - 0x1a, 0x72, 0x9c, 0x7f, 0xaa, 0xe1, 0x54, 0x8c, 0xfd, 0x12, 0x14, 0xc2, 0xf6, 0x92, 0xc2, 0x8a, - 0xfb, 0xad, 0xaf, 0x4e, 0x84, 0x4c, 0x4e, 0x59, 0x4a, 0xb6, 0x37, 0x1f, 0x89, 0x33, 0xf7, 0xb7, - 0x43, 0x50, 0xdd, 0xb0, 0x76, 0x6c, 0xbd, 0x7e, 0x9c, 0xd7, 0xe8, 0x6d, 0x28, 0x60, 0x4a, 0x24, - 0xc2, 0xd8, 0xdb, 0xfd, 0x9f, 0xa3, 0x7b, 0xae, 0xad, 0x4e, 0x30, 0xb2, 0x62, 0x2b, 0x16, 0x2c, - 0xa0, 0x43, 0x0f, 0xb9, 0x64, 0xa5, 0x84, 0xd3, 0x69, 0x26, 0xed, 0xe9, 0x74, 0x4e, 0x50, 0x8b, - 0x4d, 0xc9, 0x35, 0x98, 0x34, 0x76, 0xad, 0xba, 0xd9, 0x59, 0xc7, 0xb1, 0xeb, 0x6d, 0x7a, 0x78, - 0xc9, 0xab, 0x65, 0x3a, 0x25, 0x90, 0xbe, 0x6e, 0xd7, 0xdb, 0xca, 0x19, 0x38, 0xdd, 0x95, 0x17, - 0x2e, 0xeb, 0x7f, 0x90, 0xe0, 0x3c, 0x87, 0xb1, 0xbc, 0xdd, 0x63, 0xb7, 0x00, 0x7c, 0x5b, 0x82, - 0x39, 0x2e, 0xf5, 0x03, 0xcb, 0xdb, 0xd5, 0x92, 0xfa, 0x01, 0xee, 0x0d, 0xaa, 0x80, 0x7e, 0x1b, - 0x52, 0x67, 0x70, 0x18, 0x50, 0xd8, 0xd9, 0x0d, 0x58, 0xee, 0x4f, 0xa2, 0xe7, 0x4b, 0xae, 0xf2, - 0x57, 0x12, 0x9c, 0x56, 0x51, 0xc3, 0xd9, 0x47, 0x8c, 0xd2, 0x11, 0x1f, 0x10, 0x5e, 0xdc, 0x8d, - 0x25, 0x7c, 0xd5, 0xc8, 0x44, 0xae, 0x1a, 0x8a, 0x42, 0xc2, 0x5e, 0xb7, 0xed, 0x0b, 0xdd, 0x0f, - 0xc1, 0x99, 0x4d, 0xe4, 0x36, 0x2c, 0x5b, 0xf7, 0xd0, 0x71, 0xb4, 0xee, 0x40, 0xd9, 0x13, 0x74, - 0x22, 0xca, 0xbe, 0xd9, 0x57, 0xd9, 0x7d, 0x77, 0xa0, 0x96, 0x7c, 0xe2, 0x3f, 0x03, 0x3e, 0x77, - 0x16, 0x94, 0x5e, 0x1c, 0x71, 0xd1, 0xff, 0xb7, 0x04, 0xd5, 0x5b, 0x88, 0xa4, 0xaa, 0xe3, 0xc8, - 0xfd, 0xc5, 0x59, 0xd7, 0xab, 0x50, 0xf2, 0x29, 0xf3, 0x0a, 0x3c, 0xbf, 0x0d, 0xfb, 0xf5, 0x71, - 0x5e, 0xaa, 0xa7, 0x0f, 0x04, 0x75, 0x07, 0xa3, 0x64, 0x09, 0xc9, 0x6c, 0x2e, 0x1a, 0x96, 0xba, - 0xf2, 0xce, 0xe5, 0xf3, 0xa7, 0x12, 0x9c, 0xa2, 0x05, 0xe2, 0x63, 0xf6, 0x23, 0xb9, 0x84, 0x46, - 0xea, 0x7e, 0xa4, 0x9e, 0x2b, 0xab, 0xe3, 0x94, 0xa8, 0x88, 0x35, 0x6f, 0x40, 0xb5, 0x1b, 0x78, - 0xef, 0x08, 0xf3, 0xfb, 0x19, 0x58, 0xe2, 0x44, 0x58, 0x06, 0x3c, 0x0e, 0xab, 0x8d, 0x2e, 0x59, - 0xfc, 0xce, 0x00, 0xbc, 0x0e, 0xb0, 0x85, 0x48, 0x22, 0x97, 0xdf, 0x0c, 0xf8, 0x1f, 0x6f, 0x45, - 0x8a, 0xd7, 0x4d, 0x2a, 0x02, 0x64, 0x4d, 0x40, 0x88, 0xfa, 0x49, 0x1f, 0xf7, 0xcd, 0xbe, 0x78, - 0xf7, 0xcd, 0x75, 0x73, 0xdf, 0x65, 0x38, 0xd7, 0x4f, 0x22, 0xdc, 0x44, 0x3f, 0x1e, 0x82, 0x05, - 0x71, 0xff, 0x0f, 0x5e, 0x39, 0x3e, 0x17, 0xfe, 0x7b, 0x15, 0x66, 0x2c, 0xac, 0x25, 0x34, 0x49, - 0x51, 0xdd, 0xe4, 0xd5, 0x49, 0x0b, 0xdf, 0x89, 0x76, 0x3f, 0x75, 0xae, 0xfd, 0xd9, 0x23, 0x5e, - 0xfb, 0xab, 0x70, 0x32, 0x59, 0x22, 0x5c, 0x64, 0xff, 0x26, 0xc1, 0xf9, 0xc7, 0xc8, 0xb5, 0xb6, - 0xdb, 0xb1, 0xc5, 0xfd, 0x4a, 0xca, 0xe7, 0x42, 0x7c, 0xbe, 0x24, 0x32, 0x47, 0x94, 0xc4, 0x05, - 0x58, 0xee, 0xcf, 0x28, 0x97, 0xca, 0xff, 0x64, 0xc8, 0x5d, 0x86, 0xdc, 0xec, 0x56, 0x89, 0x39, - 0xfa, 0xbb, 0x38, 0xca, 0x3d, 0xec, 0xc5, 0x89, 0xa4, 0x06, 0xbc, 0x49, 0x32, 0xe0, 0xf0, 0xbe, - 0xab, 0x97, 0xd9, 0x94, 0xef, 0xe8, 0x6b, 0xa6, 0xfc, 0x3e, 0x4c, 0x8a, 0x3b, 0x9b, 0x79, 0x1c, - 0xdf, 0x96, 0x7d, 0x2a, 0x9d, 0xbd, 0xac, 0xfb, 0xb7, 0x4d, 0xfa, 0x54, 0x42, 0xeb, 0x8f, 0xb9, - 0x34, 0xf5, 0xc7, 0x62, 0x07, 0x9d, 0x15, 0x20, 0x7d, 0x85, 0x0f, 0x1f, 0x4d, 0xe1, 0xf2, 0x35, - 0xa8, 0xc4, 0xc4, 0x23, 0x12, 0xe7, 0x08, 0x7f, 0x93, 0x0a, 0xcb, 0x88, 0xe7, 0x4f, 0xe5, 0x3c, - 0x49, 0x03, 0x3d, 0xb5, 0x2f, 0x72, 0x62, 0x06, 0x2e, 0x32, 0xa3, 0x4a, 0x84, 0xa4, 0xb1, 0x89, - 0xd0, 0x49, 0x65, 0x30, 0x9b, 0x50, 0x8a, 0xb6, 0xd3, 0xa6, 0x37, 0x97, 0x62, 0xa4, 0x7d, 0x56, - 0x56, 0xa1, 0xc8, 0xa2, 0xee, 0x31, 0xce, 0x64, 0x05, 0x23, 0xc4, 0x65, 0x37, 0x03, 0xcc, 0x76, - 0x33, 0xc0, 0x5e, 0x1a, 0xc9, 0xf5, 0xd2, 0xc8, 0xb1, 0x8d, 0x41, 0xb9, 0x04, 0xb5, 0x41, 0x15, - 0xc5, 0x75, 0xfb, 0xc7, 0x12, 0x2c, 0xde, 0x42, 0xd8, 0x70, 0xad, 0xad, 0x63, 0x9d, 0x08, 0xbf, - 0x01, 0x23, 0x69, 0xeb, 0x13, 0xfd, 0x96, 0x55, 0x05, 0x45, 0xe5, 0xf7, 0xb2, 0x70, 0xa6, 0x07, - 0x34, 0x3f, 0xee, 0x7c, 0x13, 0x4a, 0x9d, 0x77, 0x41, 0xc3, 0xb1, 0xb7, 0xad, 0x1d, 0x5e, 0x16, - 0xbd, 0x9c, 0xbc, 0x97, 0x44, 0xf5, 0xaf, 0x52, 0x44, 0xb5, 0x88, 0xc2, 0x03, 0xf2, 0x0e, 0xcc, - 0x26, 0x3c, 0x3f, 0xd2, 0x06, 0x70, 0xc6, 0xf0, 0x4a, 0x8a, 0x45, 0xd8, 0x3b, 0xe7, 0x41, 0xd2, - 0xb0, 0xfc, 0x4d, 0x90, 0x9b, 0xc8, 0x36, 0x2d, 0x7b, 0x47, 0xe3, 0xa5, 0x51, 0x0b, 0xe1, 0x4a, - 0x86, 0x16, 0x5b, 0x2f, 0x76, 0x5f, 0x63, 0x9d, 0xe1, 0x88, 0xfa, 0x06, 0x5d, 0xa1, 0xdc, 0x0c, - 0x0d, 0x5a, 0x08, 0xcb, 0xdf, 0x82, 0x92, 0xa0, 0x4e, 0xcd, 0xdc, 0xa5, 0x6d, 0x5d, 0x84, 0xf6, - 0xd5, 0xbe, 0xb4, 0xc3, 0x46, 0x45, 0x57, 0x28, 0x36, 0x03, 0x53, 0x2e, 0xb2, 0x65, 0x04, 0xd3, - 0x82, 0x7e, 0x38, 0xfd, 0xe7, 0xfa, 0x69, 0x82, 0x2f, 0x12, 0x7b, 0x0e, 0x9e, 0x6c, 0xc6, 0x27, - 0x94, 0x5f, 0xcb, 0x40, 0x45, 0xe5, 0x5f, 0x50, 0x20, 0x1a, 0x49, 0xf1, 0xe3, 0x2b, 0x9f, 0x8b, - 0x74, 0xb5, 0x0d, 0xd3, 0xe1, 0x26, 0xa4, 0xb6, 0x66, 0x79, 0xa8, 0x21, 0x34, 0x78, 0x25, 0x55, - 0x23, 0x52, 0x7b, 0xcd, 0x43, 0x0d, 0x75, 0x72, 0x3f, 0x36, 0x86, 0xe5, 0x6b, 0x30, 0x4c, 0xf3, - 0x0f, 0xe6, 0x99, 0xad, 0xeb, 0x43, 0xcf, 0x2d, 0xdd, 0xd3, 0x6f, 0xd6, 0x9d, 0x2d, 0x95, 0xc3, - 0xcb, 0x77, 0xa0, 0x60, 0xa3, 0x03, 0xda, 0xcf, 0xc3, 0x29, 0xe4, 0x06, 0xa4, 0x30, 0x6e, 0xa3, - 0x03, 0xb5, 0xc5, 0x32, 0x17, 0x56, 0x16, 0x60, 0x2e, 0x41, 0x05, 0x3c, 0xae, 0xfc, 0x3d, 0xbd, - 0x47, 0xf1, 0xd9, 0x27, 0xc1, 0x56, 0x27, 0xa1, 0x25, 0x2d, 0xd6, 0x4e, 0xc5, 0x9c, 0xf5, 0x5a, - 0xa2, 0x84, 0x02, 0xdf, 0xb1, 0x04, 0x55, 0x11, 0x2a, 0x2f, 0x44, 0x5a, 0xaa, 0x96, 0xa0, 0xe0, - 0xa2, 0x86, 0xe3, 0x21, 0xcd, 0xa8, 0xb7, 0xb0, 0x87, 0x5c, 0xaa, 0xdf, 0x51, 0x75, 0x82, 0x8d, - 0xae, 0xb2, 0xc1, 0x98, 0xb5, 0x64, 0x62, 0xd6, 0xa2, 0x2c, 0x92, 0xab, 0x56, 0x32, 0x2f, 0x9c, - 0xdd, 0x3f, 0x94, 0x60, 0x66, 0xa3, 0x6d, 0x1b, 0x1b, 0xbb, 0xba, 0x6b, 0xf2, 0x4e, 0x2c, 0xce, - 0xe7, 0x12, 0x14, 0xb0, 0xd3, 0x72, 0x8d, 0xce, 0x36, 0x98, 0x3d, 0x4e, 0xb0, 0x51, 0xb1, 0x8d, - 0x39, 0xc8, 0x63, 0x82, 0x2c, 0x7a, 0x49, 0x72, 0xea, 0x08, 0xfd, 0xbd, 0x66, 0xca, 0x37, 0x60, - 0x8c, 0xb5, 0x84, 0xb1, 0x27, 0xc3, 0xcc, 0x80, 0x4f, 0x86, 0xc0, 0x90, 0xc8, 0xb0, 0x32, 0x07, - 0xb3, 0xb1, 0xed, 0xf1, 0xad, 0xff, 0x24, 0x07, 0x93, 0x64, 0x4e, 0x44, 0x8e, 0x14, 0x5e, 0x74, - 0x1a, 0xc6, 0x7c, 0x15, 0xf2, 0x6d, 0x8f, 0xaa, 0x20, 0x86, 0xd6, 0xcc, 0xc0, 0x0d, 0x34, 0x13, - 0xfc, 0x5a, 0xa1, 0x02, 0x23, 0x22, 0x21, 0xb2, 0x2c, 0x2a, 0x7e, 0x76, 0x79, 0x0e, 0xcf, 0x75, - 0x79, 0x0e, 0x8f, 0xb7, 0x61, 0x0c, 0x1f, 0xad, 0x0d, 0x23, 0xa9, 0xe1, 0x66, 0x24, 0xb1, 0xe1, - 0x26, 0xfa, 0x60, 0x9c, 0x3f, 0xca, 0x83, 0xf1, 0x3a, 0xef, 0x0e, 0xed, 0x3c, 0xe4, 0x50, 0x5a, - 0xa3, 0x03, 0xd2, 0x2a, 0x13, 0x64, 0xff, 0x01, 0x86, 0x52, 0xbc, 0x0e, 0x23, 0xe2, 0xdd, 0x17, - 0x06, 0x7c, 0xf7, 0x15, 0x08, 0xc1, 0xe7, 0xeb, 0xb1, 0xf0, 0xf3, 0xf5, 0x2a, 0x8c, 0xb3, 0xde, - 0x41, 0xfe, 0x41, 0xce, 0xf8, 0x80, 0x1f, 0xe4, 0x8c, 0xd1, 0x96, 0x42, 0xfe, 0x2d, 0xce, 0x25, - 0xa0, 0xdf, 0xd2, 0xd0, 0x74, 0x80, 0x5c, 0xcd, 0x32, 0x91, 0xed, 0x59, 0x5e, 0x9b, 0xb6, 0xba, - 0x8c, 0xaa, 0x32, 0x99, 0x7b, 0x42, 0xa7, 0xd6, 0xf8, 0x8c, 0xfc, 0x04, 0x8a, 0x91, 0x10, 0xca, - 0xbb, 0x38, 0x6b, 0xe9, 0x82, 0xa7, 0x5a, 0x08, 0x07, 0x4e, 0x65, 0x06, 0xa6, 0xc2, 0x96, 0xce, - 0x5d, 0xe0, 0x3b, 0x12, 0x2c, 0x88, 0xf3, 0xc5, 0x4b, 0x6e, 0xd8, 0x56, 0xfe, 0x4b, 0x82, 0x93, - 0xc9, 0x7b, 0xe1, 0xc7, 0x9c, 0x5d, 0x98, 0x34, 0x74, 0x63, 0x17, 0x85, 0x3f, 0xe1, 0x3b, 0x76, - 0xf0, 0x2c, 0x53, 0xa2, 0xc1, 0x21, 0xd9, 0x86, 0x19, 0x53, 0xf7, 0xf4, 0x2d, 0x1d, 0x47, 0x17, - 0x1b, 0x3a, 0xe6, 0x62, 0x53, 0x82, 0x6e, 0x70, 0x54, 0xf9, 0x47, 0x09, 0xe6, 0x05, 0xeb, 0x5c, - 0x65, 0xf7, 0x1c, 0x1c, 0x7c, 0x1c, 0xdd, 0x75, 0xb0, 0xa7, 0xe9, 0xa6, 0xe9, 0x22, 0x8c, 0x85, - 0x16, 0xc8, 0xd8, 0x0d, 0x36, 0xd4, 0x2b, 0x88, 0xf6, 0x0f, 0xf3, 0x5d, 0x0e, 0x05, 0xd9, 0xe3, - 0x1f, 0x0a, 0x94, 0x7f, 0x09, 0x18, 0x58, 0x88, 0x33, 0xae, 0xd3, 0x57, 0x60, 0x82, 0xee, 0x13, - 0x6b, 0x76, 0xab, 0xb1, 0xc5, 0x53, 0x44, 0x4e, 0x1d, 0x67, 0x83, 0x8f, 0xe8, 0x98, 0xbc, 0x00, - 0xa3, 0x82, 0x39, 0xf6, 0xf8, 0x9e, 0x53, 0xf3, 0x9c, 0x3b, 0x2c, 0x7f, 0x00, 0xc5, 0x0e, 0x7b, - 0x54, 0x95, 0x3d, 0xbf, 0x4b, 0xf4, 0x61, 0x09, 0x0b, 0x7e, 0xff, 0xc5, 0x2a, 0xc1, 0xa3, 0x87, - 0xae, 0x82, 0x1d, 0x1a, 0xa3, 0x31, 0x82, 0x8b, 0x9d, 0x35, 0x17, 0x89, 0x9f, 0xf7, 0xb3, 0xf9, - 0x6c, 0x29, 0xa7, 0xd4, 0xa0, 0xbc, 0x5a, 0x77, 0x30, 0xa2, 0x09, 0x46, 0x28, 0x2c, 0xa8, 0x0d, - 0x29, 0xa4, 0x0d, 0x65, 0x0a, 0xe4, 0x20, 0x3c, 0xf7, 0xc3, 0xd7, 0xa1, 0x78, 0x17, 0x79, 0x83, - 0xd2, 0xf8, 0x10, 0x4a, 0x1d, 0x68, 0x2e, 0xc8, 0x07, 0x00, 0x1c, 0x9c, 0x1c, 0xcc, 0x99, 0x4f, - 0x5c, 0x1c, 0xc4, 0x4c, 0x29, 0x19, 0xca, 0x3a, 0x13, 0x32, 0xf9, 0x53, 0xf9, 0x27, 0x09, 0xca, - 0xec, 0x31, 0x23, 0x58, 0x5f, 0xeb, 0xbe, 0x25, 0xf9, 0x0e, 0xe4, 0xc9, 0x19, 0x61, 0x87, 0x84, - 0xac, 0x21, 0xda, 0xbe, 0x7d, 0xa1, 0x77, 0x73, 0x38, 0x7b, 0x86, 0x64, 0x18, 0xaa, 0x8f, 0x1b, - 0xec, 0xf3, 0xca, 0x84, 0xfa, 0xbc, 0xd6, 0xa0, 0xb8, 0x6f, 0x61, 0x6b, 0xcb, 0xaa, 0xd3, 0x3e, - 0x8c, 0x34, 0x1d, 0x44, 0x85, 0x0e, 0x22, 0x3d, 0x12, 0x4c, 0x81, 0x1c, 0xe4, 0x4d, 0x14, 0x17, - 0x25, 0x38, 0x75, 0x17, 0x79, 0x6a, 0xe7, 0xeb, 0xe4, 0x87, 0xec, 0xcb, 0x64, 0xff, 0x3c, 0xf3, - 0x00, 0x86, 0x69, 0x5f, 0x24, 0x71, 0xc0, 0x4c, 0x57, 0x03, 0x0b, 0x7c, 0xde, 0xcc, 0x8a, 0xbd, - 0xfe, 0x4f, 0xda, 0x41, 0xa9, 0x72, 0x1a, 0xc4, 0x2d, 0xf9, 0xb1, 0x88, 0xf6, 0x07, 0xf1, 0x33, - 0xc4, 0x18, 0x1f, 0x23, 0x96, 0xa9, 0x7c, 0x7f, 0x08, 0xaa, 0xdd, 0xb6, 0xc4, 0xd5, 0xfe, 0x2b, - 0x50, 0x60, 0x2a, 0xe1, 0x9f, 0x51, 0x8b, 0xbd, 0xbd, 0x37, 0x60, 0x3f, 0x4c, 0x6f, 0xf2, 0xcc, - 0x38, 0xc4, 0x28, 0xeb, 0x85, 0x64, 0xfe, 0x2a, 0xc6, 0xe6, 0xdb, 0x20, 0xc7, 0x81, 0x82, 0x6d, - 0x8d, 0x39, 0xd6, 0xd6, 0xf8, 0x30, 0xdc, 0xd6, 0xf8, 0x46, 0x4a, 0xd9, 0xf9, 0x3b, 0xeb, 0x74, - 0x3a, 0x2a, 0x1f, 0xc1, 0xe2, 0x5d, 0xe4, 0xdd, 0x7a, 0xf0, 0x4e, 0x0f, 0x9d, 0x3d, 0xe6, 0xdf, - 0x97, 0x10, 0xaf, 0x10, 0xb2, 0x49, 0xbb, 0xb6, 0x7f, 0x21, 0xa3, 0x9f, 0x9c, 0x90, 0xbf, 0xb0, - 0xf2, 0x1b, 0x12, 0x9c, 0xe9, 0xb1, 0x38, 0xd7, 0xce, 0x87, 0x50, 0x0e, 0x90, 0xe5, 0xdd, 0x43, - 0x52, 0xf4, 0xd2, 0x39, 0xf0, 0x26, 0xd4, 0x92, 0x1b, 0x1e, 0xc0, 0xca, 0x77, 0x25, 0x98, 0xa2, - 0x2d, 0xa0, 0x22, 0x1a, 0xa7, 0xc8, 0xdc, 0x5f, 0x8f, 0x56, 0x2e, 0xbe, 0xdc, 0xb7, 0x72, 0x91, - 0xb4, 0x54, 0xa7, 0x5a, 0xb1, 0x07, 0xd3, 0x11, 0x00, 0x2e, 0x07, 0x15, 0xf2, 0x91, 0x7e, 0xad, - 0xaf, 0xa4, 0x5d, 0x8a, 0x37, 0x4d, 0xf9, 0x74, 0x94, 0xdf, 0x95, 0x60, 0x4a, 0x45, 0x7a, 0xb3, - 0x59, 0x67, 0x15, 0x46, 0x9c, 0x82, 0xf3, 0x8d, 0x28, 0xe7, 0xc9, 0x4d, 0xdb, 0xc1, 0x2f, 0xf9, - 0x99, 0x3a, 0xe2, 0xcb, 0x75, 0xb8, 0x9f, 0x85, 0xe9, 0x08, 0x00, 0xdf, 0xe9, 0x9f, 0x0d, 0xc1, - 0x34, 0xb3, 0x95, 0xa8, 0x75, 0xde, 0x86, 0xac, 0xdf, 0x99, 0x5f, 0x08, 0x96, 0x08, 0x92, 0x22, - 0xe6, 0x2d, 0xa4, 0x9b, 0x0f, 0x90, 0xe7, 0x21, 0x97, 0xf6, 0x91, 0xd1, 0x9e, 0x43, 0x8a, 0xde, - 0x2b, 0xf9, 0xc7, 0xef, 0x60, 0x99, 0xa4, 0x3b, 0xd8, 0x1b, 0x50, 0xb1, 0x6c, 0x02, 0x61, 0xed, - 0x23, 0x0d, 0xd9, 0x7e, 0x38, 0xe9, 0x94, 0xfb, 0xa6, 0xfd, 0xf9, 0xdb, 0xb6, 0x70, 0xf6, 0x35, - 0x53, 0xbe, 0x00, 0xe5, 0x86, 0x7e, 0x68, 0x35, 0x5a, 0x0d, 0xad, 0x49, 0xe0, 0xb1, 0xf5, 0x11, - 0xfb, 0x0c, 0x3f, 0xa7, 0x16, 0xf9, 0xc4, 0xba, 0xbe, 0x83, 0x36, 0xac, 0x8f, 0x90, 0x7c, 0x0e, - 0x8a, 0xb4, 0x65, 0x9f, 0x02, 0xb2, 0x0e, 0xf3, 0x61, 0xda, 0x61, 0x4e, 0x3b, 0xf9, 0x09, 0x18, - 0xfb, 0xa4, 0xee, 0xdf, 0xd9, 0x17, 0xd9, 0x21, 0x79, 0x71, 0x43, 0x7a, 0x4e, 0x02, 0x4b, 0xf4, - 0xcb, 0xa1, 0xe7, 0xe8, 0x97, 0x49, 0xbc, 0x66, 0x92, 0x78, 0xfd, 0x67, 0x09, 0x66, 0xd7, 0x5b, - 0xee, 0x0e, 0xfa, 0x79, 0xb4, 0x0e, 0x65, 0x1e, 0x2a, 0x71, 0xe6, 0x44, 0x9b, 0xd8, 0x10, 0xcc, - 0x3e, 0x44, 0x3f, 0xa7, 0x9c, 0xbf, 0x10, 0xbf, 0xb8, 0x09, 0x95, 0xb8, 0xc0, 0xb8, 0x63, 0x24, - 0xd0, 0x90, 0x92, 0x68, 0x7c, 0x9f, 0x7e, 0x91, 0xb6, 0xed, 0x22, 0xbc, 0x1b, 0x2c, 0x2b, 0xa6, - 0x09, 0x9e, 0xef, 0x47, 0x83, 0xe7, 0x2f, 0x0e, 0x18, 0x3c, 0xbb, 0xae, 0xda, 0x89, 0xa1, 0xf4, - 0x23, 0xb5, 0x24, 0x38, 0x6e, 0x34, 0xdf, 0x93, 0xe0, 0xc2, 0x5d, 0x64, 0x23, 0x57, 0xf7, 0xd0, - 0x03, 0x1d, 0x7b, 0xe2, 0xbe, 0x1b, 0x71, 0xbf, 0x97, 0x71, 0x7d, 0xbd, 0x08, 0xaf, 0x0d, 0xb4, - 0x33, 0xce, 0xc9, 0x1d, 0x58, 0x08, 0x9f, 0xbd, 0xc2, 0xb5, 0xb3, 0xf3, 0x50, 0x0c, 0x97, 0xf0, - 0xd8, 0xb9, 0x61, 0x54, 0x2d, 0x84, 0x6a, 0x78, 0x58, 0x69, 0xc1, 0xc9, 0x64, 0x3a, 0xdc, 0x30, - 0xde, 0x85, 0x61, 0x76, 0x97, 0xe2, 0xe7, 0x8e, 0x37, 0x07, 0x3c, 0x18, 0xf2, 0xdb, 0x45, 0x94, - 0x2c, 0x27, 0xa6, 0xfc, 0xf5, 0x30, 0xcc, 0x24, 0x83, 0xf4, 0xba, 0x25, 0x7c, 0x19, 0x66, 0x1b, - 0xfa, 0xa1, 0x16, 0x8d, 0xbd, 0x9d, 0xaf, 0xc8, 0xa6, 0x1a, 0xfa, 0x61, 0xf4, 0xe4, 0x65, 0xca, - 0x0f, 0xa0, 0xc4, 0x28, 0xd6, 0x1d, 0x43, 0xaf, 0x0f, 0x5a, 0x0b, 0x1c, 0x26, 0x87, 0xff, 0x8a, - 0xa4, 0xb2, 0x03, 0xf2, 0x03, 0x82, 0x4a, 0x2b, 0x46, 0x1f, 0xc5, 0x45, 0xcb, 0xde, 0x01, 0xde, - 0x39, 0x96, 0x68, 0x6a, 0x6a, 0x48, 0x31, 0xec, 0xb0, 0x1c, 0xd1, 0x96, 0xfc, 0x9b, 0x12, 0x4c, - 0xee, 0xea, 0xb6, 0xe9, 0xec, 0xf3, 0x63, 0x3f, 0x35, 0x43, 0x72, 0xb5, 0x4c, 0xf3, 0xf5, 0x52, - 0x97, 0x0d, 0xdc, 0xe3, 0x84, 0xfd, 0x5b, 0x2d, 0xdf, 0x84, 0xbc, 0x1b, 0x9b, 0x90, 0x9b, 0x70, - 0x36, 0x51, 0x13, 0xd1, 0x3b, 0xd6, 0xa0, 0x65, 0xc5, 0xc5, 0xb8, 0xe2, 0x1e, 0x87, 0x6e, 0x5d, - 0xf3, 0xdf, 0x95, 0x60, 0x32, 0x41, 0x44, 0x09, 0x5f, 0x40, 0x7d, 0x10, 0xbe, 0x2a, 0xdc, 0x3d, - 0x96, 0x54, 0xd6, 0x91, 0xcb, 0xd7, 0x0b, 0x5c, 0x1d, 0xe6, 0xbf, 0x2d, 0xc1, 0x6c, 0x17, 0x71, - 0x25, 0x6c, 0x48, 0x0d, 0x6f, 0xe8, 0x6b, 0x03, 0x6e, 0x28, 0xb6, 0x00, 0xbd, 0x44, 0x04, 0x2e, - 0x30, 0xef, 0xc1, 0x74, 0x22, 0x8c, 0xfc, 0x36, 0x9c, 0xf4, 0xad, 0x24, 0xc9, 0x59, 0x24, 0xea, - 0x2c, 0x73, 0x02, 0x26, 0xe6, 0x31, 0xca, 0x9f, 0x48, 0xb0, 0xd8, 0x4f, 0x1e, 0xb2, 0x02, 0x13, - 0xba, 0xb1, 0x87, 0xcc, 0x08, 0xd9, 0x31, 0x3a, 0xc8, 0x5d, 0xef, 0x03, 0x98, 0x0f, 0xc0, 0x44, - 0xad, 0x63, 0xd0, 0x8f, 0x86, 0x66, 0x7d, 0x92, 0x61, 0xa3, 0x50, 0x7e, 0x4b, 0x82, 0x79, 0x15, - 0x6d, 0xb5, 0xac, 0xba, 0xf9, 0xb2, 0xcb, 0x8f, 0xa7, 0x48, 0x62, 0x4c, 0xd8, 0x09, 0x8f, 0xd7, - 0x7f, 0x39, 0x04, 0x4b, 0xe1, 0x16, 0xba, 0x0e, 0x2b, 0xec, 0x6d, 0xf9, 0x65, 0xfc, 0x93, 0x8b, - 0x75, 0x98, 0x0c, 0x3e, 0x25, 0xf1, 0x7f, 0x49, 0x30, 0xf0, 0x43, 0x49, 0x39, 0xf0, 0x6e, 0xc4, - 0xfe, 0xff, 0x40, 0x88, 0x22, 0x6d, 0x24, 0x4c, 0x57, 0x6b, 0xf1, 0x29, 0xd2, 0x22, 0x17, 0xd5, - 0xf1, 0x32, 0x9c, 0xeb, 0x27, 0x38, 0x2e, 0xe3, 0xdf, 0x91, 0x60, 0xfa, 0xdd, 0xa6, 0x19, 0x78, - 0x6b, 0x4a, 0x21, 0xd3, 0xf5, 0xe8, 0xb1, 0xa4, 0xff, 0x15, 0x33, 0x71, 0xad, 0xce, 0x61, 0xa4, - 0x01, 0x33, 0x51, 0x08, 0x9e, 0x54, 0x37, 0x62, 0xf7, 0xd9, 0x37, 0x52, 0x2f, 0x16, 0xbd, 0xd0, - 0xde, 0x6c, 0x7e, 0xf2, 0x69, 0xf5, 0xc4, 0x8f, 0x3e, 0xad, 0x9e, 0xf8, 0xe9, 0xa7, 0x55, 0xe9, - 0x57, 0x9f, 0x55, 0xa5, 0x1f, 0x3c, 0xab, 0x4a, 0x7f, 0xf7, 0xac, 0x2a, 0x7d, 0xf2, 0xac, 0x2a, - 0xfd, 0xeb, 0xb3, 0xaa, 0xf4, 0x93, 0x67, 0xd5, 0x13, 0x3f, 0x7d, 0x56, 0x95, 0x3e, 0xfe, 0xac, - 0x7a, 0xe2, 0x93, 0xcf, 0xaa, 0x27, 0x7e, 0xf4, 0x59, 0xf5, 0xc4, 0xfb, 0xd7, 0x77, 0x9c, 0xce, - 0xd2, 0x96, 0xd3, 0xf3, 0x7f, 0x4f, 0xfe, 0x42, 0x78, 0x64, 0x6b, 0x98, 0xaa, 0xf1, 0xea, 0xff, - 0x06, 0x00, 0x00, 0xff, 0xff, 0x21, 0xbd, 0x83, 0xa2, 0xba, 0x52, 0x00, 0x00, + 0x12, 0x57, 0x5e, 0x0d, 0xf5, 0xb1, 0xbd, 0xb2, 0xe2, 0xdd, 0x8d, 0x44, 0xfd, 0x28, 0x48, 0x32, + 0xb7, 0xc9, 0x95, 0x16, 0x6b, 0xaf, 0x7b, 0x9b, 0xd3, 0x45, 0xb2, 0xc3, 0x99, 0xee, 0x51, 0x57, + 0x0f, 0xc9, 0xd9, 0x1c, 0xf2, 0x31, 0x12, 0xc4, 0x4e, 0x90, 0x2c, 0x90, 0x8b, 0x11, 0x38, 0x40, + 0x10, 0x20, 0x88, 0x2f, 0x41, 0x0e, 0x39, 0x04, 0x3e, 0xe4, 0x92, 0x00, 0x41, 0x90, 0xd3, 0x22, + 0x97, 0x18, 0x09, 0x10, 0x67, 0xb5, 0x87, 0x6c, 0x90, 0x1c, 0x7c, 0x0c, 0x82, 0x1c, 0x82, 0xfa, + 0xf5, 0x7f, 0x3e, 0x4d, 0x4a, 0xd1, 0xda, 0xd9, 0x1b, 0xa7, 0xea, 0xbd, 0x57, 0xf5, 0xfe, 0x55, + 0xaf, 0x5e, 0x13, 0xbe, 0xee, 0xa2, 0x66, 0xcb, 0x76, 0xf4, 0xc6, 0x0a, 0x46, 0xce, 0x3e, 0x72, + 0x56, 0xf4, 0x96, 0xb9, 0xb2, 0x6b, 0x62, 0xd7, 0x76, 0x3a, 0x64, 0xc4, 0xac, 0xa3, 0x95, 0xfd, + 0xcb, 0x2b, 0x0e, 0x7a, 0xda, 0x46, 0xd8, 0xd5, 0x1c, 0x84, 0x5b, 0xb6, 0x85, 0x51, 0xad, 0xe5, + 0xd8, 0xae, 0x2d, 0x2f, 0x09, 0xec, 0x1a, 0xc3, 0xae, 0xe9, 0x2d, 0xb3, 0x16, 0xc6, 0xae, 0xed, + 0x5f, 0x9e, 0xaf, 0xee, 0xd8, 0xf6, 0x4e, 0x03, 0xad, 0x50, 0xa4, 0xad, 0xf6, 0xf6, 0x8a, 0xd1, + 0x76, 0x74, 0xd7, 0xb4, 0x2d, 0x46, 0x66, 0xfe, 0x74, 0x74, 0xde, 0x35, 0x9b, 0x08, 0xbb, 0x7a, + 0xb3, 0xc5, 0x01, 0xce, 0x18, 0xa8, 0x85, 0x2c, 0x03, 0x59, 0x75, 0x13, 0xe1, 0x95, 0x1d, 0x7b, + 0xc7, 0xa6, 0xe3, 0xf4, 0x2f, 0x0e, 0x72, 0xd6, 0x63, 0x84, 0x70, 0x50, 0xb7, 0x9b, 0x4d, 0xdb, + 0x22, 0x3b, 0x6f, 0x22, 0x8c, 0xf5, 0x1d, 0xbe, 0xe1, 0xf9, 0xa5, 0x10, 0x14, 0xdf, 0x69, 0x1c, + 0xec, 0x7c, 0x08, 0xcc, 0xd5, 0xf1, 0xde, 0xd3, 0x36, 0x6a, 0xa3, 0x38, 0x60, 0x78, 0x55, 0x64, + 0xb5, 0x9b, 0x98, 0x00, 0x1d, 0xd8, 0xce, 0xde, 0x76, 0xc3, 0x3e, 0xe0, 0x50, 0xe7, 0x42, 0x50, + 0x62, 0x32, 0x4e, 0xed, 0x95, 0x10, 0xdc, 0xd3, 0x36, 0x4a, 0xda, 0x5b, 0x98, 0x85, 0x6d, 0xdd, + 0x6c, 0xb4, 0x9d, 0x84, 0x9d, 0x5d, 0x48, 0x52, 0x6c, 0xbd, 0x61, 0xd7, 0xf7, 0xe2, 0xb0, 0xaf, + 0xf5, 0x30, 0x82, 0x38, 0xf4, 0xab, 0x49, 0xd0, 0x1e, 0xeb, 0x4c, 0xf2, 0x1c, 0xf4, 0x4b, 0x3d, + 0x41, 0x23, 0x52, 0x3a, 0xdf, 0x13, 0x98, 0x28, 0x81, 0x03, 0x5e, 0x4c, 0x02, 0xec, 0x2e, 0xd5, + 0x5a, 0x12, 0xb8, 0xa5, 0x37, 0x11, 0x6e, 0xe9, 0xf5, 0x04, 0xc9, 0x5d, 0x4a, 0x82, 0x77, 0x50, + 0xab, 0x61, 0xd6, 0xa9, 0xd1, 0xc6, 0x31, 0xae, 0x26, 0x61, 0xb4, 0x90, 0x83, 0x4d, 0xec, 0x22, + 0x8b, 0xad, 0x81, 0x0e, 0x51, 0xbd, 0x4d, 0xd0, 0x31, 0x47, 0x7a, 0x6b, 0x00, 0x24, 0xc1, 0x94, + 0xd6, 0x6c, 0xbb, 0xfa, 0x56, 0x03, 0x69, 0xd8, 0xd5, 0x5d, 0xb1, 0xea, 0x57, 0x13, 0xad, 0xaa, + 0xaf, 0xd3, 0xce, 0x5f, 0x4f, 0x5a, 0x58, 0x37, 0x9a, 0xa6, 0xd5, 0x17, 0x57, 0xf9, 0xed, 0x61, + 0x38, 0xb5, 0xe1, 0xea, 0x8e, 0xfb, 0x84, 0x2f, 0x77, 0x5b, 0xb0, 0xa5, 0x32, 0x04, 0xf9, 0x0c, + 0x8c, 0x7b, 0xb2, 0xd5, 0x4c, 0xa3, 0x22, 0x2d, 0x4a, 0xcb, 0xa3, 0xea, 0x98, 0x37, 0xb6, 0x66, + 0xc8, 0x75, 0x98, 0xc0, 0x84, 0x86, 0xc6, 0x17, 0xa9, 0x0c, 0x2d, 0x4a, 0xcb, 0x63, 0x57, 0xde, + 0xf4, 0x14, 0x45, 0xc3, 0x48, 0x84, 0xa1, 0xda, 0xfe, 0xe5, 0x5a, 0xcf, 0x95, 0xd5, 0x71, 0x4a, + 0x54, 0xec, 0x63, 0x17, 0xa6, 0x5b, 0xba, 0x83, 0x2c, 0x57, 0xf3, 0x24, 0xaf, 0x99, 0xd6, 0xb6, + 0x5d, 0xc9, 0xd0, 0xc5, 0xbe, 0x5c, 0x4b, 0x0a, 0x5d, 0x9e, 0x45, 0xee, 0x5f, 0xae, 0xad, 0x53, + 0x6c, 0x6f, 0x95, 0x35, 0x6b, 0xdb, 0x56, 0x27, 0x5b, 0xf1, 0x41, 0xb9, 0x02, 0x23, 0xba, 0x4b, + 0xa8, 0xb9, 0x95, 0xec, 0xa2, 0xb4, 0x9c, 0x53, 0xc5, 0x4f, 0xb9, 0x09, 0x8a, 0xa7, 0x41, 0x7f, + 0x17, 0xe8, 0xb0, 0x65, 0xb2, 0xf0, 0xa7, 0x91, 0x38, 0x57, 0xc9, 0xd1, 0x0d, 0xcd, 0xd7, 0x58, + 0x10, 0xac, 0x89, 0x20, 0x58, 0xdb, 0x14, 0x41, 0xf0, 0x66, 0xf6, 0xa3, 0x9f, 0x9c, 0x96, 0xd4, + 0xd3, 0x07, 0x51, 0xce, 0x6f, 0x7b, 0x94, 0x08, 0xac, 0xbc, 0x0b, 0x73, 0x75, 0xdb, 0x72, 0x4d, + 0xab, 0x8d, 0x34, 0x1d, 0x6b, 0x16, 0x3a, 0xd0, 0x4c, 0xcb, 0x74, 0x4d, 0xdd, 0xb5, 0x9d, 0xca, + 0xf0, 0xa2, 0xb4, 0x5c, 0xb8, 0x72, 0x31, 0x2c, 0x63, 0xea, 0x5d, 0x84, 0xd9, 0x55, 0x8e, 0x77, + 0x03, 0x3f, 0x42, 0x07, 0x6b, 0x02, 0x49, 0x9d, 0xa9, 0x27, 0x8e, 0xcb, 0x0f, 0xa1, 0x2c, 0x66, + 0x0c, 0x8d, 0x87, 0xa0, 0xca, 0x08, 0xe5, 0x63, 0x31, 0xbc, 0x02, 0x9f, 0x24, 0x6b, 0xdc, 0x61, + 0x7f, 0xaa, 0x25, 0x0f, 0x95, 0x8f, 0xc8, 0x8f, 0x61, 0xa6, 0xa1, 0x63, 0x57, 0xab, 0xdb, 0xcd, + 0x56, 0x03, 0x51, 0xc9, 0x38, 0x08, 0xb7, 0x1b, 0x6e, 0x25, 0x9f, 0x44, 0x93, 0x87, 0x18, 0xaa, + 0xa3, 0x4e, 0xc3, 0xd6, 0x0d, 0xac, 0x4e, 0x11, 0xfc, 0x55, 0x0f, 0x5d, 0xa5, 0xd8, 0xf2, 0xb7, + 0x61, 0x61, 0xdb, 0x74, 0xb0, 0xab, 0x79, 0x5a, 0x20, 0x51, 0x44, 0xdb, 0xd2, 0xeb, 0x7b, 0xf6, + 0xf6, 0x76, 0x65, 0x94, 0x12, 0x9f, 0x8b, 0x09, 0xfe, 0x16, 0xcf, 0x4e, 0x37, 0xb3, 0xdf, 0x27, + 0x72, 0xaf, 0x50, 0x1a, 0xc2, 0xec, 0x36, 0x75, 0xbc, 0x77, 0x93, 0x11, 0x50, 0x3e, 0x93, 0xa0, + 0xda, 0xcd, 0x26, 0x99, 0xdb, 0xc8, 0xd3, 0x30, 0xec, 0xb4, 0x2d, 0xdf, 0x11, 0x72, 0x4e, 0xdb, + 0x5a, 0x33, 0xe4, 0xb7, 0x20, 0x47, 0x63, 0x31, 0x37, 0xfd, 0x57, 0x13, 0xad, 0x91, 0x42, 0x10, + 0x36, 0x1f, 0xa3, 0xba, 0x6b, 0x3b, 0xab, 0xe4, 0xa7, 0xca, 0xf0, 0x64, 0x0b, 0x26, 0x91, 0xbe, + 0x83, 0x9c, 0x30, 0x6b, 0xdc, 0xb8, 0xfb, 0x7b, 0xd2, 0xba, 0xdd, 0x68, 0x04, 0x39, 0x7a, 0x9b, + 0xa4, 0x37, 0xb1, 0x69, 0xb5, 0x4c, 0x49, 0x07, 0xe7, 0x95, 0xff, 0x90, 0x60, 0xe6, 0x2e, 0x72, + 0x1f, 0xb2, 0x38, 0xb4, 0x41, 0xc2, 0x50, 0x0a, 0x8f, 0xbf, 0x0b, 0xa3, 0x9e, 0xfd, 0xc7, 0x59, + 0x0e, 0xeb, 0x34, 0x2e, 0x4b, 0x1f, 0x57, 0xbe, 0x0a, 0x33, 0xe8, 0xb0, 0x85, 0xea, 0x2e, 0x32, + 0x34, 0x0b, 0x1d, 0xba, 0x1a, 0xda, 0x27, 0x2e, 0x6e, 0x1a, 0x94, 0xf3, 0x8c, 0x3a, 0x29, 0x66, + 0x1f, 0xa1, 0x43, 0xf7, 0x36, 0x99, 0x5b, 0x33, 0xe4, 0x4b, 0x30, 0x55, 0x6f, 0x3b, 0x34, 0x16, + 0x6c, 0x39, 0xba, 0x55, 0xdf, 0xd5, 0x5c, 0x7b, 0x0f, 0x59, 0xd4, 0x5b, 0xc7, 0x55, 0x99, 0xcf, + 0xdd, 0xa4, 0x53, 0x9b, 0x64, 0x46, 0xf9, 0x49, 0x1e, 0x66, 0x63, 0xdc, 0x72, 0x8d, 0x86, 0x78, + 0x91, 0x8e, 0xc1, 0xcb, 0x1a, 0x4c, 0xf8, 0xca, 0xeb, 0xb4, 0x10, 0x17, 0xcc, 0xd9, 0x7e, 0xc4, + 0x36, 0x3b, 0x2d, 0xa4, 0x8e, 0x1f, 0x04, 0x7e, 0xc9, 0x0a, 0x4c, 0x24, 0x49, 0x63, 0xcc, 0x0a, + 0x48, 0xe1, 0x6b, 0x30, 0xd7, 0x72, 0xd0, 0xbe, 0x69, 0xb7, 0xb1, 0x46, 0x23, 0x25, 0x32, 0x7c, + 0xf8, 0x2c, 0x85, 0x9f, 0x11, 0x00, 0x1b, 0x6c, 0x5e, 0xa0, 0x5e, 0x84, 0x49, 0xea, 0x9f, 0xcc, + 0x99, 0x3c, 0xa4, 0x1c, 0x45, 0x2a, 0x91, 0xa9, 0x3b, 0x64, 0x46, 0x80, 0xaf, 0x02, 0x50, 0x3f, + 0xa3, 0x67, 0x26, 0x1a, 0x78, 0x62, 0x5c, 0x79, 0x47, 0x2a, 0xc2, 0x98, 0x6f, 0x80, 0xa3, 0xae, + 0xf8, 0x53, 0x5e, 0x87, 0x32, 0x76, 0xcd, 0xfa, 0x5e, 0x47, 0x0b, 0xd0, 0x1a, 0x49, 0x41, 0xab, + 0xc8, 0xd0, 0xbd, 0x01, 0xf9, 0x97, 0xe1, 0x4b, 0x31, 0x8a, 0x1a, 0xae, 0xef, 0x22, 0xa3, 0xdd, + 0x40, 0x9a, 0x6b, 0x33, 0xa9, 0xd0, 0x98, 0x6c, 0xb7, 0xdd, 0xca, 0xd8, 0x60, 0xd1, 0x61, 0x29, + 0xb2, 0xcc, 0x06, 0x27, 0xb8, 0x69, 0x53, 0x21, 0x6e, 0x32, 0x6a, 0x5d, 0x6d, 0x70, 0xa2, 0x9b, + 0x0d, 0xca, 0xdf, 0x84, 0x82, 0x67, 0x1e, 0x34, 0xed, 0x57, 0x8a, 0x34, 0x84, 0x27, 0x67, 0x2e, + 0x2f, 0x92, 0xc7, 0x4c, 0x8e, 0x59, 0xaf, 0x67, 0x6a, 0xf4, 0xa7, 0xfc, 0x04, 0x8a, 0x21, 0xe2, + 0x6d, 0x5c, 0x29, 0x51, 0xea, 0xb5, 0x2e, 0x09, 0x22, 0x91, 0x6c, 0x1b, 0xab, 0x85, 0x20, 0xdd, + 0x36, 0x96, 0xdf, 0x87, 0xf2, 0x3e, 0x39, 0xc3, 0xd8, 0x96, 0xc6, 0x0e, 0x90, 0x26, 0xc2, 0x95, + 0x32, 0x15, 0xe5, 0xa5, 0x5a, 0x8f, 0xdb, 0x02, 0x0b, 0x73, 0x14, 0xf1, 0x9e, 0xc0, 0x53, 0x4b, + 0xfb, 0x91, 0x11, 0xf9, 0x4d, 0x38, 0x69, 0x12, 0xf3, 0x8d, 0xaa, 0x11, 0x59, 0xc4, 0x51, 0x8d, + 0x8a, 0xbc, 0x28, 0x2d, 0xe7, 0xd5, 0x8a, 0x89, 0x37, 0xc2, 0x5a, 0xb9, 0xcd, 0xe6, 0xe5, 0x2f, + 0xc3, 0x6c, 0xcc, 0x92, 0xdd, 0x43, 0x1a, 0x9f, 0x27, 0x59, 0x00, 0x09, 0x5b, 0xf3, 0xe6, 0x21, + 0x89, 0xd6, 0x57, 0x61, 0x86, 0x23, 0x78, 0x49, 0x9c, 0x07, 0xf5, 0x29, 0x1a, 0xeb, 0x26, 0xe9, + 0xac, 0xef, 0xe4, 0x24, 0xc4, 0xdf, 0xcf, 0xe6, 0xf3, 0xa5, 0xd1, 0xfb, 0xd9, 0xfc, 0x68, 0x09, + 0xee, 0x67, 0xf3, 0x50, 0x1a, 0xbb, 0x9f, 0xcd, 0x8f, 0x97, 0x26, 0xee, 0x67, 0xf3, 0x85, 0x52, + 0x51, 0xf9, 0x4f, 0x09, 0x66, 0x49, 0x10, 0xfe, 0x7f, 0x12, 0x50, 0xff, 0x20, 0x0f, 0x95, 0x38, + 0xbb, 0x5f, 0x44, 0xd4, 0x2f, 0x22, 0xea, 0x73, 0x8f, 0xa8, 0xe3, 0x5d, 0x23, 0x6a, 0x62, 0x6c, + 0x2a, 0x3c, 0xb7, 0xd8, 0xf4, 0xb3, 0x19, 0xb0, 0x7b, 0x44, 0xc4, 0xf2, 0x51, 0x22, 0xa2, 0x9c, + 0x2e, 0x22, 0x4e, 0x94, 0x0a, 0xca, 0x77, 0x25, 0x58, 0x50, 0x11, 0x46, 0x6e, 0x24, 0x68, 0xbf, + 0x84, 0x78, 0xa8, 0x54, 0xe1, 0x64, 0xf2, 0x56, 0x58, 0xac, 0x52, 0x7e, 0x98, 0x81, 0x45, 0x15, + 0xd5, 0x6d, 0xc7, 0x08, 0x1e, 0x8f, 0xb9, 0x77, 0xa7, 0xd8, 0xf0, 0xbb, 0x20, 0xc7, 0xaf, 0x86, + 0xe9, 0x77, 0x5e, 0x8e, 0xdd, 0x09, 0xe5, 0xd7, 0x40, 0x16, 0x2e, 0x68, 0x44, 0xc3, 0x57, 0xc9, + 0x9b, 0x11, 0x91, 0x65, 0x16, 0x46, 0xa8, 0xef, 0x7a, 0x11, 0x6b, 0x98, 0xfc, 0x5c, 0x33, 0xe4, + 0x53, 0x00, 0xa2, 0x06, 0xc0, 0x03, 0xd3, 0xa8, 0x3a, 0xca, 0x47, 0xd6, 0x0c, 0xf9, 0x03, 0x18, + 0x6f, 0xd9, 0x8d, 0x86, 0x77, 0x85, 0x67, 0x31, 0xe9, 0x8d, 0xa3, 0x5e, 0x3c, 0xd8, 0x0d, 0x7e, + 0x8c, 0x90, 0x14, 0x42, 0xf4, 0xae, 0x48, 0x23, 0x47, 0xbb, 0x22, 0x29, 0x7f, 0x94, 0x87, 0x33, + 0x3d, 0x54, 0xc5, 0x93, 0x4f, 0x2c, 0x67, 0x48, 0x47, 0xce, 0x19, 0x3d, 0xf3, 0xc1, 0x50, 0xcf, + 0x7c, 0x90, 0x4e, 0x69, 0xcb, 0x50, 0xea, 0x92, 0x6f, 0x0a, 0x38, 0x4c, 0x37, 0x96, 0xc6, 0x72, + 0xf1, 0x34, 0x16, 0xa8, 0x5f, 0x0c, 0x87, 0xeb, 0x17, 0xd7, 0xa0, 0xc2, 0xe3, 0x7b, 0xa0, 0x7a, + 0xc1, 0x4f, 0x5a, 0x23, 0xf4, 0xa4, 0x35, 0xc3, 0xe6, 0xfd, 0x8a, 0x04, 0x3f, 0x67, 0x3d, 0x85, + 0x59, 0xd7, 0xd1, 0x2d, 0x6c, 0x92, 0x65, 0xc3, 0x57, 0x54, 0x76, 0xa5, 0xff, 0x5a, 0xbf, 0x80, + 0xbb, 0x29, 0xd0, 0x83, 0xca, 0xa3, 0x45, 0x98, 0x69, 0x37, 0x69, 0x4a, 0xde, 0x81, 0x53, 0x09, + 0xc5, 0x96, 0x40, 0xaa, 0x1b, 0x4d, 0x91, 0xea, 0xe6, 0x63, 0x7e, 0xe5, 0x67, 0xbd, 0x33, 0x30, + 0x1e, 0x4a, 0x38, 0x63, 0x34, 0xe1, 0x8c, 0x6d, 0x05, 0x32, 0xcd, 0x5d, 0x28, 0xf8, 0xea, 0xa4, + 0x45, 0x9e, 0xf1, 0x01, 0x8b, 0x3c, 0x13, 0x1e, 0x1e, 0x2d, 0xe9, 0xac, 0xc2, 0xb8, 0xd0, 0x34, + 0x25, 0x33, 0x31, 0x20, 0x99, 0x31, 0x8e, 0x45, 0x89, 0xd8, 0x30, 0xf2, 0xb4, 0x8d, 0x78, 0xb6, + 0xcb, 0x2c, 0x8f, 0x5d, 0x79, 0xa7, 0x36, 0x50, 0xdd, 0xbe, 0xd6, 0xd7, 0x7b, 0x6a, 0x6f, 0x33, + 0xba, 0xb7, 0x2d, 0xd7, 0xe9, 0xa8, 0x62, 0x15, 0xdf, 0x75, 0x8b, 0x47, 0x73, 0xdd, 0xf9, 0x0f, + 0x60, 0x3c, 0x48, 0x59, 0x2e, 0x41, 0x66, 0x0f, 0x75, 0x78, 0x1c, 0x25, 0x7f, 0xca, 0xd7, 0x21, + 0xb7, 0xaf, 0x37, 0xda, 0x5d, 0x8e, 0x78, 0xb4, 0x74, 0x1e, 0xf4, 0x56, 0x42, 0xad, 0xa3, 0x32, + 0x94, 0xeb, 0x43, 0xd7, 0x24, 0x96, 0x7f, 0x02, 0xd1, 0xfc, 0x46, 0xdd, 0x35, 0xf7, 0x4d, 0xb7, + 0xf3, 0x45, 0x34, 0x4f, 0x1b, 0xcd, 0x83, 0x92, 0x7b, 0x81, 0xd1, 0xfc, 0x6f, 0xb2, 0x22, 0x9a, + 0x27, 0xaa, 0x8a, 0x47, 0xf3, 0x47, 0x50, 0x8c, 0x88, 0x8b, 0xc7, 0xf3, 0xa5, 0x30, 0x2f, 0x81, + 0x40, 0xc3, 0x0e, 0x70, 0x1d, 0x2a, 0x42, 0xb5, 0x10, 0x16, 0x69, 0xcc, 0xff, 0x86, 0x8e, 0xe2, + 0x7f, 0x81, 0x00, 0x9b, 0x09, 0x07, 0x58, 0x04, 0x55, 0x71, 0x86, 0xe5, 0x43, 0x5a, 0x24, 0x6e, + 0x64, 0x07, 0x5c, 0x70, 0x81, 0xd3, 0xb9, 0xc1, 0xc8, 0x6c, 0x84, 0xa2, 0xc8, 0x43, 0x28, 0xef, + 0x22, 0xdd, 0x71, 0xb7, 0x90, 0xee, 0x6a, 0x06, 0x72, 0x75, 0xb3, 0x81, 0x79, 0xd9, 0xb9, 0x7f, + 0x69, 0xb5, 0xe4, 0xa1, 0xde, 0x62, 0x98, 0xf1, 0x94, 0x39, 0x7c, 0xe4, 0x94, 0x79, 0x31, 0xe0, + 0x38, 0x9e, 0x43, 0x51, 0x1b, 0x19, 0xf5, 0xbd, 0xe1, 0x91, 0x98, 0xf0, 0xad, 0x28, 0x7f, 0x44, + 0x2b, 0xfa, 0x91, 0x04, 0xaf, 0x30, 0x63, 0x09, 0x85, 0x35, 0x5e, 0x39, 0x4e, 0xe5, 0xf3, 0x36, + 0x94, 0x78, 0xbd, 0x1a, 0x45, 0x1e, 0x32, 0x6e, 0xf5, 0xf5, 0x9b, 0x01, 0xb6, 0xa0, 0x16, 0x05, + 0x75, 0x3e, 0xa0, 0xfc, 0xfa, 0x10, 0x9c, 0xed, 0x8d, 0xc8, 0x9d, 0x00, 0xfb, 0xc7, 0x03, 0xf1, + 0x7c, 0xc3, 0xbd, 0xe0, 0xde, 0xf3, 0x0a, 0xfc, 0xe4, 0x2e, 0x18, 0xf6, 0x3c, 0x04, 0x05, 0x9d, + 0x3b, 0x26, 0x4d, 0xba, 0xb8, 0x32, 0x44, 0x73, 0xcd, 0x9b, 0x47, 0x0d, 0x22, 0x7c, 0xa1, 0x09, + 0x3d, 0x30, 0x85, 0x95, 0x3f, 0x97, 0x48, 0xc4, 0x8e, 0x09, 0xe1, 0x8e, 0x6e, 0x36, 0x52, 0x69, + 0x6f, 0x17, 0x0a, 0xdb, 0x14, 0x27, 0xa2, 0xbb, 0x1b, 0x47, 0xd1, 0x5d, 0x68, 0x75, 0x75, 0x62, + 0x3b, 0xf8, 0x53, 0x79, 0x85, 0xc4, 0xad, 0xae, 0x28, 0xfc, 0x5a, 0xf1, 0x23, 0x09, 0x94, 0x78, + 0x74, 0xbb, 0x27, 0x3c, 0x2f, 0x05, 0x63, 0xad, 0xa0, 0xaf, 0x87, 0x79, 0x5b, 0x1d, 0x80, 0xb7, + 0x7e, 0x5b, 0x08, 0x84, 0x03, 0xc1, 0xe0, 0x3a, 0x71, 0xa9, 0x1e, 0x78, 0xdc, 0x40, 0x5e, 0x85, + 0x52, 0x5d, 0xb7, 0xea, 0xc8, 0xcb, 0x32, 0x88, 0xed, 0x3f, 0xaf, 0x16, 0xd9, 0xb8, 0x2a, 0x86, + 0x83, 0x5e, 0x1a, 0xa4, 0xf9, 0x92, 0xbc, 0xb4, 0xd7, 0x16, 0xe2, 0x5e, 0x7a, 0xce, 0x73, 0xd2, + 0x2e, 0x78, 0x5c, 0xe3, 0x01, 0x43, 0x0e, 0x02, 0xfe, 0xdf, 0x1b, 0x72, 0xd7, 0xd5, 0xbb, 0x1b, + 0x72, 0x12, 0x0a, 0x67, 0xeb, 0x2f, 0xa8, 0x21, 0xc7, 0xf9, 0xa7, 0x1a, 0x4e, 0xc5, 0xd8, 0x2f, + 0x41, 0x21, 0x6c, 0x2f, 0x29, 0xac, 0xb8, 0xdf, 0xfa, 0xea, 0x44, 0xc8, 0xe4, 0x94, 0xa5, 0x64, + 0x7b, 0xf3, 0x90, 0x38, 0x73, 0x7f, 0x3b, 0x04, 0xd5, 0x0d, 0x73, 0xc7, 0xd2, 0x1b, 0xc7, 0x79, + 0xfe, 0xde, 0x86, 0x02, 0xa6, 0x44, 0x22, 0x8c, 0xbd, 0xd5, 0xff, 0xfd, 0xbb, 0xe7, 0xda, 0xea, + 0x04, 0x23, 0x2b, 0xb6, 0x62, 0xc2, 0x02, 0x3a, 0x74, 0x91, 0x43, 0x56, 0x4a, 0x38, 0x9d, 0x66, + 0xd2, 0x9e, 0x4e, 0xe7, 0x04, 0xb5, 0xd8, 0x94, 0x5c, 0x83, 0xc9, 0xfa, 0xae, 0xd9, 0x30, 0xfc, + 0x75, 0x6c, 0xab, 0xd1, 0xa1, 0x87, 0x97, 0xbc, 0x5a, 0xa6, 0x53, 0x02, 0xe9, 0x1b, 0x56, 0xa3, + 0xa3, 0x9c, 0x81, 0xd3, 0x5d, 0x79, 0xe1, 0xb2, 0xfe, 0x07, 0x09, 0xce, 0x73, 0x18, 0xd3, 0xdd, + 0x3d, 0x76, 0xcf, 0xc1, 0x77, 0x24, 0x98, 0xe3, 0x52, 0x3f, 0x30, 0xdd, 0x5d, 0x2d, 0xa9, 0x01, + 0xe1, 0xde, 0xa0, 0x0a, 0xe8, 0xb7, 0x21, 0x75, 0x06, 0x87, 0x01, 0x85, 0x9d, 0xdd, 0x80, 0xe5, + 0xfe, 0x24, 0x7a, 0xbe, 0x1c, 0x2b, 0x7f, 0x25, 0xc1, 0x69, 0x15, 0x35, 0xed, 0x7d, 0xc4, 0x28, + 0x1d, 0xf1, 0x01, 0xe1, 0xc5, 0xdd, 0x58, 0xc2, 0x57, 0x8d, 0x4c, 0xe4, 0xaa, 0xa1, 0x28, 0x24, + 0xec, 0x75, 0xdb, 0xbe, 0xd0, 0xfd, 0x10, 0x9c, 0xd9, 0x44, 0x4e, 0xd3, 0xb4, 0x74, 0x17, 0x1d, + 0x47, 0xeb, 0x36, 0x94, 0x5d, 0x41, 0x27, 0xa2, 0xec, 0x9b, 0x7d, 0x95, 0xdd, 0x77, 0x07, 0x6a, + 0xc9, 0x23, 0xfe, 0x33, 0xe0, 0x73, 0x67, 0x41, 0xe9, 0xc5, 0x11, 0x17, 0xfd, 0x7f, 0x4b, 0x50, + 0xbd, 0x85, 0x48, 0xaa, 0x3a, 0x8e, 0xdc, 0x5f, 0x9c, 0x75, 0xbd, 0x0a, 0x25, 0x8f, 0x32, 0xaf, + 0xc0, 0xf3, 0xdb, 0xb0, 0x57, 0x1f, 0xe7, 0xa5, 0x7a, 0xfa, 0x40, 0xd0, 0xb0, 0x31, 0x4a, 0x96, + 0x90, 0xcc, 0xe6, 0xa2, 0x61, 0xa9, 0x2b, 0xef, 0x5c, 0x3e, 0x7f, 0x2a, 0xc1, 0x29, 0x5a, 0x20, + 0x3e, 0x66, 0x03, 0x94, 0x43, 0x68, 0xa4, 0x6e, 0x80, 0xea, 0xb9, 0xb2, 0x3a, 0x4e, 0x89, 0x8a, + 0x58, 0xf3, 0x3a, 0x54, 0xbb, 0x81, 0xf7, 0x8e, 0x30, 0xbf, 0x9f, 0x81, 0x25, 0x4e, 0x84, 0x65, + 0xc0, 0xe3, 0xb0, 0xda, 0xec, 0x92, 0xc5, 0xef, 0x0c, 0xc0, 0xeb, 0x00, 0x5b, 0x88, 0x24, 0x72, + 0xf9, 0x8d, 0x80, 0xff, 0xf1, 0xde, 0xa7, 0x78, 0xdd, 0xa4, 0x22, 0x40, 0xd6, 0x04, 0x84, 0xa8, + 0x9f, 0xf4, 0x71, 0xdf, 0xec, 0x8b, 0x77, 0xdf, 0x5c, 0x37, 0xf7, 0x5d, 0x86, 0x73, 0xfd, 0x24, + 0xc2, 0x4d, 0xf4, 0xa3, 0x21, 0x58, 0x10, 0xf7, 0xff, 0xe0, 0x95, 0xe3, 0x73, 0xe1, 0xbf, 0x57, + 0x61, 0xc6, 0xc4, 0x5a, 0x42, 0x57, 0x16, 0xd5, 0x4d, 0x5e, 0x9d, 0x34, 0xf1, 0x9d, 0x68, 0xbb, + 0x95, 0x7f, 0xed, 0xcf, 0x1e, 0xf1, 0xda, 0x5f, 0x85, 0x93, 0xc9, 0x12, 0xe1, 0x22, 0xfb, 0x37, + 0x09, 0xce, 0x3f, 0x46, 0x8e, 0xb9, 0xdd, 0x89, 0x2d, 0xee, 0x55, 0x52, 0x3e, 0x17, 0xe2, 0xf3, + 0x24, 0x91, 0x39, 0xa2, 0x24, 0x2e, 0xc0, 0x72, 0x7f, 0x46, 0xb9, 0x54, 0xfe, 0x27, 0x43, 0xee, + 0x32, 0xe4, 0x66, 0xb7, 0x4a, 0xcc, 0xd1, 0xdb, 0xc5, 0x51, 0xee, 0x61, 0x2f, 0x4e, 0x24, 0x35, + 0xe0, 0x5d, 0x99, 0x01, 0x87, 0xf7, 0x5c, 0xbd, 0xcc, 0xa6, 0x3c, 0x47, 0x5f, 0x33, 0xe4, 0xf7, + 0x60, 0x52, 0xdc, 0xd9, 0x8c, 0xe3, 0xf8, 0xb6, 0xec, 0x51, 0xf1, 0xf7, 0xb2, 0xee, 0xdd, 0x36, + 0xe9, 0x53, 0x09, 0xad, 0x3f, 0xe6, 0xd2, 0xd4, 0x1f, 0x8b, 0x3e, 0x3a, 0x2b, 0x40, 0x7a, 0x0a, + 0x1f, 0x3e, 0x62, 0xa3, 0xe0, 0x35, 0xa8, 0xc4, 0xc4, 0x23, 0x12, 0xe7, 0x08, 0x7f, 0x93, 0x0a, + 0xcb, 0x88, 0xe7, 0x4f, 0xe5, 0x3c, 0x49, 0x03, 0x3d, 0xb5, 0x2f, 0x72, 0x62, 0x06, 0x2e, 0x32, + 0xa3, 0x4a, 0x84, 0xa4, 0xb1, 0x89, 0xd0, 0x49, 0x65, 0x30, 0x9b, 0x50, 0x8a, 0xf6, 0xef, 0xa6, + 0x37, 0x97, 0x62, 0xa4, 0x5f, 0x57, 0x56, 0xa1, 0xc8, 0xa2, 0xee, 0x31, 0xce, 0x64, 0x85, 0x7a, + 0x88, 0xcb, 0x6e, 0x06, 0x98, 0xed, 0x66, 0x80, 0xbd, 0x34, 0x92, 0xeb, 0xa5, 0x91, 0x63, 0x1b, + 0x83, 0x72, 0x09, 0x6a, 0x83, 0x2a, 0x8a, 0xeb, 0xf6, 0x8f, 0x25, 0x58, 0xbc, 0x85, 0x70, 0xdd, + 0x31, 0xb7, 0x8e, 0x75, 0x22, 0xfc, 0x26, 0x8c, 0xa4, 0xad, 0x4f, 0xf4, 0x5b, 0x56, 0x15, 0x14, + 0x95, 0xdf, 0xcb, 0xc2, 0x99, 0x1e, 0xd0, 0xfc, 0xb8, 0xf3, 0x2d, 0x28, 0xf9, 0xef, 0x82, 0x75, + 0xdb, 0xda, 0x36, 0x77, 0x78, 0x59, 0xf4, 0x72, 0xf2, 0x5e, 0x12, 0xd5, 0xbf, 0x4a, 0x11, 0xd5, + 0x22, 0x0a, 0x0f, 0xc8, 0x3b, 0x30, 0x9b, 0xf0, 0xfc, 0x48, 0x3b, 0xce, 0x19, 0xc3, 0x2b, 0x29, + 0x16, 0x61, 0xef, 0x9c, 0x07, 0x49, 0xc3, 0xf2, 0xb7, 0x40, 0x6e, 0x21, 0xcb, 0x30, 0xad, 0x1d, + 0x8d, 0x97, 0x46, 0x4d, 0x84, 0x2b, 0x19, 0x5a, 0x6c, 0xbd, 0xd8, 0x7d, 0x8d, 0x75, 0x86, 0x23, + 0xea, 0x1b, 0x74, 0x85, 0x72, 0x2b, 0x34, 0x68, 0x22, 0x2c, 0x7f, 0x1b, 0x4a, 0x82, 0x3a, 0x35, + 0x73, 0x87, 0xb6, 0x75, 0x11, 0xda, 0x57, 0xfb, 0xd2, 0x0e, 0x1b, 0x15, 0x5d, 0xa1, 0xd8, 0x0a, + 0x4c, 0x39, 0xc8, 0x92, 0x11, 0x4c, 0x0b, 0xfa, 0xe1, 0xf4, 0x9f, 0xeb, 0xa7, 0x09, 0xbe, 0x48, + 0xec, 0x39, 0x78, 0xb2, 0x15, 0x9f, 0x50, 0x7e, 0x2d, 0x03, 0x15, 0x95, 0x7f, 0xb2, 0x81, 0x68, + 0x24, 0xc5, 0x8f, 0xaf, 0x7c, 0x2e, 0xd2, 0xd5, 0x36, 0x4c, 0x87, 0x9b, 0x90, 0x3a, 0x9a, 0xe9, + 0xa2, 0xa6, 0xd0, 0xe0, 0x95, 0x54, 0x8d, 0x48, 0x9d, 0x35, 0x17, 0x35, 0xd5, 0xc9, 0xfd, 0xd8, + 0x18, 0x96, 0xaf, 0xc1, 0x30, 0xcd, 0x3f, 0x98, 0x67, 0xb6, 0xae, 0x0f, 0x3d, 0xb7, 0x74, 0x57, + 0xbf, 0xd9, 0xb0, 0xb7, 0x54, 0x0e, 0x2f, 0xdf, 0x81, 0x82, 0x85, 0x0e, 0x68, 0x3f, 0x0f, 0xa7, + 0x90, 0x1b, 0x90, 0xc2, 0xb8, 0x85, 0x0e, 0xd4, 0x36, 0xcb, 0x5c, 0x58, 0x59, 0x80, 0xb9, 0x04, + 0x15, 0xf0, 0xb8, 0xf2, 0xf7, 0xf4, 0x1e, 0xc5, 0x67, 0x9f, 0x04, 0x5b, 0x9d, 0x84, 0x96, 0xb4, + 0x58, 0x3b, 0x15, 0x73, 0xd6, 0x6b, 0x89, 0x12, 0x0a, 0x7c, 0x38, 0x13, 0x54, 0x45, 0xa8, 0xbc, + 0x10, 0x69, 0xa9, 0x5a, 0x82, 0x82, 0x83, 0x9a, 0xb6, 0x8b, 0xb4, 0x7a, 0xa3, 0x8d, 0x5d, 0xe4, + 0x50, 0xfd, 0x8e, 0xaa, 0x13, 0x6c, 0x74, 0x95, 0x0d, 0xc6, 0xac, 0x25, 0x13, 0xb3, 0x16, 0x65, + 0x91, 0x5c, 0xb5, 0x92, 0x79, 0xe1, 0xec, 0xfe, 0xa1, 0x04, 0x33, 0x1b, 0x1d, 0xab, 0xbe, 0xb1, + 0xab, 0x3b, 0x06, 0xef, 0xc4, 0xe2, 0x7c, 0x2e, 0x41, 0x01, 0xdb, 0x6d, 0xa7, 0xee, 0x6f, 0x83, + 0xd9, 0xe3, 0x04, 0x1b, 0x15, 0xdb, 0x98, 0x83, 0x3c, 0x26, 0xc8, 0xa2, 0x97, 0x24, 0xa7, 0x8e, + 0xd0, 0xdf, 0x6b, 0x86, 0x7c, 0x03, 0xc6, 0x58, 0x4b, 0x18, 0x7b, 0x32, 0xcc, 0x0c, 0xf8, 0x64, + 0x08, 0x0c, 0x89, 0x0c, 0x2b, 0x73, 0x30, 0x1b, 0xdb, 0x1e, 0xdf, 0xfa, 0x67, 0x39, 0x98, 0x24, + 0x73, 0x22, 0x72, 0xa4, 0xf0, 0xa2, 0xd3, 0x30, 0xe6, 0xa9, 0x90, 0x6f, 0x7b, 0x54, 0x05, 0x31, + 0xb4, 0x66, 0x04, 0x6e, 0xa0, 0x99, 0xe0, 0xd7, 0x11, 0x15, 0x18, 0x11, 0x09, 0x91, 0x65, 0x51, + 0xf1, 0xb3, 0xcb, 0x73, 0x78, 0xae, 0xcb, 0x73, 0x78, 0xbc, 0x0d, 0x63, 0xf8, 0x68, 0x6d, 0x18, + 0x49, 0x0d, 0x37, 0x23, 0x89, 0x0d, 0x37, 0xd1, 0x07, 0xe3, 0xfc, 0x51, 0x1e, 0x8c, 0xd7, 0x79, + 0x77, 0xa8, 0xff, 0x90, 0x43, 0x69, 0x8d, 0x0e, 0x48, 0xab, 0x4c, 0x90, 0xbd, 0x07, 0x18, 0x4a, + 0xf1, 0x3a, 0x8c, 0x88, 0x77, 0x5f, 0x18, 0xf0, 0xdd, 0x57, 0x20, 0x04, 0x9f, 0xaf, 0xc7, 0xc2, + 0xcf, 0xd7, 0xab, 0x30, 0xce, 0x7a, 0x07, 0xf9, 0x17, 0x40, 0xe3, 0x03, 0x7e, 0x01, 0x34, 0x46, + 0x5b, 0x0a, 0xf9, 0xc7, 0x3f, 0x97, 0x80, 0x7e, 0xbc, 0x43, 0xd3, 0x01, 0x72, 0x34, 0xd3, 0x40, + 0x96, 0x6b, 0xba, 0x1d, 0xda, 0xea, 0x32, 0xaa, 0xca, 0x64, 0xee, 0x09, 0x9d, 0x5a, 0xe3, 0x33, + 0xf2, 0x13, 0x28, 0x46, 0x42, 0x28, 0xef, 0xe2, 0xac, 0xa5, 0x0b, 0x9e, 0x6a, 0x21, 0x1c, 0x38, + 0x95, 0x19, 0x98, 0x0a, 0x5b, 0x3a, 0x77, 0x81, 0xef, 0x4a, 0xb0, 0x20, 0xce, 0x17, 0x2f, 0xb9, + 0x61, 0x5b, 0xf9, 0x2f, 0x09, 0x4e, 0x26, 0xef, 0x85, 0x1f, 0x73, 0x76, 0x61, 0xb2, 0xae, 0xd7, + 0x77, 0x51, 0xf8, 0x9b, 0xc1, 0x63, 0x07, 0xcf, 0x32, 0x25, 0x1a, 0x1c, 0x92, 0x2d, 0x98, 0x31, + 0x74, 0x57, 0xdf, 0xd2, 0x71, 0x74, 0xb1, 0xa1, 0x63, 0x2e, 0x36, 0x25, 0xe8, 0x06, 0x47, 0x95, + 0x7f, 0x94, 0x60, 0x5e, 0xb0, 0xce, 0x55, 0x76, 0xcf, 0xc6, 0xc1, 0xc7, 0xd1, 0x5d, 0x1b, 0xbb, + 0x9a, 0x6e, 0x18, 0x0e, 0xc2, 0x58, 0x68, 0x81, 0x8c, 0xdd, 0x60, 0x43, 0xbd, 0x82, 0x68, 0xff, + 0x30, 0xdf, 0xe5, 0x50, 0x90, 0x3d, 0xfe, 0xa1, 0x40, 0xf9, 0x97, 0x80, 0x81, 0x85, 0x38, 0xe3, + 0x3a, 0x7d, 0x05, 0x26, 0xe8, 0x3e, 0xb1, 0x66, 0xb5, 0x9b, 0x5b, 0x3c, 0x45, 0xe4, 0xd4, 0x71, + 0x36, 0xf8, 0x88, 0x8e, 0xc9, 0x0b, 0x30, 0x2a, 0x98, 0x63, 0x8f, 0xef, 0x39, 0x35, 0xcf, 0xb9, + 0xc3, 0xf2, 0xfb, 0x50, 0xf4, 0xd9, 0xa3, 0xaa, 0xec, 0xf9, 0x21, 0xa4, 0x07, 0x4b, 0x58, 0xf0, + 0xfa, 0x2f, 0x56, 0x09, 0x1e, 0x3d, 0x74, 0x15, 0xac, 0xd0, 0x18, 0x8d, 0x11, 0x5c, 0xec, 0xac, + 0xb9, 0x48, 0xfc, 0xbc, 0x9f, 0xcd, 0x67, 0x4b, 0x39, 0xa5, 0x06, 0xe5, 0xd5, 0x86, 0x8d, 0x11, + 0x4d, 0x30, 0x42, 0x61, 0x41, 0x6d, 0x48, 0x21, 0x6d, 0x28, 0x53, 0x20, 0x07, 0xe1, 0xb9, 0x1f, + 0xbe, 0x06, 0xc5, 0xbb, 0xc8, 0x1d, 0x94, 0xc6, 0x07, 0x50, 0xf2, 0xa1, 0xb9, 0x20, 0x1f, 0x00, + 0x70, 0x70, 0x72, 0x30, 0x67, 0x3e, 0x71, 0x71, 0x10, 0x33, 0xa5, 0x64, 0x28, 0xeb, 0x4c, 0xc8, + 0xe4, 0x4f, 0xe5, 0x9f, 0x24, 0x28, 0xb3, 0xc7, 0x8c, 0x60, 0x7d, 0xad, 0xfb, 0x96, 0xe4, 0x3b, + 0x90, 0x27, 0x67, 0x84, 0x1d, 0x12, 0xb2, 0x86, 0x68, 0xfb, 0xf6, 0x85, 0xde, 0xcd, 0xe1, 0xec, + 0x19, 0x92, 0x61, 0xa8, 0x1e, 0x6e, 0xb0, 0xcf, 0x2b, 0x13, 0xea, 0xf3, 0x5a, 0x83, 0xe2, 0xbe, + 0x89, 0xcd, 0x2d, 0xb3, 0x41, 0xfb, 0x30, 0xd2, 0x74, 0x10, 0x15, 0x7c, 0x44, 0x7a, 0x24, 0x98, + 0x02, 0x39, 0xc8, 0x9b, 0x28, 0x2e, 0x4a, 0x70, 0xea, 0x2e, 0x72, 0x55, 0xff, 0x73, 0xe8, 0x87, + 0xec, 0x53, 0x68, 0xef, 0x3c, 0xf3, 0x00, 0x86, 0x69, 0x5f, 0x24, 0x71, 0xc0, 0x4c, 0x57, 0x03, + 0x0b, 0x7c, 0x4f, 0xcd, 0x8a, 0xbd, 0xde, 0x4f, 0xda, 0x41, 0xa9, 0x72, 0x1a, 0xc4, 0x2d, 0xf9, + 0xb1, 0x88, 0xf6, 0x07, 0xf1, 0x33, 0xc4, 0x18, 0x1f, 0x23, 0x96, 0xa9, 0xfc, 0x60, 0x08, 0xaa, + 0xdd, 0xb6, 0xc4, 0xd5, 0xfe, 0x2b, 0x50, 0x60, 0x2a, 0xe1, 0xdf, 0x6d, 0x8b, 0xbd, 0xbd, 0x3b, + 0x60, 0x3f, 0x4c, 0x6f, 0xf2, 0xcc, 0x38, 0xc4, 0x28, 0xeb, 0x85, 0x64, 0xfe, 0x2a, 0xc6, 0xe6, + 0x3b, 0x20, 0xc7, 0x81, 0x82, 0x6d, 0x8d, 0x39, 0xd6, 0xd6, 0xf8, 0x30, 0xdc, 0xd6, 0xf8, 0x7a, + 0x4a, 0xd9, 0x79, 0x3b, 0xf3, 0x3b, 0x1d, 0x95, 0x0f, 0x61, 0xf1, 0x2e, 0x72, 0x6f, 0x3d, 0x78, + 0xbb, 0x87, 0xce, 0x1e, 0xf3, 0xef, 0x4b, 0x88, 0x57, 0x08, 0xd9, 0xa4, 0x5d, 0xdb, 0xbb, 0x90, + 0xd1, 0x4f, 0x4e, 0xc8, 0x5f, 0x58, 0xf9, 0x0d, 0x09, 0xce, 0xf4, 0x58, 0x9c, 0x6b, 0xe7, 0x03, + 0x28, 0x07, 0xc8, 0xf2, 0xee, 0x21, 0x29, 0x7a, 0xe9, 0x1c, 0x78, 0x13, 0x6a, 0xc9, 0x09, 0x0f, + 0x60, 0xe5, 0x7b, 0x12, 0x4c, 0xd1, 0x16, 0x50, 0x11, 0x8d, 0x53, 0x64, 0xee, 0x6f, 0x44, 0x2b, + 0x17, 0x5f, 0xe9, 0x5b, 0xb9, 0x48, 0x5a, 0xca, 0xaf, 0x56, 0xec, 0xc1, 0x74, 0x04, 0x80, 0xcb, + 0x41, 0x85, 0x7c, 0xa4, 0x5f, 0xeb, 0xab, 0x69, 0x97, 0xe2, 0x4d, 0x53, 0x1e, 0x1d, 0xe5, 0x77, + 0x25, 0x98, 0x52, 0x91, 0xde, 0x6a, 0x35, 0x58, 0x85, 0x11, 0xa7, 0xe0, 0x7c, 0x23, 0xca, 0x79, + 0x72, 0xd3, 0x76, 0xf0, 0x5f, 0x07, 0x30, 0x75, 0xc4, 0x97, 0xf3, 0xb9, 0x9f, 0x85, 0xe9, 0x08, + 0x00, 0xdf, 0xe9, 0x9f, 0x0d, 0xc1, 0x34, 0xb3, 0x95, 0xa8, 0x75, 0xde, 0x86, 0xac, 0xd7, 0x99, + 0x5f, 0x08, 0x96, 0x08, 0x92, 0x22, 0xe6, 0x2d, 0xa4, 0x1b, 0x0f, 0x90, 0xeb, 0x22, 0x87, 0xf6, + 0x91, 0xd1, 0x9e, 0x43, 0x8a, 0xde, 0x2b, 0xf9, 0xc7, 0xef, 0x60, 0x99, 0xa4, 0x3b, 0xd8, 0xeb, + 0x50, 0x31, 0x2d, 0x02, 0x61, 0xee, 0x23, 0x0d, 0x59, 0x5e, 0x38, 0xf1, 0xcb, 0x7d, 0xd3, 0xde, + 0xfc, 0x6d, 0x4b, 0x38, 0xfb, 0x9a, 0x21, 0x5f, 0x80, 0x72, 0x53, 0x3f, 0x34, 0x9b, 0xed, 0xa6, + 0xd6, 0x22, 0xf0, 0xd8, 0xfc, 0x90, 0x7d, 0xf7, 0x9f, 0x53, 0x8b, 0x7c, 0x62, 0x5d, 0xdf, 0x41, + 0x1b, 0xe6, 0x87, 0x48, 0x3e, 0x07, 0x45, 0xda, 0xb2, 0x4f, 0x01, 0x59, 0x87, 0xf9, 0x30, 0xed, + 0x30, 0xa7, 0x9d, 0xfc, 0x04, 0x8c, 0x7d, 0x52, 0xf7, 0xef, 0xec, 0x8b, 0xec, 0x90, 0xbc, 0xb8, + 0x21, 0x3d, 0x27, 0x81, 0x25, 0xfa, 0xe5, 0xd0, 0x73, 0xf4, 0xcb, 0x24, 0x5e, 0x33, 0x49, 0xbc, + 0xfe, 0xb3, 0x04, 0xb3, 0xeb, 0x6d, 0x67, 0x07, 0xfd, 0x3c, 0x5a, 0x87, 0x32, 0x0f, 0x95, 0x38, + 0x73, 0xa2, 0x4d, 0x6c, 0x08, 0x66, 0x1f, 0xa2, 0x9f, 0x53, 0xce, 0x5f, 0x88, 0x5f, 0xdc, 0x84, + 0x4a, 0x5c, 0x60, 0xdc, 0x31, 0x12, 0x68, 0x48, 0x49, 0x34, 0x7e, 0x40, 0xbf, 0x48, 0xdb, 0x76, + 0x10, 0xde, 0x0d, 0x96, 0x15, 0xd3, 0x04, 0xcf, 0xf7, 0xa2, 0xc1, 0xf3, 0x17, 0x07, 0x0c, 0x9e, + 0x5d, 0x57, 0xf5, 0x63, 0x28, 0xfd, 0x48, 0x2d, 0x09, 0x8e, 0x1b, 0xcd, 0xf7, 0x25, 0xb8, 0x70, + 0x17, 0x59, 0xc8, 0xd1, 0x5d, 0xf4, 0x40, 0xc7, 0xae, 0xb8, 0xef, 0x46, 0xdc, 0xef, 0x65, 0x5c, + 0x5f, 0x2f, 0xc2, 0x97, 0x06, 0xda, 0x19, 0xe7, 0xe4, 0x0e, 0x2c, 0x84, 0xcf, 0x5e, 0xe1, 0xda, + 0xd9, 0x79, 0x28, 0x86, 0x4b, 0x78, 0xec, 0xdc, 0x30, 0xaa, 0x16, 0x42, 0x35, 0x3c, 0xac, 0xb4, + 0xe1, 0x64, 0x32, 0x1d, 0x6e, 0x18, 0xef, 0xc0, 0x30, 0xbb, 0x4b, 0xf1, 0x73, 0xc7, 0x1b, 0x03, + 0x1e, 0x0c, 0xf9, 0xed, 0x22, 0x4a, 0x96, 0x13, 0x53, 0xfe, 0x7a, 0x18, 0x66, 0x92, 0x41, 0x7a, + 0xdd, 0x12, 0xbe, 0x02, 0xb3, 0x4d, 0xfd, 0x50, 0x8b, 0xc6, 0x5e, 0xff, 0x2b, 0xb2, 0xa9, 0xa6, + 0x7e, 0x18, 0x3d, 0x79, 0x19, 0xf2, 0x03, 0x28, 0x31, 0x8a, 0x0d, 0xbb, 0xae, 0x37, 0x06, 0xad, + 0x05, 0x0e, 0x93, 0xc3, 0x7f, 0x45, 0x52, 0xd9, 0x01, 0xf9, 0x01, 0x41, 0xa5, 0x15, 0xa3, 0x0f, + 0xe3, 0xa2, 0x65, 0xef, 0x00, 0x6f, 0x1f, 0x4b, 0x34, 0x35, 0x35, 0xa4, 0x18, 0x76, 0x58, 0x8e, + 0x68, 0x4b, 0xfe, 0x4d, 0x09, 0x26, 0x77, 0x75, 0xcb, 0xb0, 0xf7, 0xf9, 0xb1, 0x9f, 0x9a, 0x21, + 0xb9, 0x5a, 0xa6, 0xf9, 0x7a, 0xa9, 0xcb, 0x06, 0xee, 0x71, 0xc2, 0xde, 0xad, 0x96, 0x6f, 0x42, + 0xde, 0x8d, 0x4d, 0xc8, 0x2d, 0x38, 0x9b, 0xa8, 0x89, 0xe8, 0x1d, 0x6b, 0xd0, 0xb2, 0xe2, 0x62, + 0x5c, 0x71, 0x8f, 0x43, 0xb7, 0xae, 0xf9, 0xef, 0x49, 0x30, 0x99, 0x20, 0xa2, 0x84, 0x2f, 0xa0, + 0xde, 0x0f, 0x5f, 0x15, 0xee, 0x1e, 0x4b, 0x2a, 0xeb, 0xc8, 0xe1, 0xeb, 0x05, 0xae, 0x0e, 0xf3, + 0xdf, 0x91, 0x60, 0xb6, 0x8b, 0xb8, 0x12, 0x36, 0xa4, 0x86, 0x37, 0xf4, 0xf5, 0x01, 0x37, 0x14, + 0x5b, 0x80, 0x5e, 0x22, 0x02, 0x17, 0x98, 0x77, 0x61, 0x3a, 0x11, 0x46, 0x7e, 0x0b, 0x4e, 0x7a, + 0x56, 0x92, 0xe4, 0x2c, 0x12, 0x75, 0x96, 0x39, 0x01, 0x13, 0xf3, 0x18, 0xe5, 0x4f, 0x24, 0x58, + 0xec, 0x27, 0x0f, 0x59, 0x81, 0x09, 0xbd, 0xbe, 0x87, 0x8c, 0x08, 0xd9, 0x31, 0x3a, 0xc8, 0x5d, + 0xef, 0x7d, 0x98, 0x0f, 0xc0, 0x44, 0xad, 0x63, 0xd0, 0x8f, 0x86, 0x66, 0x3d, 0x92, 0x61, 0xa3, + 0x50, 0x7e, 0x4b, 0x82, 0x79, 0x15, 0x6d, 0xb5, 0xcd, 0x86, 0xf1, 0xb2, 0xcb, 0x8f, 0xa7, 0x48, + 0x62, 0x4c, 0xd8, 0x09, 0x8f, 0xd7, 0x7f, 0x39, 0x04, 0x4b, 0xe1, 0x16, 0x3a, 0x9f, 0x15, 0xf6, + 0xb6, 0xfc, 0x32, 0xfe, 0xc9, 0xc5, 0x3a, 0x4c, 0x06, 0x9f, 0x92, 0xf8, 0xbf, 0x24, 0x18, 0xf8, + 0xa1, 0xa4, 0x1c, 0x78, 0x37, 0x62, 0xff, 0x7f, 0x20, 0x44, 0x91, 0x36, 0x12, 0xa6, 0xab, 0xb5, + 0x78, 0x14, 0x69, 0x91, 0x8b, 0xea, 0x78, 0x19, 0xce, 0xf5, 0x13, 0x1c, 0x97, 0xf1, 0xef, 0x48, + 0x30, 0xfd, 0x4e, 0xcb, 0x08, 0xbc, 0x35, 0xa5, 0x90, 0xe9, 0x7a, 0xf4, 0x58, 0xd2, 0xff, 0x8a, + 0x99, 0xb8, 0x96, 0x7f, 0x18, 0x69, 0xc2, 0x4c, 0x14, 0x82, 0x27, 0xd5, 0x8d, 0xd8, 0x7d, 0xf6, + 0xf5, 0xd4, 0x8b, 0x45, 0x2f, 0xb4, 0x37, 0x5b, 0x1f, 0x7f, 0x52, 0x3d, 0xf1, 0xe3, 0x4f, 0xaa, + 0x27, 0x7e, 0xfa, 0x49, 0x55, 0xfa, 0xd5, 0x67, 0x55, 0xe9, 0x87, 0xcf, 0xaa, 0xd2, 0xdf, 0x3d, + 0xab, 0x4a, 0x1f, 0x3f, 0xab, 0x4a, 0xff, 0xfa, 0xac, 0x2a, 0x7d, 0xf6, 0xac, 0x7a, 0xe2, 0xa7, + 0xcf, 0xaa, 0xd2, 0x47, 0x9f, 0x56, 0x4f, 0x7c, 0xfc, 0x69, 0xf5, 0xc4, 0x8f, 0x3f, 0xad, 0x9e, + 0x78, 0xef, 0xfa, 0x8e, 0xed, 0x2f, 0x6d, 0xda, 0x3d, 0xff, 0xd9, 0xe5, 0x2f, 0x84, 0x47, 0xb6, + 0x86, 0xa9, 0x1a, 0xaf, 0xfe, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x57, 0xe7, 0x82, 0x20, 0x2b, + 0x53, 0x00, 0x00, } func (this *StartWorkflowExecutionRequest) Equal(that interface{}) bool { @@ -5933,6 +5944,9 @@ func (this *StartWorkflowExecutionResponse) Equal(that interface{}) bool { if !this.Clock.Equal(that1.Clock) { return false } + if !this.EagerWorkflowTask.Equal(that1.EagerWorkflowTask) { + return false + } return true } func (this *GetMutableStateRequest) Equal(that interface{}) bool { @@ -8784,12 +8798,15 @@ func (this *StartWorkflowExecutionResponse) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 6) + s := make([]string, 0, 7) s = append(s, "&historyservice.StartWorkflowExecutionResponse{") s = append(s, "RunId: "+fmt.Sprintf("%#v", this.RunId)+",\n") if this.Clock != nil { s = append(s, "Clock: "+fmt.Sprintf("%#v", this.Clock)+",\n") } + if this.EagerWorkflowTask != nil { + s = append(s, "EagerWorkflowTask: "+fmt.Sprintf("%#v", this.EagerWorkflowTask)+",\n") + } s = append(s, "}") return strings.Join(s, "") } @@ -10235,6 +10252,18 @@ func (m *StartWorkflowExecutionResponse) MarshalToSizedBuffer(dAtA []byte) (int, _ = i var l int _ = l + if m.EagerWorkflowTask != nil { + { + size, err := m.EagerWorkflowTask.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRequestResponse(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } if m.Clock != nil { { size, err := m.Clock.MarshalToSizedBuffer(dAtA[:i]) @@ -10393,12 +10422,12 @@ func (m *GetMutableStateResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) dAtA[i] = 0x6a } if m.StickyTaskQueueScheduleToStartTimeout != nil { - n10, err10 := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.StickyTaskQueueScheduleToStartTimeout, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(*m.StickyTaskQueueScheduleToStartTimeout):]) - if err10 != nil { - return 0, err10 + n11, err11 := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.StickyTaskQueueScheduleToStartTimeout, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(*m.StickyTaskQueueScheduleToStartTimeout):]) + if err11 != nil { + return 0, err11 } - i -= n10 - i = encodeVarintRequestResponse(dAtA, i, uint64(n10)) + i -= n11 + i = encodeVarintRequestResponse(dAtA, i, uint64(n11)) i-- dAtA[i] = 0x5a } @@ -10590,12 +10619,12 @@ func (m *PollMutableStateResponse) MarshalToSizedBuffer(dAtA []byte) (int, error dAtA[i] = 0x62 } if m.StickyTaskQueueScheduleToStartTimeout != nil { - n17, err17 := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.StickyTaskQueueScheduleToStartTimeout, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(*m.StickyTaskQueueScheduleToStartTimeout):]) - if err17 != nil { - return 0, err17 + n18, err18 := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.StickyTaskQueueScheduleToStartTimeout, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(*m.StickyTaskQueueScheduleToStartTimeout):]) + if err18 != nil { + return 0, err18 } - i -= n17 - i = encodeVarintRequestResponse(dAtA, i, uint64(n17)) + i -= n18 + i = encodeVarintRequestResponse(dAtA, i, uint64(n18)) i-- dAtA[i] = 0x5a } @@ -10872,22 +10901,22 @@ func (m *RecordWorkflowTaskStartedResponse) MarshalToSizedBuffer(dAtA []byte) (i } } if m.StartedTime != nil { - n28, err28 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.StartedTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.StartedTime):]) - if err28 != nil { - return 0, err28 + n29, err29 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.StartedTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.StartedTime):]) + if err29 != nil { + return 0, err29 } - i -= n28 - i = encodeVarintRequestResponse(dAtA, i, uint64(n28)) + i -= n29 + i = encodeVarintRequestResponse(dAtA, i, uint64(n29)) i-- dAtA[i] = 0x6a } if m.ScheduledTime != nil { - n29, err29 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.ScheduledTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.ScheduledTime):]) - if err29 != nil { - return 0, err29 + n30, err30 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.ScheduledTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.ScheduledTime):]) + if err30 != nil { + return 0, err30 } - i -= n29 - i = encodeVarintRequestResponse(dAtA, i, uint64(n29)) + i -= n30 + i = encodeVarintRequestResponse(dAtA, i, uint64(n30)) i-- dAtA[i] = 0x62 } @@ -11119,12 +11148,12 @@ func (m *RecordActivityTaskStartedResponse) MarshalToSizedBuffer(dAtA []byte) (i dAtA[i] = 0x2a } if m.CurrentAttemptScheduledTime != nil { - n39, err39 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.CurrentAttemptScheduledTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.CurrentAttemptScheduledTime):]) - if err39 != nil { - return 0, err39 + n40, err40 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.CurrentAttemptScheduledTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.CurrentAttemptScheduledTime):]) + if err40 != nil { + return 0, err40 } - i -= n39 - i = encodeVarintRequestResponse(dAtA, i, uint64(n39)) + i -= n40 + i = encodeVarintRequestResponse(dAtA, i, uint64(n40)) i-- dAtA[i] = 0x22 } @@ -11134,12 +11163,12 @@ func (m *RecordActivityTaskStartedResponse) MarshalToSizedBuffer(dAtA []byte) (i dAtA[i] = 0x18 } if m.StartedTime != nil { - n40, err40 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.StartedTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.StartedTime):]) - if err40 != nil { - return 0, err40 + n41, err41 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.StartedTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.StartedTime):]) + if err41 != nil { + return 0, err41 } - i -= n40 - i = encodeVarintRequestResponse(dAtA, i, uint64(n40)) + i -= n41 + i = encodeVarintRequestResponse(dAtA, i, uint64(n41)) i-- dAtA[i] = 0x12 } @@ -12845,12 +12874,12 @@ func (m *SyncShardStatusRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) var l int _ = l if m.StatusTime != nil { - n78, err78 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.StatusTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.StatusTime):]) - if err78 != nil { - return 0, err78 + n79, err79 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.StatusTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.StatusTime):]) + if err79 != nil { + return 0, err79 } - i -= n78 - i = encodeVarintRequestResponse(dAtA, i, uint64(n78)) + i -= n79 + i = encodeVarintRequestResponse(dAtA, i, uint64(n79)) i-- dAtA[i] = 0x1a } @@ -12961,22 +12990,22 @@ func (m *SyncActivityRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x52 } if m.LastHeartbeatTime != nil { - n82, err82 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.LastHeartbeatTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.LastHeartbeatTime):]) - if err82 != nil { - return 0, err82 + n83, err83 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.LastHeartbeatTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.LastHeartbeatTime):]) + if err83 != nil { + return 0, err83 } - i -= n82 - i = encodeVarintRequestResponse(dAtA, i, uint64(n82)) + i -= n83 + i = encodeVarintRequestResponse(dAtA, i, uint64(n83)) i-- dAtA[i] = 0x4a } if m.StartedTime != nil { - n83, err83 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.StartedTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.StartedTime):]) - if err83 != nil { - return 0, err83 + n84, err84 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.StartedTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.StartedTime):]) + if err84 != nil { + return 0, err84 } - i -= n83 - i = encodeVarintRequestResponse(dAtA, i, uint64(n83)) + i -= n84 + i = encodeVarintRequestResponse(dAtA, i, uint64(n84)) i-- dAtA[i] = 0x42 } @@ -12986,12 +13015,12 @@ func (m *SyncActivityRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x38 } if m.ScheduledTime != nil { - n84, err84 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.ScheduledTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.ScheduledTime):]) - if err84 != nil { - return 0, err84 + n85, err85 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.ScheduledTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.ScheduledTime):]) + if err85 != nil { + return 0, err85 } - i -= n84 - i = encodeVarintRequestResponse(dAtA, i, uint64(n84)) + i -= n85 + i = encodeVarintRequestResponse(dAtA, i, uint64(n85)) i-- dAtA[i] = 0x32 } @@ -13235,21 +13264,21 @@ func (m *DescribeHistoryHostResponse) MarshalToSizedBuffer(dAtA []byte) (int, er dAtA[i] = 0x1a } if len(m.ShardIds) > 0 { - dAtA91 := make([]byte, len(m.ShardIds)*10) - var j90 int + dAtA92 := make([]byte, len(m.ShardIds)*10) + var j91 int for _, num1 := range m.ShardIds { num := uint64(num1) for num >= 1<<7 { - dAtA91[j90] = uint8(uint64(num)&0x7f | 0x80) + dAtA92[j91] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j90++ + j91++ } - dAtA91[j90] = uint8(num) - j90++ + dAtA92[j91] = uint8(num) + j91++ } - i -= j90 - copy(dAtA[i:], dAtA91[:j90]) - i = encodeVarintRequestResponse(dAtA, i, uint64(j90)) + i -= j91 + copy(dAtA[i:], dAtA92[:j91]) + i = encodeVarintRequestResponse(dAtA, i, uint64(j91)) i-- dAtA[i] = 0x12 } @@ -13396,12 +13425,12 @@ func (m *RemoveTaskRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if m.VisibilityTime != nil { - n93, err93 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.VisibilityTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.VisibilityTime):]) - if err93 != nil { - return 0, err93 + n94, err94 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.VisibilityTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.VisibilityTime):]) + if err94 != nil { + return 0, err94 } - i -= n93 - i = encodeVarintRequestResponse(dAtA, i, uint64(n93)) + i -= n94 + i = encodeVarintRequestResponse(dAtA, i, uint64(n94)) i-- dAtA[i] = 0x22 } @@ -14234,12 +14263,12 @@ func (m *ShardReplicationStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) var l int _ = l if m.MaxReplicationTaskVisibilityTime != nil { - n100, err100 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.MaxReplicationTaskVisibilityTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.MaxReplicationTaskVisibilityTime):]) - if err100 != nil { - return 0, err100 + n101, err101 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.MaxReplicationTaskVisibilityTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.MaxReplicationTaskVisibilityTime):]) + if err101 != nil { + return 0, err101 } - i -= n100 - i = encodeVarintRequestResponse(dAtA, i, uint64(n100)) + i -= n101 + i = encodeVarintRequestResponse(dAtA, i, uint64(n101)) i-- dAtA[i] = 0x32 } @@ -14296,12 +14325,12 @@ func (m *ShardReplicationStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) } } if m.ShardLocalTime != nil { - n103, err103 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.ShardLocalTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.ShardLocalTime):]) - if err103 != nil { - return 0, err103 + n104, err104 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.ShardLocalTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.ShardLocalTime):]) + if err104 != nil { + return 0, err104 } - i -= n103 - i = encodeVarintRequestResponse(dAtA, i, uint64(n103)) + i -= n104 + i = encodeVarintRequestResponse(dAtA, i, uint64(n104)) i-- dAtA[i] = 0x1a } @@ -14367,12 +14396,12 @@ func (m *ShardReplicationStatusPerCluster) MarshalToSizedBuffer(dAtA []byte) (in var l int _ = l if m.AckedTaskVisibilityTime != nil { - n104, err104 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.AckedTaskVisibilityTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.AckedTaskVisibilityTime):]) - if err104 != nil { - return 0, err104 + n105, err105 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.AckedTaskVisibilityTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.AckedTaskVisibilityTime):]) + if err105 != nil { + return 0, err105 } - i -= n104 - i = encodeVarintRequestResponse(dAtA, i, uint64(n104)) + i -= n105 + i = encodeVarintRequestResponse(dAtA, i, uint64(n105)) i-- dAtA[i] = 0x12 } @@ -14470,22 +14499,22 @@ func (m *DeleteWorkflowVisibilityRecordRequest) MarshalToSizedBuffer(dAtA []byte var l int _ = l if m.WorkflowCloseTime != nil { - n106, err106 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.WorkflowCloseTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.WorkflowCloseTime):]) - if err106 != nil { - return 0, err106 + n107, err107 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.WorkflowCloseTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.WorkflowCloseTime):]) + if err107 != nil { + return 0, err107 } - i -= n106 - i = encodeVarintRequestResponse(dAtA, i, uint64(n106)) + i -= n107 + i = encodeVarintRequestResponse(dAtA, i, uint64(n107)) i-- dAtA[i] = 0x22 } if m.WorkflowStartTime != nil { - n107, err107 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.WorkflowStartTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.WorkflowStartTime):]) - if err107 != nil { - return 0, err107 + n108, err108 := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.WorkflowStartTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(*m.WorkflowStartTime):]) + if err108 != nil { + return 0, err108 } - i -= n107 - i = encodeVarintRequestResponse(dAtA, i, uint64(n107)) + i -= n108 + i = encodeVarintRequestResponse(dAtA, i, uint64(n108)) i-- dAtA[i] = 0x1a } @@ -14679,6 +14708,10 @@ func (m *StartWorkflowExecutionResponse) Size() (n int) { l = m.Clock.Size() n += 1 + l + sovRequestResponse(uint64(l)) } + if m.EagerWorkflowTask != nil { + l = m.EagerWorkflowTask.Size() + n += 1 + l + sovRequestResponse(uint64(l)) + } return n } @@ -16507,6 +16540,7 @@ func (this *StartWorkflowExecutionResponse) String() string { s := strings.Join([]string{`&StartWorkflowExecutionResponse{`, `RunId:` + fmt.Sprintf("%v", this.RunId) + `,`, `Clock:` + strings.Replace(fmt.Sprintf("%v", this.Clock), "VectorClock", "v15.VectorClock", 1) + `,`, + `EagerWorkflowTask:` + strings.Replace(fmt.Sprintf("%v", this.EagerWorkflowTask), "PollWorkflowTaskQueueResponse", "v1.PollWorkflowTaskQueueResponse", 1) + `,`, `}`, }, "") return s @@ -18118,6 +18152,42 @@ func (m *StartWorkflowExecutionResponse) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EagerWorkflowTask", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRequestResponse + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRequestResponse + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRequestResponse + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.EagerWorkflowTask == nil { + m.EagerWorkflowTask = &v1.PollWorkflowTaskQueueResponse{} + } + if err := m.EagerWorkflowTask.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRequestResponse(dAtA[iNdEx:]) diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 9cc12988144..ffe4621fa27 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1465,6 +1465,7 @@ var ( CommandTypeModifyWorkflowPropertiesCounter = NewCounterDef("modify_workflow_properties_command") CommandTypeChildWorkflowCounter = NewCounterDef("child_workflow_command") ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") + WorkflowEagerExecutionCounter = NewCounterDef("workflow_eager_execution") EmptyCompletionCommandsCounter = NewCounterDef("empty_completion_commands") MultipleCompletionCommandsCounter = NewCounterDef("multiple_completion_commands") FailedWorkflowTasksCounter = NewCounterDef("failed_workflow_tasks") diff --git a/go.mod b/go.mod index a28a3a60f9a..761d568515a 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( go.opentelemetry.io/otel/metric v0.33.0 go.opentelemetry.io/otel/sdk v1.11.1 go.opentelemetry.io/otel/sdk/metric v0.31.0 - go.temporal.io/api v1.14.1-0.20230123181040-6d7a91e07c31 + go.temporal.io/api v1.15.0 go.temporal.io/sdk v1.19.0 go.temporal.io/version v0.3.0 go.uber.org/atomic v1.10.0 @@ -55,7 +55,7 @@ require ( golang.org/x/oauth2 v0.2.0 golang.org/x/time v0.2.0 google.golang.org/api v0.103.0 - google.golang.org/grpc v1.52.0 + google.golang.org/grpc v1.52.1 google.golang.org/grpc/examples v0.0.0-20221201195934-736197138d20 gopkg.in/square/go-jose.v2 v2.6.0 gopkg.in/validator.v2 v2.0.1 @@ -127,7 +127,7 @@ require ( golang.org/x/tools v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230119192704-9d59e20e5cd1 // indirect + google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect lukechampine.com/uint128 v1.2.0 // indirect diff --git a/go.sum b/go.sum index 89491f69af6..1c44c8c2655 100644 --- a/go.sum +++ b/go.sum @@ -841,8 +841,8 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.temporal.io/api v1.13.0/go.mod h1:egkGgTG/L4wDvVsv9jK2aLWzg18oCs5ycQViN0G/jiE= -go.temporal.io/api v1.14.1-0.20230123181040-6d7a91e07c31 h1:1uSLdXq1HOjzFNzDLJxmEDC63BEBZzm1ugq4Kk3kKsc= -go.temporal.io/api v1.14.1-0.20230123181040-6d7a91e07c31/go.mod h1:iOTECB2ZzJykSBxAyMSqhIluACJgjCWszdN8wPKKjao= +go.temporal.io/api v1.15.0 h1:fCIBQsmi+zTb6r33pAZzdF5SNp9VE/imPNjmerVcI0c= +go.temporal.io/api v1.15.0/go.mod h1:dbi2T2T/PVw0jOHF0pdc8XFDFGQoMQeNq7F0ClTKwlU= go.temporal.io/sdk v1.19.0 h1:qGSU8SRbwWoJy0XJpDFtEu/PUpU7Q9oE+5OhT11je3U= go.temporal.io/sdk v1.19.0/go.mod h1:d+iTyxAZzRdhemExMVJ/9GyHv93eHO0vYeaAqcLchss= go.temporal.io/version v0.3.0 h1:dMrei9l9NyHt8nG6EB8vAwDLLTwx2SvRyucCSumAiig= @@ -1367,8 +1367,8 @@ google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZV google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20230119192704-9d59e20e5cd1 h1:wSjSSQW7LuPdv3m1IrSN33nVxH/kID6OIKy+FMwGB2k= -google.golang.org/genproto v0.0.0-20230119192704-9d59e20e5cd1/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2 h1:O97sLx/Xmb/KIZHB/2/BzofxBs5QmmR0LcihPtllmbc= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1407,8 +1407,8 @@ google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= -google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= -google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.52.1 h1:2NpOPk5g5Xtb0qebIEs7hNIa++PdtZLo2AQUpc1YnSU= +google.golang.org/grpc v1.52.1/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20221201195934-736197138d20 h1:c4fmRt11lwyTYMMEA3IYCZR/xN7FEAJpwvPziaGXugs= google.golang.org/grpc/examples v0.0.0-20221201195934-736197138d20/go.mod h1:jMMKnsR3nPEOSsgT/Le2GxpknJOBOQms0nWb0JRgeUM= diff --git a/proto/api b/proto/api index 9645a99d585..a995d4b6fae 160000 --- a/proto/api +++ b/proto/api @@ -1 +1 @@ -Subproject commit 9645a99d585b426ebb7555b8f61f5b1ed1aa42af +Subproject commit a995d4b6faeb61d0b9737042bcad08cee880fd56 diff --git a/proto/internal/temporal/server/api/historyservice/v1/request_response.proto b/proto/internal/temporal/server/api/historyservice/v1/request_response.proto index bc5b841c641..3a3f9de957d 100644 --- a/proto/internal/temporal/server/api/historyservice/v1/request_response.proto +++ b/proto/internal/temporal/server/api/historyservice/v1/request_response.proto @@ -65,6 +65,8 @@ message StartWorkflowExecutionRequest { message StartWorkflowExecutionResponse { string run_id = 1; temporal.server.api.clock.v1.VectorClock clock = 2; + // Set if request_eager_execution is set on the start request + temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse eager_workflow_task = 3; } message GetMutableStateRequest { diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 621e1452d36..a7d5dd77eb9 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -404,7 +404,7 @@ func (wh *WorkflowHandler) StartWorkflowExecution(ctx context.Context, request * if err != nil { return nil, err } - return &workflowservice.StartWorkflowExecutionResponse{RunId: resp.GetRunId()}, nil + return &workflowservice.StartWorkflowExecutionResponse{RunId: resp.GetRunId(), EagerWorkflowTask: resp.GetEagerWorkflowTask()}, nil } // GetWorkflowExecutionHistory returns the history of specified workflow execution. It fails with 'EntityNotExistError' if specified workflow @@ -2862,6 +2862,7 @@ func (wh *WorkflowHandler) GetSystemInfo(ctx context.Context, request *workflows SupportsSchedules: true, EncodedFailureAttributes: true, UpsertMemo: true, + EagerWorkflowStart: true, }, }, nil } diff --git a/service/history/api/create_workflow_util.go b/service/history/api/create_workflow_util.go index 16d44756778..3ff471c0346 100644 --- a/service/history/api/create_workflow_util.go +++ b/service/history/api/create_workflow_util.go @@ -29,6 +29,7 @@ import ( "time" commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" @@ -62,7 +63,7 @@ func NewWorkflowWithSignal( runID string, startRequest *historyservice.StartWorkflowExecutionRequest, signalWithStartRequest *workflowservice.SignalWithStartWorkflowExecutionRequest, -) (WorkflowContext, error) { +) (WorkflowContext, *workflow.WorkflowTaskInfo, error) { newMutableState, err := CreateMutableState( ctx, shard, @@ -72,7 +73,7 @@ func NewWorkflowWithSignal( runID, ) if err != nil { - return nil, err + return nil, nil, err } startEvent, err := newMutableState.AddWorkflowExecutionStartedEvent( @@ -83,7 +84,7 @@ func NewWorkflowWithSignal( startRequest, ) if err != nil { - return nil, err + return nil, nil, err } if signalWithStartRequest != nil { @@ -96,17 +97,34 @@ func NewWorkflowWithSignal( signalWithStartRequest.GetIdentity(), signalWithStartRequest.GetHeader(), ); err != nil { - return nil, err + return nil, nil, err } } - + requestEagerExecution := startRequest.StartRequest.GetRequestEagerExecution() // Generate first workflow task event if not child WF and no first workflow task backoff - if err := GenerateFirstWorkflowTask( + scheduledEventID, err := GenerateFirstWorkflowTask( newMutableState, startRequest.ParentExecutionInfo, startEvent, - ); err != nil { - return nil, err + requestEagerExecution, + ) + if err != nil { + return nil, nil, err + } + + var workflowTaskInfo *workflow.WorkflowTaskInfo + + if requestEagerExecution { + _, workflowTaskInfo, err = newMutableState.AddWorkflowTaskStartedEvent( + scheduledEventID, + startRequest.StartRequest.RequestId, + startRequest.StartRequest.TaskQueue, + startRequest.StartRequest.Identity, + ) + if err != nil { + // Unable to add WorkflowTaskStarted event to history + return nil, nil, err + } } newWorkflowContext := workflow.NewContext( @@ -118,7 +136,7 @@ func NewWorkflowWithSignal( ), shard.GetLogger(), ) - return NewWorkflowContext(newWorkflowContext, wcache.NoopReleaseFn, newMutableState), nil + return NewWorkflowContext(newWorkflowContext, wcache.NoopReleaseFn, newMutableState), workflowTaskInfo, nil } func CreateMutableState( @@ -146,17 +164,13 @@ func GenerateFirstWorkflowTask( mutableState workflow.MutableState, parentInfo *workflowspb.ParentExecutionInfo, startEvent *historypb.HistoryEvent, -) error { - + bypassTaskGeneration bool, +) (int64, error) { if parentInfo == nil { // WorkflowTask is only created when it is not a Child Workflow and no backoff is needed - if err := mutableState.AddFirstWorkflowTaskScheduled( - startEvent, - ); err != nil { - return err - } + return mutableState.AddFirstWorkflowTaskScheduled(startEvent, bypassTaskGeneration) } - return nil + return 0, nil } func NewWorkflowVersionCheck( @@ -272,6 +286,10 @@ func ValidateStartWorkflowExecutionRequest( if err := common.ValidateRetryPolicy(request.RetryPolicy); err != nil { return err } + // TODO(bergundy): Support this case + if request.GetRequestEagerExecution() && request.GetWorkflowIdReusePolicy() == enums.WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING { + return serviceerror.NewInvalidArgument("Eager workflow execution not supported for WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING") + } if err := ValidateStart( ctx, diff --git a/service/history/api/signalwithstartworkflow/signal_with_start_workflow.go b/service/history/api/signalwithstartworkflow/signal_with_start_workflow.go index 8cece6ec72b..3ccfb2cecaa 100644 --- a/service/history/api/signalwithstartworkflow/signal_with_start_workflow.go +++ b/service/history/api/signalwithstartworkflow/signal_with_start_workflow.go @@ -86,7 +86,8 @@ func startAndSignalWorkflow( ) (string, error) { workflowID := signalWithStartRequest.GetWorkflowId() runID := uuid.New().String() - newWorkflowContext, err := api.NewWorkflowWithSignal( + // TODO(bergundy): Support eager workflow task + newWorkflowContext, _, err := api.NewWorkflowWithSignal( ctx, shard, namespaceEntry, diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index a32c1571e94..84b29a82d03 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -28,9 +28,14 @@ import ( "context" "github.com/google/uuid" + "go.temporal.io/api/common/v1" + "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/api/historyservice/v1" + serverCommon "go.temporal.io/server/common" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -46,28 +51,39 @@ func Invoke( startRequest *historyservice.StartWorkflowExecutionRequest, shard shard.Context, workflowConsistencyChecker api.WorkflowConsistencyChecker, + tokenSerializer serverCommon.TaskTokenSerializer, ) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { namespaceEntry, err := api.GetActiveNamespace(shard, namespace.ID(startRequest.GetNamespaceId())) if err != nil { return nil, err } - namespaceID := namespaceEntry.ID() + namespaceID := namespaceEntry.ID().String() request := startRequest.StartRequest - api.OverrideStartWorkflowExecutionRequest(request, metrics.HistoryStartWorkflowExecutionScope, shard, shard.GetMetricsHandler()) + metricsHandler := shard.GetMetricsHandler() + + api.OverrideStartWorkflowExecutionRequest(request, metrics.HistoryStartWorkflowExecutionScope, shard, metricsHandler) err = api.ValidateStartWorkflowExecutionRequest(ctx, request, shard, namespaceEntry, "StartWorkflowExecution") if err != nil { return nil, err } + if request.GetRequestEagerExecution() { + metricsHandler.Counter(metrics.WorkflowEagerExecutionCounter.GetMetricName()).Record( + 1, + metrics.NamespaceTag(namespaceEntry.Name().String()), + metrics.TaskQueueTag(request.TaskQueue.Name), + ) + } + workflowID := request.GetWorkflowId() - runID := uuid.New() - workflowContext, err := api.NewWorkflowWithSignal( + runID := uuid.NewString() + workflowContext, workflowTaskInfo, err := api.NewWorkflowWithSignal( ctx, shard, namespaceEntry, workflowID, - runID.String(), + runID, startRequest, nil, ) @@ -102,9 +118,7 @@ func Invoke( newWorkflowEventsSeq, ) if err == nil { - return &historyservice.StartWorkflowExecutionResponse{ - RunId: runID.String(), - }, nil + return generateResponse(shard, tokenSerializer, request, namespaceID, runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) } t, ok := err.(*persistence.CurrentWorkflowConditionFailedError) @@ -112,12 +126,11 @@ func Invoke( return nil, err } - // handle CurrentWorkflowConditionFailedError + // Handle CurrentWorkflowConditionFailedError where a previous request with the same request ID already created a + // workflow execution with a different run ID. + // The history we generated above should be deleted by a background process. if t.RequestID == request.GetRequestId() { - return &historyservice.StartWorkflowExecutionResponse{ - RunId: t.RunID, - }, nil - // delete history is expected here because duplicate start request will create history with different rid + return respondToRetriedRequest(ctx, request, namespaceEntry.ID(), t.RunID, shard, workflowConsistencyChecker, tokenSerializer) } // create as ID reuse @@ -139,7 +152,7 @@ func Invoke( t.State, t.Status, workflowID, - runID.String(), + runID, startRequest.StartRequest.GetWorkflowIdReusePolicy(), ) if err != nil { @@ -153,18 +166,18 @@ func Invoke( nil, api.BypassMutableStateConsistencyPredicate, definition.NewWorkflowKey( - namespaceID.String(), + namespaceID, workflowID, prevRunID, ), prevExecutionUpdateAction, func() (workflow.Context, workflow.MutableState, error) { - workflowContext, err := api.NewWorkflowWithSignal( + workflowContext, _, err := api.NewWorkflowWithSignal( ctx, shard, namespaceEntry, workflowID, - runID.String(), + runID, startRequest, nil) if err != nil { @@ -177,8 +190,11 @@ func Invoke( ) switch err { case nil: + // We don't support TERMINATE_IF_RUNNING + eager workflow dispatch, it's okay not to return an inline task here. + // TODO(bergundy): support TERMINATE_IF_RUNNING and get newWorkflowEventsSeq + // return generateResponse(shard, tokenSerializer, request, namespaceID, runID, workflowTaskInfo, newWorkflowEventsSeq) return &historyservice.StartWorkflowExecutionResponse{ - RunId: runID.String(), + RunId: runID, }, nil case consts.ErrWorkflowCompleted: // previous workflow already closed @@ -200,8 +216,144 @@ func Invoke( ); err != nil { return nil, err } + return generateResponse(shard, tokenSerializer, request, namespaceID, runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) +} + +// respondToRetriedRequest provides a response in case a start request is retried. +func respondToRetriedRequest( + ctx context.Context, + request *workflowservice.StartWorkflowExecutionRequest, + namespaceID namespace.ID, + runID string, + shard shard.Context, + workflowConsistencyChecker api.WorkflowConsistencyChecker, + tokenSerializer serverCommon.TaskTokenSerializer, +) (*historyservice.StartWorkflowExecutionResponse, error) { + if !request.GetRequestEagerExecution() { + return &historyservice.StartWorkflowExecutionResponse{ + RunId: runID, + }, nil + } + + // For eager workflow execution, we need to get the task info and history events in order to construct a poll response. + // We techincally never want to create a new execution but in practice this should not happen. + workflowContext, releaseFn, err := workflowConsistencyChecker.GetWorkflowCache().GetOrCreateWorkflowExecution( + ctx, + namespaceID, + common.WorkflowExecution{WorkflowId: request.WorkflowId, RunId: runID}, + workflow.CallerTypeAPI, + ) + if err != nil { + return nil, err + } + defer func() { + // TODO: figure out which err to pass here + releaseFn(err) + }() + mutableState, err := workflowContext.LoadMutableState(ctx) + if err != nil { + return nil, err + } + + // Future work extend the task timeout (by failing / timing out the current task). + workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() + + // The current workflow task is not inflight or not the first task or we exceeded the first attempt and fell back to + // matching based dispatch. + if !hasInflight || workflowTaskInfo.StartedEventID != 3 || workflowTaskInfo.Attempt > 1 { + // TODO: show error here or return just the RunID with an empty poll response? + // TODO: which error? + return nil, serviceerror.NewAlreadyExist("Workflow task cannot be delivered") + } + + branchToken, err := mutableState.GetCurrentBranchToken() + if err != nil { + return nil, err + } + + var events []*history.HistoryEvent + + // Future optimization: generate the task from mutable state to save the extra DB read. + // NOTE: While unlikely that there'll be more than one page, it's safer to make less assumptions + for { + response, err := shard.GetExecutionManager().ReadHistoryBranch(ctx, &persistence.ReadHistoryBranchRequest{ + ShardID: shard.GetShardID(), + BranchToken: branchToken, + MinEventID: 1, + MaxEventID: workflowTaskInfo.StartedEventID, + PageSize: 1024, + }) + if err != nil { + return nil, err + } + events = append(events, response.HistoryEvents...) + if response.NextPageToken == nil || len(response.NextPageToken) == 0 { + break + } + } + + return generateResponse(shard, tokenSerializer, request, namespaceID.String(), runID, workflowTaskInfo, events) +} + +// extractHistoryEvents extracts all history events from a batch of events sent to persistence. +// It's unlikely that persistence events would span multiple batches but better safe than sorry. +func extractHistoryEvents(persistenceEvents []*persistence.WorkflowEvents) []*history.HistoryEvent { + if len(persistenceEvents) == 1 { + return persistenceEvents[0].Events + } + var events []*history.HistoryEvent + for _, page := range persistenceEvents { + events = append(events, page.Events...) + } + return events +} + +func generateResponse( + shard shard.Context, + tokenSerializer serverCommon.TaskTokenSerializer, + request *workflowservice.StartWorkflowExecutionRequest, + namespaceID string, + runID string, + workflowTaskInfo *workflow.WorkflowTaskInfo, + historyEvents []*history.HistoryEvent, +) (*historyservice.StartWorkflowExecutionResponse, error) { + workflowID := request.WorkflowId + if !request.GetRequestEagerExecution() { + return &historyservice.StartWorkflowExecutionResponse{ + RunId: runID, + }, nil + } + clock, err := shard.NewVectorClock() + if err != nil { + return nil, err + } + taskToken := &tokenspb.Task{ + NamespaceId: namespaceID, + WorkflowId: workflowID, + RunId: runID, + ScheduledEventId: workflowTaskInfo.ScheduledEventID, + Attempt: 1, + Clock: clock, + } + serializedToken, err := tokenSerializer.Serialize(taskToken) + if err != nil { + return nil, err + } return &historyservice.StartWorkflowExecutionResponse{ - RunId: runID.String(), + RunId: runID, + Clock: clock, + EagerWorkflowTask: &workflowservice.PollWorkflowTaskQueueResponse{ + TaskToken: serializedToken, + WorkflowExecution: &common.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, + WorkflowType: request.GetWorkflowType(), + PreviousStartedEventId: 0, + StartedEventId: workflowTaskInfo.StartedEventID, + Attempt: 1, + History: &history.History{Events: historyEvents}, + NextPageToken: nil, + WorkflowExecutionTaskQueue: workflowTaskInfo.TaskQueue, + ScheduledTime: workflowTaskInfo.ScheduledTime, + StartedTime: workflowTaskInfo.StartedTime, + }, }, nil - } diff --git a/service/history/handler.go b/service/history/handler.go index 5e5191f6732..1eda1289365 100644 --- a/service/history/handler.go +++ b/service/history/handler.go @@ -530,7 +530,11 @@ func (h *Handler) StartWorkflowExecution(ctx context.Context, request *historyse if err != nil { return nil, h.convertError(err) } - response.Clock, err = shardContext.NewVectorClock() + if response.Clock == nil { + // TODO: do we still need to create a new clock with "eager" execution? + // We already put the clock in the task token and the response in that case. + response.Clock, err = shardContext.NewVectorClock() + } if err != nil { return nil, h.convertError(err) } diff --git a/service/history/historyEngine.go b/service/history/historyEngine.go index 203610e3a97..dd83e825190 100644 --- a/service/history/historyEngine.go +++ b/service/history/historyEngine.go @@ -328,7 +328,7 @@ func (e *historyEngineImpl) StartWorkflowExecution( ctx context.Context, startRequest *historyservice.StartWorkflowExecutionRequest, ) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { - return startworkflow.Invoke(ctx, startRequest, e.shard, e.workflowConsistencyChecker) + return startworkflow.Invoke(ctx, startRequest, e.shard, e.workflowConsistencyChecker, e.tokenSerializer) } // GetMutableState retrieves the mutable state of the workflow execution diff --git a/service/history/historyEngine_test.go b/service/history/historyEngine_test.go index 795d1f1eb4e..4ffb388814e 100644 --- a/service/history/historyEngine_test.go +++ b/service/history/historyEngine_test.go @@ -46,6 +46,7 @@ import ( "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" + "google.golang.org/grpc" clockspb "go.temporal.io/server/api/clock/v1" enumsspb "go.temporal.io/server/api/enums/v1" @@ -68,6 +69,7 @@ import ( "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/primitives/timestamp" + "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" @@ -5066,6 +5068,40 @@ func (s *engineSuite) TestReapplyEvents_ResetWorkflow() { s.NoError(err) } +func (s *engineSuite) TestEagerWorkflowDispatch_DoesNotCreateTransferTask() { + var recordedTasks []tasks.Task + + s.mockExecutionMgr.EXPECT().CreateWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, request *persistence.CreateWorkflowExecutionRequest) (*persistence.CreateWorkflowExecutionResponse, error) { + recordedTasks = request.NewWorkflowSnapshot.Tasks[tasks.CategoryTransfer] + persistenceResponse := persistence.CreateWorkflowExecutionResponse{NewMutableStateStats: tests.CreateWorkflowExecutionResponse.NewMutableStateStats} + return &persistenceResponse, nil + }) + + interceptor := interceptor.NewTelemetryInterceptor(s.mockShard.GetNamespaceRegistry(), s.mockShard.GetMetricsHandler(), s.mockShard.Resource.Logger) + response, err := interceptor.Intercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req interface{}) (interface{}, error) { + response, err := s.mockHistoryEngine.StartWorkflowExecution(ctx, &historyservice.StartWorkflowExecutionRequest{ + NamespaceId: tests.NamespaceID.String(), + Attempt: 1, + StartRequest: &workflowservice.StartWorkflowExecutionRequest{ + WorkflowId: "test", + Namespace: tests.Namespace.String(), + WorkflowType: &commonpb.WorkflowType{Name: "test"}, + TaskQueue: &taskqueuepb.TaskQueue{Kind: enumspb.TASK_QUEUE_KIND_NORMAL, Name: "test"}, + Identity: "test", + RequestId: "test", + RequestEagerExecution: true, + }, + }) + return response, err + }) + s.NoError(err) + s.Equal(len(response.(*historyservice.StartWorkflowExecutionResponse).EagerWorkflowTask.History.Events), 3) + s.Equal(response.(*historyservice.StartWorkflowExecutionResponse).EagerWorkflowTask.History.Events[0].EventType, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED) + s.Equal(response.(*historyservice.StartWorkflowExecutionResponse).EagerWorkflowTask.History.Events[1].EventType, enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED) + s.Equal(response.(*historyservice.StartWorkflowExecutionResponse).EagerWorkflowTask.History.Events[2].EventType, enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED) + s.Equal(len(recordedTasks), 0) +} + func (s *engineSuite) getMutableState(testNamespaceID namespace.ID, we commonpb.WorkflowExecution) workflow.MutableState { context, release, err := s.workflowCache.GetOrCreateWorkflowExecution( context.Background(), diff --git a/service/history/workflow/mutable_state.go b/service/history/workflow/mutable_state.go index 1b4466d8901..e7e82311be0 100644 --- a/service/history/workflow/mutable_state.go +++ b/service/history/workflow/mutable_state.go @@ -109,7 +109,7 @@ type ( AddWorkflowTaskCompletedEvent(int64, int64, *workflowservice.RespondWorkflowTaskCompletedRequest, int) (*historypb.HistoryEvent, error) AddWorkflowTaskFailedEvent(scheduledEventID int64, startedEventID int64, cause enumspb.WorkflowTaskFailedCause, failure *failurepb.Failure, identity, binChecksum, baseRunID, newRunID string, forkEventVersion int64) (*historypb.HistoryEvent, error) AddWorkflowTaskScheduleToStartTimeoutEvent(int64) (*historypb.HistoryEvent, error) - AddFirstWorkflowTaskScheduled(*historypb.HistoryEvent) error + AddFirstWorkflowTaskScheduled(event *historypb.HistoryEvent, bypassTaskGeneration bool) (int64, error) AddWorkflowTaskScheduledEvent(bypassTaskGeneration bool, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error) AddWorkflowTaskScheduledEventAsHeartbeat(bypassTaskGeneration bool, originalScheduledTimestamp *time.Time, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error) AddWorkflowTaskStartedEvent(int64, string, *taskqueuepb.TaskQueue, string) (*historypb.HistoryEvent, *WorkflowTaskInfo, error) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 31435f7124f..bee1fbb1d8a 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -1506,7 +1506,7 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( if err != nil { return nil, err } - if err = ms.AddFirstWorkflowTaskScheduled(event); err != nil { + if _, err = ms.AddFirstWorkflowTaskScheduled(event, false); err != nil { return nil, err } @@ -1708,14 +1708,18 @@ func (ms *MutableStateImpl) ReplicateWorkflowExecutionStartedEvent( return nil } +// AddFirstWorkflowTaskScheduled adds the first workflow task scehduled event unless it should be delayed as indicated +// by the startEvent's FirstWorkflowTaskBackoff. +// Returns the workflow task's scheduled event ID if a task was scheduled, 0 otherwise. func (ms *MutableStateImpl) AddFirstWorkflowTaskScheduled( startEvent *historypb.HistoryEvent, -) error { + bypassTaskGeneration bool, +) (int64, error) { opTag := tag.WorkflowActionWorkflowTaskScheduled if err := ms.checkMutability(opTag); err != nil { - return err + return 0, err } - return ms.workflowTaskManager.AddFirstWorkflowTaskScheduled(startEvent) + return ms.workflowTaskManager.AddFirstWorkflowTaskScheduled(startEvent, bypassTaskGeneration) } func (ms *MutableStateImpl) AddWorkflowTaskScheduledEvent( diff --git a/service/history/workflow/mutable_state_mock.go b/service/history/workflow/mutable_state_mock.go index 100289bc157..22fd53625de 100644 --- a/service/history/workflow/mutable_state_mock.go +++ b/service/history/workflow/mutable_state_mock.go @@ -350,11 +350,12 @@ func (mr *MockMutableStateMockRecorder) AddFailWorkflowEvent(arg0, arg1, arg2, a } // AddFirstWorkflowTaskScheduled mocks base method. -func (m *MockMutableState) AddFirstWorkflowTaskScheduled(arg0 *v13.HistoryEvent) error { +func (m *MockMutableState) AddFirstWorkflowTaskScheduled(arg0 *v13.HistoryEvent, arg1 bool) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddFirstWorkflowTaskScheduled", arg0) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "AddFirstWorkflowTaskScheduled", arg0, arg1) + ret0, _ := ret[0].(int64) + ret1, _ := ret[0].(error) + return ret0, ret1 } // AddFirstWorkflowTaskScheduled indicates an expected call of AddFirstWorkflowTaskScheduled. diff --git a/service/history/workflow/retry.go b/service/history/workflow/retry.go index 1a177fc4844..68ac6f851ae 100644 --- a/service/history/workflow/retry.go +++ b/service/history/workflow/retry.go @@ -266,9 +266,7 @@ func SetupNewWorkflowForRetryOrCron( if err != nil { return serviceerror.NewInternal("Failed to add workflow execution started event.") } - if err = newMutableState.AddFirstWorkflowTaskScheduled( - event, - ); err != nil { + if _, err = newMutableState.AddFirstWorkflowTaskScheduled(event, false); err != nil { return err } diff --git a/service/history/workflow/workflow_task_state_machine.go b/service/history/workflow/workflow_task_state_machine.go index c46d9a92c81..f2e4e16f952 100644 --- a/service/history/workflow/workflow_task_state_machine.go +++ b/service/history/workflow/workflow_task_state_machine.go @@ -251,6 +251,7 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskScheduleToStartTimeoutEvent( } // AddWorkflowTaskScheduledEventAsHeartbeat is to record the first scheduled workflow task during workflow task heartbeat. +// If bypassTaskGeneration is specified, a transfer task will not be generated. func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEventAsHeartbeat( bypassTaskGeneration bool, originalScheduledTimestamp *time.Time, @@ -344,6 +345,8 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEventAsHeartbeat( return workflowTask, nil } +// AddWorkflowTaskScheduledEvent adds a WorkflowTaskScheduled event to the mutable state and generates a transfer task +// unless bypassTaskGeneration is specified. func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEvent( bypassTaskGeneration bool, workflowTaskType enumsspb.WorkflowTaskType, @@ -351,11 +354,14 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEvent( return m.AddWorkflowTaskScheduledEventAsHeartbeat(bypassTaskGeneration, timestamp.TimePtr(m.ms.timeSource.Now()), workflowTaskType) } +// AddFirstWorkflowTaskScheduled adds the first workflow task scehduled event unless it should be delayed as indicated +// by the startEvent's FirstWorkflowTaskBackoff. +// If bypassTaskGeneration is specified, a transfer task will not be created. +// Returns the workflow task's scheduled event ID if a task was scheduled, 0 otherwise. func (m *workflowTaskStateMachine) AddFirstWorkflowTaskScheduled( startEvent *historypb.HistoryEvent, -) error { - // handle first workflow task case, i.e. possible delayed workflow task - // + bypassTaskGeneration bool, +) (int64, error) { // below handles the following cases: // 1. if not continue as new & if workflow has no parent // -> schedule workflow task & schedule delayed workflow task @@ -366,27 +372,25 @@ func (m *workflowTaskStateMachine) AddFirstWorkflowTaskScheduled( // if continue as new // 1. whether has parent workflow or not // -> schedule workflow task & schedule delayed workflow task - // + startAttr := startEvent.GetWorkflowExecutionStartedEventAttributes() workflowTaskBackoffDuration := timestamp.DurationValue(startAttr.GetFirstWorkflowTaskBackoff()) - var err error if workflowTaskBackoffDuration != 0 { - if err = m.ms.taskGenerator.GenerateDelayedWorkflowTasks( + err := m.ms.taskGenerator.GenerateDelayedWorkflowTasks( startEvent, - ); err != nil { - return err - } + ) + return 0, err } else { - if _, err = m.AddWorkflowTaskScheduledEvent( - false, + info, err := m.AddWorkflowTaskScheduledEvent( + bypassTaskGeneration, enumsspb.WORKFLOW_TASK_TYPE_NORMAL, - ); err != nil { - return err + ) + if err != nil { + return 0, err } + return info.ScheduledEventID, nil } - - return nil } func (m *workflowTaskStateMachine) AddWorkflowTaskStartedEvent( diff --git a/service/history/workflowTaskHandlerCallbacks.go b/service/history/workflowTaskHandlerCallbacks.go index 2df2257db74..451731a79dc 100644 --- a/service/history/workflowTaskHandlerCallbacks.go +++ b/service/history/workflowTaskHandlerCallbacks.go @@ -146,9 +146,7 @@ func (handler *workflowTaskHandlerCallbacksImpl) handleWorkflowTaskScheduled( if err != nil { return nil, err } - if err := mutableState.AddFirstWorkflowTaskScheduled( - startEvent, - ); err != nil { + if _, err := mutableState.AddFirstWorkflowTaskScheduled(startEvent, false); err != nil { return nil, err } From 5f23b7c89224917dc88039f8ca60f2104d833af9 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Tue, 24 Jan 2023 17:39:14 -0800 Subject: [PATCH 02/17] Revert returning workflowTaskInfo from NewWorkflowWithSignal --- service/history/api/create_workflow_util.go | 18 ++++++++---------- .../signal_with_start_workflow.go | 2 +- service/history/api/startworkflow/api.go | 9 +++++++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/service/history/api/create_workflow_util.go b/service/history/api/create_workflow_util.go index 3ff471c0346..7722ba39d7f 100644 --- a/service/history/api/create_workflow_util.go +++ b/service/history/api/create_workflow_util.go @@ -63,7 +63,7 @@ func NewWorkflowWithSignal( runID string, startRequest *historyservice.StartWorkflowExecutionRequest, signalWithStartRequest *workflowservice.SignalWithStartWorkflowExecutionRequest, -) (WorkflowContext, *workflow.WorkflowTaskInfo, error) { +) (WorkflowContext, error) { newMutableState, err := CreateMutableState( ctx, shard, @@ -73,7 +73,7 @@ func NewWorkflowWithSignal( runID, ) if err != nil { - return nil, nil, err + return nil, err } startEvent, err := newMutableState.AddWorkflowExecutionStartedEvent( @@ -84,7 +84,7 @@ func NewWorkflowWithSignal( startRequest, ) if err != nil { - return nil, nil, err + return nil, err } if signalWithStartRequest != nil { @@ -97,7 +97,7 @@ func NewWorkflowWithSignal( signalWithStartRequest.GetIdentity(), signalWithStartRequest.GetHeader(), ); err != nil { - return nil, nil, err + return nil, err } } requestEagerExecution := startRequest.StartRequest.GetRequestEagerExecution() @@ -109,13 +109,11 @@ func NewWorkflowWithSignal( requestEagerExecution, ) if err != nil { - return nil, nil, err + return nil, err } - var workflowTaskInfo *workflow.WorkflowTaskInfo - if requestEagerExecution { - _, workflowTaskInfo, err = newMutableState.AddWorkflowTaskStartedEvent( + _, _, err = newMutableState.AddWorkflowTaskStartedEvent( scheduledEventID, startRequest.StartRequest.RequestId, startRequest.StartRequest.TaskQueue, @@ -123,7 +121,7 @@ func NewWorkflowWithSignal( ) if err != nil { // Unable to add WorkflowTaskStarted event to history - return nil, nil, err + return nil, err } } @@ -136,7 +134,7 @@ func NewWorkflowWithSignal( ), shard.GetLogger(), ) - return NewWorkflowContext(newWorkflowContext, wcache.NoopReleaseFn, newMutableState), workflowTaskInfo, nil + return NewWorkflowContext(newWorkflowContext, wcache.NoopReleaseFn, newMutableState), nil } func CreateMutableState( diff --git a/service/history/api/signalwithstartworkflow/signal_with_start_workflow.go b/service/history/api/signalwithstartworkflow/signal_with_start_workflow.go index 3ccfb2cecaa..a41b63f43ac 100644 --- a/service/history/api/signalwithstartworkflow/signal_with_start_workflow.go +++ b/service/history/api/signalwithstartworkflow/signal_with_start_workflow.go @@ -87,7 +87,7 @@ func startAndSignalWorkflow( workflowID := signalWithStartRequest.GetWorkflowId() runID := uuid.New().String() // TODO(bergundy): Support eager workflow task - newWorkflowContext, _, err := api.NewWorkflowWithSignal( + newWorkflowContext, err := api.NewWorkflowWithSignal( ctx, shard, namespaceEntry, diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 84b29a82d03..a3da18e2889 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -78,7 +78,7 @@ func Invoke( workflowID := request.GetWorkflowId() runID := uuid.NewString() - workflowContext, workflowTaskInfo, err := api.NewWorkflowWithSignal( + workflowContext, err := api.NewWorkflowWithSignal( ctx, shard, namespaceEntry, @@ -92,6 +92,11 @@ func Invoke( } now := shard.GetTimeSource().Now() + mutableState := workflowContext.GetMutableState() + workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() + if !hasInflight { + return nil, serviceerror.NewInternal("unexpected error: mutable state did not have an inflight workflow task") + } newWorkflow, newWorkflowEventsSeq, err := workflowContext.GetMutableState().CloseTransactionAsSnapshot( now, workflow.TransactionPolicyActive, @@ -172,7 +177,7 @@ func Invoke( ), prevExecutionUpdateAction, func() (workflow.Context, workflow.MutableState, error) { - workflowContext, _, err := api.NewWorkflowWithSignal( + workflowContext, err := api.NewWorkflowWithSignal( ctx, shard, namespaceEntry, From bb221c7896249f23173e48e9edf6293ec418e359 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Tue, 24 Jan 2023 21:16:27 -0800 Subject: [PATCH 03/17] Fix lint issues --- service/history/api/startworkflow/api.go | 68 ++++++++++++++---------- service/history/historyEngine.go | 7 ++- service/history/historyEngine_test.go | 4 +- 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index a3da18e2889..1cb9118c47b 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -46,24 +46,32 @@ import ( "go.temporal.io/server/service/history/workflow" ) +type StartArgs struct { + ShardCtx shard.Context + WorkflowConsistencyChecker api.WorkflowConsistencyChecker + TokenSerializer serverCommon.TaskTokenSerializer + Request *historyservice.StartWorkflowExecutionRequest +} + func Invoke( ctx context.Context, - startRequest *historyservice.StartWorkflowExecutionRequest, - shard shard.Context, - workflowConsistencyChecker api.WorkflowConsistencyChecker, - tokenSerializer serverCommon.TaskTokenSerializer, + args *StartArgs, ) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { - namespaceEntry, err := api.GetActiveNamespace(shard, namespace.ID(startRequest.GetNamespaceId())) + shardCtx := args.ShardCtx + workflowConsistencyChecker := args.WorkflowConsistencyChecker + startRequest := args.Request + + namespaceEntry, err := api.GetActiveNamespace(shardCtx, namespace.ID(startRequest.GetNamespaceId())) if err != nil { return nil, err } namespaceID := namespaceEntry.ID().String() request := startRequest.StartRequest - metricsHandler := shard.GetMetricsHandler() + metricsHandler := shardCtx.GetMetricsHandler() - api.OverrideStartWorkflowExecutionRequest(request, metrics.HistoryStartWorkflowExecutionScope, shard, metricsHandler) - err = api.ValidateStartWorkflowExecutionRequest(ctx, request, shard, namespaceEntry, "StartWorkflowExecution") + api.OverrideStartWorkflowExecutionRequest(request, metrics.HistoryStartWorkflowExecutionScope, shardCtx, metricsHandler) + err = api.ValidateStartWorkflowExecutionRequest(ctx, request, shardCtx, namespaceEntry, "StartWorkflowExecution") if err != nil { return nil, err } @@ -80,7 +88,7 @@ func Invoke( runID := uuid.NewString() workflowContext, err := api.NewWorkflowWithSignal( ctx, - shard, + shardCtx, namespaceEntry, workflowID, runID, @@ -91,7 +99,7 @@ func Invoke( return nil, err } - now := shard.GetTimeSource().Now() + now := shardCtx.GetTimeSource().Now() mutableState := workflowContext.GetMutableState() workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() if !hasInflight { @@ -123,7 +131,7 @@ func Invoke( newWorkflowEventsSeq, ) if err == nil { - return generateResponse(shard, tokenSerializer, request, namespaceID, runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) + return generateResponse(args, namespaceID, runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) } t, ok := err.(*persistence.CurrentWorkflowConditionFailedError) @@ -135,14 +143,14 @@ func Invoke( // workflow execution with a different run ID. // The history we generated above should be deleted by a background process. if t.RequestID == request.GetRequestId() { - return respondToRetriedRequest(ctx, request, namespaceEntry.ID(), t.RunID, shard, workflowConsistencyChecker, tokenSerializer) + return respondToRetriedRequest(ctx, args, namespaceEntry.ID(), t.RunID) } // create as ID reuse prevRunID = t.RunID prevLastWriteVersion = t.LastWriteVersion if workflowContext.GetMutableState().GetCurrentVersion() < prevLastWriteVersion { - clusterMetadata := shard.GetClusterMetadata() + clusterMetadata := shardCtx.GetClusterMetadata() clusterName := clusterMetadata.ClusterNameForFailoverVersion(namespaceEntry.IsGlobalNamespace(), prevLastWriteVersion) return nil, serviceerror.NewNamespaceNotActive( request.GetNamespace(), @@ -179,7 +187,7 @@ func Invoke( func() (workflow.Context, workflow.MutableState, error) { workflowContext, err := api.NewWorkflowWithSignal( ctx, - shard, + shardCtx, namespaceEntry, workflowID, runID, @@ -190,7 +198,7 @@ func Invoke( } return workflowContext.GetContext(), workflowContext.GetMutableState(), nil }, - shard, + shardCtx, workflowConsistencyChecker, ) switch err { @@ -221,19 +229,20 @@ func Invoke( ); err != nil { return nil, err } - return generateResponse(shard, tokenSerializer, request, namespaceID, runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) + return generateResponse(args, namespaceID, runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) } // respondToRetriedRequest provides a response in case a start request is retried. func respondToRetriedRequest( ctx context.Context, - request *workflowservice.StartWorkflowExecutionRequest, + args *StartArgs, namespaceID namespace.ID, runID string, - shard shard.Context, - workflowConsistencyChecker api.WorkflowConsistencyChecker, - tokenSerializer serverCommon.TaskTokenSerializer, ) (*historyservice.StartWorkflowExecutionResponse, error) { + shardCtx := args.ShardCtx + workflowConsistencyChecker := args.WorkflowConsistencyChecker + request := args.Request.StartRequest + if !request.GetRequestEagerExecution() { return &historyservice.StartWorkflowExecutionResponse{ RunId: runID, @@ -266,8 +275,7 @@ func respondToRetriedRequest( // The current workflow task is not inflight or not the first task or we exceeded the first attempt and fell back to // matching based dispatch. if !hasInflight || workflowTaskInfo.StartedEventID != 3 || workflowTaskInfo.Attempt > 1 { - // TODO: show error here or return just the RunID with an empty poll response? - // TODO: which error? + // TODO: Should we error here or return just the RunID with an empty poll response? If so which error type? return nil, serviceerror.NewAlreadyExist("Workflow task cannot be delivered") } @@ -281,8 +289,8 @@ func respondToRetriedRequest( // Future optimization: generate the task from mutable state to save the extra DB read. // NOTE: While unlikely that there'll be more than one page, it's safer to make less assumptions for { - response, err := shard.GetExecutionManager().ReadHistoryBranch(ctx, &persistence.ReadHistoryBranchRequest{ - ShardID: shard.GetShardID(), + response, err := shardCtx.GetExecutionManager().ReadHistoryBranch(ctx, &persistence.ReadHistoryBranchRequest{ + ShardID: shardCtx.GetShardID(), BranchToken: branchToken, MinEventID: 1, MaxEventID: workflowTaskInfo.StartedEventID, @@ -297,7 +305,7 @@ func respondToRetriedRequest( } } - return generateResponse(shard, tokenSerializer, request, namespaceID.String(), runID, workflowTaskInfo, events) + return generateResponse(args, namespaceID.String(), runID, workflowTaskInfo, events) } // extractHistoryEvents extracts all history events from a batch of events sent to persistence. @@ -314,21 +322,23 @@ func extractHistoryEvents(persistenceEvents []*persistence.WorkflowEvents) []*hi } func generateResponse( - shard shard.Context, - tokenSerializer serverCommon.TaskTokenSerializer, - request *workflowservice.StartWorkflowExecutionRequest, + args *StartArgs, namespaceID string, runID string, workflowTaskInfo *workflow.WorkflowTaskInfo, historyEvents []*history.HistoryEvent, ) (*historyservice.StartWorkflowExecutionResponse, error) { + shardCtx := args.ShardCtx + tokenSerializer := args.TokenSerializer + request := args.Request.StartRequest workflowID := request.WorkflowId + if !request.GetRequestEagerExecution() { return &historyservice.StartWorkflowExecutionResponse{ RunId: runID, }, nil } - clock, err := shard.NewVectorClock() + clock, err := shardCtx.NewVectorClock() if err != nil { return nil, err } diff --git a/service/history/historyEngine.go b/service/history/historyEngine.go index dd83e825190..215ad6789ad 100644 --- a/service/history/historyEngine.go +++ b/service/history/historyEngine.go @@ -328,7 +328,12 @@ func (e *historyEngineImpl) StartWorkflowExecution( ctx context.Context, startRequest *historyservice.StartWorkflowExecutionRequest, ) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { - return startworkflow.Invoke(ctx, startRequest, e.shard, e.workflowConsistencyChecker, e.tokenSerializer) + return startworkflow.Invoke(ctx, &startworkflow.StartArgs{ + ShardCtx: e.shard, + WorkflowConsistencyChecker: e.workflowConsistencyChecker, + TokenSerializer: e.tokenSerializer, + Request: startRequest, + }) } // GetMutableState retrieves the mutable state of the workflow execution diff --git a/service/history/historyEngine_test.go b/service/history/historyEngine_test.go index 4ffb388814e..2f10a3c924e 100644 --- a/service/history/historyEngine_test.go +++ b/service/history/historyEngine_test.go @@ -5077,8 +5077,8 @@ func (s *engineSuite) TestEagerWorkflowDispatch_DoesNotCreateTransferTask() { return &persistenceResponse, nil }) - interceptor := interceptor.NewTelemetryInterceptor(s.mockShard.GetNamespaceRegistry(), s.mockShard.GetMetricsHandler(), s.mockShard.Resource.Logger) - response, err := interceptor.Intercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req interface{}) (interface{}, error) { + i := interceptor.NewTelemetryInterceptor(s.mockShard.GetNamespaceRegistry(), s.mockShard.GetMetricsHandler(), s.mockShard.Resource.Logger) + response, err := i.Intercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req interface{}) (interface{}, error) { response, err := s.mockHistoryEngine.StartWorkflowExecution(ctx, &historyservice.StartWorkflowExecutionRequest{ NamespaceId: tests.NamespaceID.String(), Attempt: 1, From c3a4cd9ec78d79c33c95d2978b492b69141d3766 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 25 Jan 2023 11:15:33 -0800 Subject: [PATCH 04/17] Minor restructuring --- service/history/api/startworkflow/api.go | 90 +++++++++++++++--------- service/history/historyEngine.go | 16 +++-- 2 files changed, 66 insertions(+), 40 deletions(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 1cb9118c47b..547ee340b63 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -46,26 +46,45 @@ import ( "go.temporal.io/server/service/history/workflow" ) -type StartArgs struct { - ShardCtx shard.Context - WorkflowConsistencyChecker api.WorkflowConsistencyChecker - TokenSerializer serverCommon.TaskTokenSerializer - Request *historyservice.StartWorkflowExecutionRequest +type Starter struct { + shardCtx shard.Context + workflowConsistencyChecker api.WorkflowConsistencyChecker + tokenSerializer serverCommon.TaskTokenSerializer + request *historyservice.StartWorkflowExecutionRequest + namespaceID namespace.ID } -func Invoke( +func NewStarter( + shardCtx shard.Context, + workflowConsistencyChecker api.WorkflowConsistencyChecker, + tokenSerializer serverCommon.TaskTokenSerializer, + request *historyservice.StartWorkflowExecutionRequest, +) (*Starter, error) { + namespaceEntry, err := api.GetActiveNamespace(shardCtx, namespace.ID(request.GetNamespaceId())) + if err != nil { + return nil, err + } + return &Starter{ + shardCtx: shardCtx, + workflowConsistencyChecker: workflowConsistencyChecker, + tokenSerializer: tokenSerializer, + request: request, + namespaceID: namespaceEntry.ID(), + }, nil +} + +func (s *Starter) Invoke( ctx context.Context, - args *StartArgs, ) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { - shardCtx := args.ShardCtx - workflowConsistencyChecker := args.WorkflowConsistencyChecker - startRequest := args.Request + shardCtx := s.shardCtx + workflowConsistencyChecker := s.workflowConsistencyChecker + startRequest := s.request namespaceEntry, err := api.GetActiveNamespace(shardCtx, namespace.ID(startRequest.GetNamespaceId())) if err != nil { return nil, err } - namespaceID := namespaceEntry.ID().String() + s.namespaceID = namespaceEntry.ID() request := startRequest.StartRequest metricsHandler := shardCtx.GetMetricsHandler() @@ -131,7 +150,7 @@ func Invoke( newWorkflowEventsSeq, ) if err == nil { - return generateResponse(args, namespaceID, runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) + return s.generateResponse(runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) } t, ok := err.(*persistence.CurrentWorkflowConditionFailedError) @@ -143,7 +162,7 @@ func Invoke( // workflow execution with a different run ID. // The history we generated above should be deleted by a background process. if t.RequestID == request.GetRequestId() { - return respondToRetriedRequest(ctx, args, namespaceEntry.ID(), t.RunID) + return s.respondToRetriedRequest(ctx, t.RunID) } // create as ID reuse @@ -179,7 +198,7 @@ func Invoke( nil, api.BypassMutableStateConsistencyPredicate, definition.NewWorkflowKey( - namespaceID, + s.namespaceID.String(), workflowID, prevRunID, ), @@ -229,19 +248,16 @@ func Invoke( ); err != nil { return nil, err } - return generateResponse(args, namespaceID, runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) + return s.generateResponse(runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) } // respondToRetriedRequest provides a response in case a start request is retried. -func respondToRetriedRequest( +func (s *Starter) respondToRetriedRequest( ctx context.Context, - args *StartArgs, - namespaceID namespace.ID, runID string, ) (*historyservice.StartWorkflowExecutionResponse, error) { - shardCtx := args.ShardCtx - workflowConsistencyChecker := args.WorkflowConsistencyChecker - request := args.Request.StartRequest + workflowConsistencyChecker := s.workflowConsistencyChecker + request := s.request.StartRequest if !request.GetRequestEagerExecution() { return &historyservice.StartWorkflowExecutionResponse{ @@ -253,7 +269,7 @@ func respondToRetriedRequest( // We techincally never want to create a new execution but in practice this should not happen. workflowContext, releaseFn, err := workflowConsistencyChecker.GetWorkflowCache().GetOrCreateWorkflowExecution( ctx, - namespaceID, + s.namespaceID, common.WorkflowExecution{WorkflowId: request.WorkflowId, RunId: runID}, workflow.CallerTypeAPI, ) @@ -279,21 +295,29 @@ func respondToRetriedRequest( return nil, serviceerror.NewAlreadyExist("Workflow task cannot be delivered") } + events, err := s.getWorkflowHistory(ctx, mutableState) + if err != nil { + return nil, err + } + + return s.generateResponse(runID, workflowTaskInfo, events) +} + +func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState workflow.MutableState) ([]*history.HistoryEvent, error) { branchToken, err := mutableState.GetCurrentBranchToken() if err != nil { return nil, err } var events []*history.HistoryEvent - // Future optimization: generate the task from mutable state to save the extra DB read. // NOTE: While unlikely that there'll be more than one page, it's safer to make less assumptions for { - response, err := shardCtx.GetExecutionManager().ReadHistoryBranch(ctx, &persistence.ReadHistoryBranchRequest{ - ShardID: shardCtx.GetShardID(), + response, err := s.shardCtx.GetExecutionManager().ReadHistoryBranch(ctx, &persistence.ReadHistoryBranchRequest{ + ShardID: s.shardCtx.GetShardID(), BranchToken: branchToken, MinEventID: 1, - MaxEventID: workflowTaskInfo.StartedEventID, + MaxEventID: mutableState.GetNextEventID() - 1, PageSize: 1024, }) if err != nil { @@ -305,7 +329,7 @@ func respondToRetriedRequest( } } - return generateResponse(args, namespaceID.String(), runID, workflowTaskInfo, events) + return events, nil } // extractHistoryEvents extracts all history events from a batch of events sent to persistence. @@ -321,16 +345,14 @@ func extractHistoryEvents(persistenceEvents []*persistence.WorkflowEvents) []*hi return events } -func generateResponse( - args *StartArgs, - namespaceID string, +func (s *Starter) generateResponse( runID string, workflowTaskInfo *workflow.WorkflowTaskInfo, historyEvents []*history.HistoryEvent, ) (*historyservice.StartWorkflowExecutionResponse, error) { - shardCtx := args.ShardCtx - tokenSerializer := args.TokenSerializer - request := args.Request.StartRequest + shardCtx := s.shardCtx + tokenSerializer := s.tokenSerializer + request := s.request.StartRequest workflowID := request.WorkflowId if !request.GetRequestEagerExecution() { @@ -343,7 +365,7 @@ func generateResponse( return nil, err } taskToken := &tokenspb.Task{ - NamespaceId: namespaceID, + NamespaceId: s.namespaceID.String(), WorkflowId: workflowID, RunId: runID, ScheduledEventId: workflowTaskInfo.ScheduledEventID, diff --git a/service/history/historyEngine.go b/service/history/historyEngine.go index 215ad6789ad..a41fe337fdf 100644 --- a/service/history/historyEngine.go +++ b/service/history/historyEngine.go @@ -328,12 +328,16 @@ func (e *historyEngineImpl) StartWorkflowExecution( ctx context.Context, startRequest *historyservice.StartWorkflowExecutionRequest, ) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { - return startworkflow.Invoke(ctx, &startworkflow.StartArgs{ - ShardCtx: e.shard, - WorkflowConsistencyChecker: e.workflowConsistencyChecker, - TokenSerializer: e.tokenSerializer, - Request: startRequest, - }) + starter, err := startworkflow.NewStarter( + e.shard, + e.workflowConsistencyChecker, + e.tokenSerializer, + startRequest, + ) + if err != nil { + return nil, err + } + return starter.Invoke(ctx) } // GetMutableState retrieves the mutable state of the workflow execution From e8b8fbabc572037bcb609cdb93a42772972dfdd7 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 25 Jan 2023 13:04:12 -0800 Subject: [PATCH 05/17] Support eager start with TERMINATE_IF_RUNNING --- service/history/api/create_workflow_util.go | 6 --- service/history/api/startworkflow/api.go | 57 +++++++++++---------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/service/history/api/create_workflow_util.go b/service/history/api/create_workflow_util.go index 7722ba39d7f..56539aeb94b 100644 --- a/service/history/api/create_workflow_util.go +++ b/service/history/api/create_workflow_util.go @@ -29,7 +29,6 @@ import ( "time" commonpb "go.temporal.io/api/common/v1" - "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" @@ -284,11 +283,6 @@ func ValidateStartWorkflowExecutionRequest( if err := common.ValidateRetryPolicy(request.RetryPolicy); err != nil { return err } - // TODO(bergundy): Support this case - if request.GetRequestEagerExecution() && request.GetWorkflowIdReusePolicy() == enums.WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING { - return serviceerror.NewInvalidArgument("Eager workflow execution not supported for WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING") - } - if err := ValidateStart( ctx, shard, diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 547ee340b63..ca98ac5a056 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -51,7 +51,7 @@ type Starter struct { workflowConsistencyChecker api.WorkflowConsistencyChecker tokenSerializer serverCommon.TaskTokenSerializer request *historyservice.StartWorkflowExecutionRequest - namespaceID namespace.ID + namespace *namespace.Namespace } func NewStarter( @@ -69,7 +69,7 @@ func NewStarter( workflowConsistencyChecker: workflowConsistencyChecker, tokenSerializer: tokenSerializer, request: request, - namespaceID: namespaceEntry.ID(), + namespace: namespaceEntry, }, nil } @@ -80,17 +80,11 @@ func (s *Starter) Invoke( workflowConsistencyChecker := s.workflowConsistencyChecker startRequest := s.request - namespaceEntry, err := api.GetActiveNamespace(shardCtx, namespace.ID(startRequest.GetNamespaceId())) - if err != nil { - return nil, err - } - s.namespaceID = namespaceEntry.ID() - request := startRequest.StartRequest metricsHandler := shardCtx.GetMetricsHandler() api.OverrideStartWorkflowExecutionRequest(request, metrics.HistoryStartWorkflowExecutionScope, shardCtx, metricsHandler) - err = api.ValidateStartWorkflowExecutionRequest(ctx, request, shardCtx, namespaceEntry, "StartWorkflowExecution") + err := api.ValidateStartWorkflowExecutionRequest(ctx, request, shardCtx, s.namespace, "StartWorkflowExecution") if err != nil { return nil, err } @@ -98,7 +92,7 @@ func (s *Starter) Invoke( if request.GetRequestEagerExecution() { metricsHandler.Counter(metrics.WorkflowEagerExecutionCounter.GetMetricName()).Record( 1, - metrics.NamespaceTag(namespaceEntry.Name().String()), + metrics.NamespaceTag(s.namespace.Name().String()), metrics.TaskQueueTag(request.TaskQueue.Name), ) } @@ -108,7 +102,7 @@ func (s *Starter) Invoke( workflowContext, err := api.NewWorkflowWithSignal( ctx, shardCtx, - namespaceEntry, + s.namespace, workflowID, runID, startRequest, @@ -170,7 +164,7 @@ func (s *Starter) Invoke( prevLastWriteVersion = t.LastWriteVersion if workflowContext.GetMutableState().GetCurrentVersion() < prevLastWriteVersion { clusterMetadata := shardCtx.GetClusterMetadata() - clusterName := clusterMetadata.ClusterNameForFailoverVersion(namespaceEntry.IsGlobalNamespace(), prevLastWriteVersion) + clusterName := clusterMetadata.ClusterNameForFailoverVersion(s.namespace.IsGlobalNamespace(), prevLastWriteVersion) return nil, serviceerror.NewNamespaceNotActive( request.GetNamespace(), clusterMetadata.GetCurrentClusterName(), @@ -192,13 +186,14 @@ func (s *Starter) Invoke( } if prevExecutionUpdateAction != nil { + var mutableState workflow.MutableState // update prev execution and create new execution in one transaction err := api.GetAndUpdateWorkflowWithNew( ctx, nil, api.BypassMutableStateConsistencyPredicate, definition.NewWorkflowKey( - s.namespaceID.String(), + s.namespace.ID().String(), workflowID, prevRunID, ), @@ -207,7 +202,7 @@ func (s *Starter) Invoke( workflowContext, err := api.NewWorkflowWithSignal( ctx, shardCtx, - namespaceEntry, + s.namespace, workflowID, runID, startRequest, @@ -215,19 +210,25 @@ func (s *Starter) Invoke( if err != nil { return nil, nil, err } - return workflowContext.GetContext(), workflowContext.GetMutableState(), nil + mutableState = workflowContext.GetMutableState() + return workflowContext.GetContext(), mutableState, nil }, shardCtx, workflowConsistencyChecker, ) switch err { case nil: - // We don't support TERMINATE_IF_RUNNING + eager workflow dispatch, it's okay not to return an inline task here. - // TODO(bergundy): support TERMINATE_IF_RUNNING and get newWorkflowEventsSeq - // return generateResponse(shard, tokenSerializer, request, namespaceID, runID, workflowTaskInfo, newWorkflowEventsSeq) - return &historyservice.StartWorkflowExecutionResponse{ - RunId: runID, - }, nil + if !request.GetRequestEagerExecution() { + return &historyservice.StartWorkflowExecutionResponse{ + RunId: runID, + }, nil + } + workflowTaskInfo, _ := mutableState.GetInFlightWorkflowTask() + events, err := s.getWorkflowHistory(ctx, mutableState) + if err != nil { + return nil, err + } + return s.generateResponse(runID, workflowTaskInfo, events) case consts.ErrWorkflowCompleted: // previous workflow already closed // fallthough to the logic for only creating the new workflow below @@ -256,7 +257,6 @@ func (s *Starter) respondToRetriedRequest( ctx context.Context, runID string, ) (*historyservice.StartWorkflowExecutionResponse, error) { - workflowConsistencyChecker := s.workflowConsistencyChecker request := s.request.StartRequest if !request.GetRequestEagerExecution() { @@ -267,9 +267,9 @@ func (s *Starter) respondToRetriedRequest( // For eager workflow execution, we need to get the task info and history events in order to construct a poll response. // We techincally never want to create a new execution but in practice this should not happen. - workflowContext, releaseFn, err := workflowConsistencyChecker.GetWorkflowCache().GetOrCreateWorkflowExecution( + workflowContext, releaseFn, err := s.workflowConsistencyChecker.GetWorkflowCache().GetOrCreateWorkflowExecution( ctx, - s.namespaceID, + s.namespace.ID(), common.WorkflowExecution{WorkflowId: request.WorkflowId, RunId: runID}, workflow.CallerTypeAPI, ) @@ -291,8 +291,11 @@ func (s *Starter) respondToRetriedRequest( // The current workflow task is not inflight or not the first task or we exceeded the first attempt and fell back to // matching based dispatch. if !hasInflight || workflowTaskInfo.StartedEventID != 3 || workflowTaskInfo.Attempt > 1 { - // TODO: Should we error here or return just the RunID with an empty poll response? If so which error type? - return nil, serviceerror.NewAlreadyExist("Workflow task cannot be delivered") + return nil, serviceerror.NewWorkflowExecutionAlreadyStarted( + "First workflow task is no longer inflight, retried request came in too late", + request.RequestId, + runID, + ) } events, err := s.getWorkflowHistory(ctx, mutableState) @@ -365,7 +368,7 @@ func (s *Starter) generateResponse( return nil, err } taskToken := &tokenspb.Task{ - NamespaceId: s.namespaceID.String(), + NamespaceId: s.namespace.ID().String(), WorkflowId: workflowID, RunId: runID, ScheduledEventId: workflowTaskInfo.ScheduledEventID, From fa4dccf8eb3376ef4c6904c0b6733e946d24a869 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 25 Jan 2023 14:17:35 -0800 Subject: [PATCH 06/17] Properly release workflow context --- service/history/api/startworkflow/api.go | 83 ++++++++++++++++-------- 1 file changed, 55 insertions(+), 28 deletions(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index ca98ac5a056..c2dade71341 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -186,7 +186,7 @@ func (s *Starter) Invoke( } if prevExecutionUpdateAction != nil { - var mutableState workflow.MutableState + var mutableStateInfo *MutableStateInfo // update prev execution and create new execution in one transaction err := api.GetAndUpdateWorkflowWithNew( ctx, @@ -211,6 +211,10 @@ func (s *Starter) Invoke( return nil, nil, err } mutableState = workflowContext.GetMutableState() + mutableStateInfo, err = extractMutableStateInfo(mutableState) + if err != nil { + return nil, nil, err + } return workflowContext.GetContext(), mutableState, nil }, shardCtx, @@ -223,12 +227,11 @@ func (s *Starter) Invoke( RunId: runID, }, nil } - workflowTaskInfo, _ := mutableState.GetInFlightWorkflowTask() - events, err := s.getWorkflowHistory(ctx, mutableState) + events, err := s.getWorkflowHistory(ctx, mutableStateInfo) if err != nil { return nil, err } - return s.generateResponse(runID, workflowTaskInfo, events) + return s.generateResponse(runID, mutableStateInfo.workflowTaskInfo, events) case consts.ErrWorkflowCompleted: // previous workflow already closed // fallthough to the logic for only creating the new workflow below @@ -266,31 +269,14 @@ func (s *Starter) respondToRetriedRequest( } // For eager workflow execution, we need to get the task info and history events in order to construct a poll response. - // We techincally never want to create a new execution but in practice this should not happen. - workflowContext, releaseFn, err := s.workflowConsistencyChecker.GetWorkflowCache().GetOrCreateWorkflowExecution( - ctx, - s.namespace.ID(), - common.WorkflowExecution{WorkflowId: request.WorkflowId, RunId: runID}, - workflow.CallerTypeAPI, - ) - if err != nil { - return nil, err - } - defer func() { - // TODO: figure out which err to pass here - releaseFn(err) - }() - mutableState, err := workflowContext.LoadMutableState(ctx) + mutableStateInfo, err := s.getMutableStateInfo(ctx, runID) if err != nil { return nil, err } - // Future work extend the task timeout (by failing / timing out the current task). - workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() - // The current workflow task is not inflight or not the first task or we exceeded the first attempt and fell back to // matching based dispatch. - if !hasInflight || workflowTaskInfo.StartedEventID != 3 || workflowTaskInfo.Attempt > 1 { + if !mutableStateInfo.hasInflight || mutableStateInfo.workflowTaskInfo.StartedEventID != 3 || mutableStateInfo.workflowTaskInfo.Attempt > 1 { return nil, serviceerror.NewWorkflowExecutionAlreadyStarted( "First workflow task is no longer inflight, retried request came in too late", request.RequestId, @@ -298,29 +284,70 @@ func (s *Starter) respondToRetriedRequest( ) } - events, err := s.getWorkflowHistory(ctx, mutableState) + events, err := s.getWorkflowHistory(ctx, mutableStateInfo) + if err != nil { + return nil, err + } + + return s.generateResponse(runID, mutableStateInfo.workflowTaskInfo, events) +} + +type MutableStateInfo struct { + branchToken []byte + lastEventID int64 + workflowTaskInfo *workflow.WorkflowTaskInfo + hasInflight bool +} + +func (s *Starter) getMutableStateInfo(ctx context.Context, runID string) (*MutableStateInfo, error) { + // We techincally never want to create a new execution but in practice this should not happen. + workflowContext, releaseFn, err := s.workflowConsistencyChecker.GetWorkflowCache().GetOrCreateWorkflowExecution( + ctx, + s.namespace.ID(), + common.WorkflowExecution{WorkflowId: s.request.StartRequest.WorkflowId, RunId: runID}, + workflow.CallerTypeAPI, + ) + var releaseErr error + defer func() { + releaseFn(releaseErr) + }() + if err != nil { return nil, err } - return s.generateResponse(runID, workflowTaskInfo, events) + var mutableState workflow.MutableState + mutableState, releaseErr = workflowContext.LoadMutableState(ctx) + return extractMutableStateInfo(mutableState) } -func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState workflow.MutableState) ([]*history.HistoryEvent, error) { +func extractMutableStateInfo(mutableState workflow.MutableState) (*MutableStateInfo, error) { branchToken, err := mutableState.GetCurrentBranchToken() if err != nil { return nil, err } + // Future work for the request retry path: extend the task timeout (by failing / timing out the current task). + workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() + + return &MutableStateInfo{ + branchToken: branchToken, + lastEventID: mutableState.GetNextEventID() - 1, + workflowTaskInfo: workflowTaskInfo, + hasInflight: hasInflight, + }, nil +} + +func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState *MutableStateInfo) ([]*history.HistoryEvent, error) { var events []*history.HistoryEvent // Future optimization: generate the task from mutable state to save the extra DB read. // NOTE: While unlikely that there'll be more than one page, it's safer to make less assumptions for { response, err := s.shardCtx.GetExecutionManager().ReadHistoryBranch(ctx, &persistence.ReadHistoryBranchRequest{ ShardID: s.shardCtx.GetShardID(), - BranchToken: branchToken, + BranchToken: mutableState.branchToken, MinEventID: 1, - MaxEventID: mutableState.GetNextEventID() - 1, + MaxEventID: mutableState.lastEventID, PageSize: 1024, }) if err != nil { From 4e498754b383f41d4890814fd5bc7378206af003 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 25 Jan 2023 15:11:10 -0800 Subject: [PATCH 07/17] Add documentation and restructure for better readability --- service/history/api/startworkflow/api.go | 165 +++++++++++++---------- service/history/workflow/cache/cache.go | 4 + 2 files changed, 98 insertions(+), 71 deletions(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index c2dade71341..e5736af28df 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -46,6 +46,7 @@ import ( "go.temporal.io/server/service/history/workflow" ) +// Starter starts a new workflow execution. type Starter struct { shardCtx shard.Context workflowConsistencyChecker api.WorkflowConsistencyChecker @@ -54,6 +55,16 @@ type Starter struct { namespace *namespace.Namespace } +// MutableStateInfo is a container for the relevant mutable state information to generate a start response with an eager +// workflow task. +type MutableStateInfo struct { + branchToken []byte + lastEventID int64 + workflowTaskInfo *workflow.WorkflowTaskInfo + hasInflight bool +} + +// NewStarter creates a new starter, fails if getting the active namespace fails. func NewStarter( shardCtx shard.Context, workflowConsistencyChecker api.WorkflowConsistencyChecker, @@ -73,11 +84,11 @@ func NewStarter( }, nil } +// Invoke starts a new workflow execution func (s *Starter) Invoke( ctx context.Context, ) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { shardCtx := s.shardCtx - workflowConsistencyChecker := s.workflowConsistencyChecker startRequest := s.request request := startRequest.StartRequest @@ -131,14 +142,12 @@ func (s *Starter) Invoke( // create as brand new createMode := persistence.CreateWorkflowModeBrandNew - prevRunID := "" - prevLastWriteVersion := int64(0) err = workflowContext.GetContext().CreateWorkflowExecution( ctx, now, createMode, - prevRunID, - prevLastWriteVersion, + "", // prevRunID + 0, // prevLastWriteVersion workflowContext.GetMutableState(), newWorkflow, newWorkflowEventsSeq, @@ -159,70 +168,19 @@ func (s *Starter) Invoke( return s.respondToRetriedRequest(ctx, t.RunID) } - // create as ID reuse - prevRunID = t.RunID - prevLastWriteVersion = t.LastWriteVersion - if workflowContext.GetMutableState().GetCurrentVersion() < prevLastWriteVersion { - clusterMetadata := shardCtx.GetClusterMetadata() - clusterName := clusterMetadata.ClusterNameForFailoverVersion(s.namespace.IsGlobalNamespace(), prevLastWriteVersion) + if workflowContext.GetMutableState().GetCurrentVersion() < t.LastWriteVersion { + clusterMetadata := s.shardCtx.GetClusterMetadata() + clusterName := clusterMetadata.ClusterNameForFailoverVersion(s.namespace.IsGlobalNamespace(), t.LastWriteVersion) return nil, serviceerror.NewNamespaceNotActive( - request.GetNamespace(), + s.namespace.Name().String(), clusterMetadata.GetCurrentClusterName(), clusterName, ) } - - prevExecutionUpdateAction, err := api.ApplyWorkflowIDReusePolicy( - t.RequestID, - prevRunID, - t.State, - t.Status, - workflowID, - runID, - startRequest.StartRequest.GetWorkflowIdReusePolicy(), - ) - if err != nil { - return nil, err - } - - if prevExecutionUpdateAction != nil { - var mutableStateInfo *MutableStateInfo - // update prev execution and create new execution in one transaction - err := api.GetAndUpdateWorkflowWithNew( - ctx, - nil, - api.BypassMutableStateConsistencyPredicate, - definition.NewWorkflowKey( - s.namespace.ID().String(), - workflowID, - prevRunID, - ), - prevExecutionUpdateAction, - func() (workflow.Context, workflow.MutableState, error) { - workflowContext, err := api.NewWorkflowWithSignal( - ctx, - shardCtx, - s.namespace, - workflowID, - runID, - startRequest, - nil) - if err != nil { - return nil, nil, err - } - mutableState = workflowContext.GetMutableState() - mutableStateInfo, err = extractMutableStateInfo(mutableState) - if err != nil { - return nil, nil, err - } - return workflowContext.GetContext(), mutableState, nil - }, - shardCtx, - workflowConsistencyChecker, - ) + if mutableStateInfo, err := s.applyWorkflowIDReusePolicy(ctx, t, workflowContext, runID); err != nil { switch err { case nil: - if !request.GetRequestEagerExecution() { + if !s.request.StartRequest.GetRequestEagerExecution() { return &historyservice.StartWorkflowExecutionResponse{ RunId: runID, }, nil @@ -244,8 +202,8 @@ func (s *Starter) Invoke( ctx, now, persistence.CreateWorkflowModeUpdateCurrent, - prevRunID, - prevLastWriteVersion, + t.RunID, + t.LastWriteVersion, workflowContext.GetMutableState(), newWorkflow, newWorkflowEventsSeq, @@ -255,6 +213,72 @@ func (s *Starter) Invoke( return s.generateResponse(runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) } +// applyWorkflowIDReusePolicy applies the workflow ID reuse policy in case a workflow start requests fails with a +// duplicate execution. +// At the time of this writing, the only possible action here is to terminate the current execution in case the start +// request's ID reuse policy is TERMINATE_IF_RUNNING. +// Returns non-nil MutableStateInfo if an action was required and completed successfully resulting in a newly created +// execution. +func (s *Starter) applyWorkflowIDReusePolicy( + ctx context.Context, + t *persistence.CurrentWorkflowConditionFailedError, + workflowContext api.WorkflowContext, + runID string, +) (*MutableStateInfo, error) { + workflowID := s.request.StartRequest.WorkflowId + prevExecutionUpdateAction, err := api.ApplyWorkflowIDReusePolicy( + t.RequestID, + t.RunID, + t.State, + t.Status, + workflowID, + runID, + s.request.StartRequest.GetWorkflowIdReusePolicy(), + ) + if err != nil { + return nil, err + } + + if prevExecutionUpdateAction == nil { + return nil, nil + } + var mutableStateInfo *MutableStateInfo + // update prev execution and create new execution in one transaction + err = api.GetAndUpdateWorkflowWithNew( + ctx, + nil, + api.BypassMutableStateConsistencyPredicate, + definition.NewWorkflowKey( + s.namespace.ID().String(), + workflowID, + t.RunID, + ), + prevExecutionUpdateAction, + func() (workflow.Context, workflow.MutableState, error) { + workflowContext, err := api.NewWorkflowWithSignal( + ctx, + s.shardCtx, + s.namespace, + workflowID, + runID, + s.request, + nil) + if err != nil { + return nil, nil, err + } + mutableState := workflowContext.GetMutableState() + mutableStateInfo, err = extractMutableStateInfo(mutableState) + if err != nil { + return nil, nil, err + } + return workflowContext.GetContext(), mutableState, nil + }, + s.shardCtx, + s.workflowConsistencyChecker, + ) + return mutableStateInfo, err +} + // respondToRetriedRequest provides a response in case a start request is retried. func (s *Starter) respondToRetriedRequest( ctx context.Context, @@ -292,13 +316,8 @@ func (s *Starter) respondToRetriedRequest( return s.generateResponse(runID, mutableStateInfo.workflowTaskInfo, events) } -type MutableStateInfo struct { - branchToken []byte - lastEventID int64 - workflowTaskInfo *workflow.WorkflowTaskInfo - hasInflight bool -} - +// getMutableStateInfo gets the relevant mutable state information while getting the state for the given run from the +// workflow cache and managing the cache lease. func (s *Starter) getMutableStateInfo(ctx context.Context, runID string) (*MutableStateInfo, error) { // We techincally never want to create a new execution but in practice this should not happen. workflowContext, releaseFn, err := s.workflowConsistencyChecker.GetWorkflowCache().GetOrCreateWorkflowExecution( @@ -321,6 +340,7 @@ func (s *Starter) getMutableStateInfo(ctx context.Context, runID string) (*Mutab return extractMutableStateInfo(mutableState) } +// extractMutableStateInfo extracts the relevant information to generate a start response with an eager workflow task. func extractMutableStateInfo(mutableState workflow.MutableState) (*MutableStateInfo, error) { branchToken, err := mutableState.GetCurrentBranchToken() if err != nil { @@ -338,6 +358,7 @@ func extractMutableStateInfo(mutableState workflow.MutableState) (*MutableStateI }, nil } +// getWorkflowHistory loads the workflow history based on given mutable state information from the DB. func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState *MutableStateInfo) ([]*history.HistoryEvent, error) { var events []*history.HistoryEvent // Future optimization: generate the task from mutable state to save the extra DB read. @@ -375,6 +396,8 @@ func extractHistoryEvents(persistenceEvents []*persistence.WorkflowEvents) []*hi return events } +// generateResponse is a helper for generating StartWorkflowExecutionResponse for eager and non eager workflow start +// requests. func (s *Starter) generateResponse( runID string, workflowTaskInfo *workflow.WorkflowTaskInfo, diff --git a/service/history/workflow/cache/cache.go b/service/history/workflow/cache/cache.go index bd9e35e0610..bb8e07be39d 100644 --- a/service/history/workflow/cache/cache.go +++ b/service/history/workflow/cache/cache.go @@ -48,6 +48,10 @@ import ( ) type ( + // ReleaseCacheFunc must be called to release the workflow context from the cache. + // Make sure not to access the mutable state or workflow context after releasing back to the cache. + // If there is any error when using the mutable state (e.g. mutable state is mutated and dirty), call release with + // the error so the in-memory copy will be thrown away. ReleaseCacheFunc func(err error) Cache interface { From fc50446ed1c9814717ac54c1b2b096c51c7e06c6 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 25 Jan 2023 15:33:34 -0800 Subject: [PATCH 08/17] Fix lint --- service/history/api/startworkflow/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index e5736af28df..8942c3b4a7b 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -85,6 +85,7 @@ func NewStarter( } // Invoke starts a new workflow execution +// nolint:cyclomatic // Simplified this existing function but it is still too complex func (s *Starter) Invoke( ctx context.Context, ) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { @@ -177,7 +178,7 @@ func (s *Starter) Invoke( clusterName, ) } - if mutableStateInfo, err := s.applyWorkflowIDReusePolicy(ctx, t, workflowContext, runID); err != nil { + if mutableStateInfo, err := s.applyWorkflowIDReusePolicy(ctx, t, runID); err != nil { switch err { case nil: if !s.request.StartRequest.GetRequestEagerExecution() { @@ -222,7 +223,6 @@ func (s *Starter) Invoke( func (s *Starter) applyWorkflowIDReusePolicy( ctx context.Context, t *persistence.CurrentWorkflowConditionFailedError, - workflowContext api.WorkflowContext, runID string, ) (*MutableStateInfo, error) { workflowID := s.request.StartRequest.WorkflowId From 15158f9082bf44d5d6c4d915ec174eaa04cb918a Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 25 Jan 2023 15:36:23 -0800 Subject: [PATCH 09/17] Run go-generate --- service/history/workflow/mutable_state_mock.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/service/history/workflow/mutable_state_mock.go b/service/history/workflow/mutable_state_mock.go index 22fd53625de..bc2ebc275ed 100644 --- a/service/history/workflow/mutable_state_mock.go +++ b/service/history/workflow/mutable_state_mock.go @@ -350,18 +350,18 @@ func (mr *MockMutableStateMockRecorder) AddFailWorkflowEvent(arg0, arg1, arg2, a } // AddFirstWorkflowTaskScheduled mocks base method. -func (m *MockMutableState) AddFirstWorkflowTaskScheduled(arg0 *v13.HistoryEvent, arg1 bool) (int64, error) { +func (m *MockMutableState) AddFirstWorkflowTaskScheduled(event *v13.HistoryEvent, bypassTaskGeneration bool) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddFirstWorkflowTaskScheduled", arg0, arg1) + ret := m.ctrl.Call(m, "AddFirstWorkflowTaskScheduled", event, bypassTaskGeneration) ret0, _ := ret[0].(int64) - ret1, _ := ret[0].(error) + ret1, _ := ret[1].(error) return ret0, ret1 } // AddFirstWorkflowTaskScheduled indicates an expected call of AddFirstWorkflowTaskScheduled. -func (mr *MockMutableStateMockRecorder) AddFirstWorkflowTaskScheduled(arg0 interface{}) *gomock.Call { +func (mr *MockMutableStateMockRecorder) AddFirstWorkflowTaskScheduled(event, bypassTaskGeneration interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddFirstWorkflowTaskScheduled", reflect.TypeOf((*MockMutableState)(nil).AddFirstWorkflowTaskScheduled), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddFirstWorkflowTaskScheduled", reflect.TypeOf((*MockMutableState)(nil).AddFirstWorkflowTaskScheduled), event, bypassTaskGeneration) } // AddRecordMarkerEvent mocks base method. From cc64258e31e377c85fe8918f156a37c49dbf7ce1 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 25 Jan 2023 15:38:58 -0800 Subject: [PATCH 10/17] Fix nolint directive --- service/history/api/startworkflow/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 8942c3b4a7b..158bb67e202 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -85,7 +85,7 @@ func NewStarter( } // Invoke starts a new workflow execution -// nolint:cyclomatic // Simplified this existing function but it is still too complex +// nolint:gocyclo // Simplified this existing function but it is still too complex func (s *Starter) Invoke( ctx context.Context, ) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { From 06aebdc10e096e2c3beec27c22675d0dad072562 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 25 Jan 2023 17:11:48 -0800 Subject: [PATCH 11/17] More restructuring, get rid of cyclo complexity --- service/history/api/startworkflow/api.go | 209 +++++++++++++++-------- 1 file changed, 134 insertions(+), 75 deletions(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 158bb67e202..f5f3da4e1c6 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -55,9 +55,20 @@ type Starter struct { namespace *namespace.Namespace } -// MutableStateInfo is a container for the relevant mutable state information to generate a start response with an eager +// creationContext is a container for all information obtained from creating the uncommitted execution. +// The information is later used to create a new execution and handle conflicts. +type creationContext struct { + workflowID string + runID string + workflowContext api.WorkflowContext + workflowTaskInfo *workflow.WorkflowTaskInfo + workflowSnapshot *persistence.WorkflowSnapshot + workflowEvents []*persistence.WorkflowEvents +} + +// mutableStateInfo is a container for the relevant mutable state information to generate a start response with an eager // workflow task. -type MutableStateInfo struct { +type mutableStateInfo struct { branchToken []byte lastEventID int64 workflowTaskInfo *workflow.WorkflowTaskInfo @@ -84,21 +95,15 @@ func NewStarter( }, nil } -// Invoke starts a new workflow execution -// nolint:gocyclo // Simplified this existing function but it is still too complex -func (s *Starter) Invoke( - ctx context.Context, -) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { - shardCtx := s.shardCtx - startRequest := s.request - - request := startRequest.StartRequest - metricsHandler := shardCtx.GetMetricsHandler() +// prepare applies request overrides, validates the request, and records eager execution metrics. +func (s *Starter) prepare(ctx context.Context) error { + request := s.request.StartRequest + metricsHandler := s.shardCtx.GetMetricsHandler() - api.OverrideStartWorkflowExecutionRequest(request, metrics.HistoryStartWorkflowExecutionScope, shardCtx, metricsHandler) - err := api.ValidateStartWorkflowExecutionRequest(ctx, request, shardCtx, s.namespace, "StartWorkflowExecution") + api.OverrideStartWorkflowExecutionRequest(request, metrics.HistoryStartWorkflowExecutionScope, s.shardCtx, metricsHandler) + err := api.ValidateStartWorkflowExecutionRequest(ctx, request, s.shardCtx, s.namespace, "StartWorkflowExecution") if err != nil { - return nil, err + return err } if request.GetRequestEagerExecution() { @@ -108,89 +113,121 @@ func (s *Starter) Invoke( metrics.TaskQueueTag(request.TaskQueue.Name), ) } + return nil +} + +// Invoke starts a new workflow execution +func (s *Starter) Invoke( + ctx context.Context, +) (resp *historyservice.StartWorkflowExecutionResponse, retError error) { + request := s.request.StartRequest + if err := s.prepare(ctx); err != nil { + return nil, err + } - workflowID := request.GetWorkflowId() runID := uuid.NewString() + + creationCtx, err := s.createNewMutableState(ctx, request.GetWorkflowId(), runID) + if err != nil { + return nil, err + } + err = s.createNewExecution(ctx, creationCtx) + if err == nil { + return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEvents)) + } + t, ok := err.(*persistence.CurrentWorkflowConditionFailedError) + if !ok { + return nil, err + } + return s.handleConflict(ctx, creationCtx, t) +} + +// createNewMutableState creates a new workflow context, and closes its mutable state transaction as snapshot. +// It returns the creationContext which can later be used to insert into the executions table. +func (s *Starter) createNewMutableState(ctx context.Context, workflowID string, runID string) (*creationContext, error) { workflowContext, err := api.NewWorkflowWithSignal( ctx, - shardCtx, + s.shardCtx, s.namespace, workflowID, runID, - startRequest, + s.request, nil, ) if err != nil { return nil, err } - now := shardCtx.GetTimeSource().Now() + now := s.shardCtx.GetTimeSource().Now() mutableState := workflowContext.GetMutableState() workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() if !hasInflight { return nil, serviceerror.NewInternal("unexpected error: mutable state did not have an inflight workflow task") } - newWorkflow, newWorkflowEventsSeq, err := workflowContext.GetMutableState().CloseTransactionAsSnapshot( + workflowSnapshot, workflowEvents, err := mutableState.CloseTransactionAsSnapshot( now, workflow.TransactionPolicyActive, ) if err != nil { return nil, err } - if len(newWorkflowEventsSeq) != 1 { + if len(workflowEvents) != 1 { return nil, serviceerror.NewInternal("unable to create 1st event batch") } - // create as brand new - createMode := persistence.CreateWorkflowModeBrandNew - err = workflowContext.GetContext().CreateWorkflowExecution( + return &creationContext{ + workflowID: workflowID, + runID: runID, + workflowContext: workflowContext, + workflowTaskInfo: workflowTaskInfo, + workflowSnapshot: workflowSnapshot, + workflowEvents: workflowEvents, + }, nil +} + +// createNewExecution creates a new "brand new" execution in the executions table. +func (s *Starter) createNewExecution(ctx context.Context, creationCtx *creationContext) error { + now := s.shardCtx.GetTimeSource().Now() + return creationCtx.workflowContext.GetContext().CreateWorkflowExecution( ctx, now, - createMode, + persistence.CreateWorkflowModeBrandNew, "", // prevRunID 0, // prevLastWriteVersion - workflowContext.GetMutableState(), - newWorkflow, - newWorkflowEventsSeq, + creationCtx.workflowContext.GetMutableState(), + creationCtx.workflowSnapshot, + creationCtx.workflowEvents, ) - if err == nil { - return s.generateResponse(runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) - } - - t, ok := err.(*persistence.CurrentWorkflowConditionFailedError) - if !ok { - return nil, err - } +} - // Handle CurrentWorkflowConditionFailedError where a previous request with the same request ID already created a - // workflow execution with a different run ID. - // The history we generated above should be deleted by a background process. - if t.RequestID == request.GetRequestId() { - return s.respondToRetriedRequest(ctx, t.RunID) +// handleConflict handles CurrentWorkflowConditionFailedError where a previous request with the same request ID already +// created a workflow execution with a different run ID. +// The history we generated above should be deleted by a background process. +func (s *Starter) handleConflict( + ctx context.Context, + creationCtx *creationContext, + workflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, +) (*historyservice.StartWorkflowExecutionResponse, error) { + request := s.request.StartRequest + if workflowConditionFailed.RequestID == request.GetRequestId() { + return s.respondToRetriedRequest(ctx, workflowConditionFailed.RunID) } - - if workflowContext.GetMutableState().GetCurrentVersion() < t.LastWriteVersion { - clusterMetadata := s.shardCtx.GetClusterMetadata() - clusterName := clusterMetadata.ClusterNameForFailoverVersion(s.namespace.IsGlobalNamespace(), t.LastWriteVersion) - return nil, serviceerror.NewNamespaceNotActive( - s.namespace.Name().String(), - clusterMetadata.GetCurrentClusterName(), - clusterName, - ) + if err := s.verifyNamespaceActive(creationCtx, workflowConditionFailed); err != nil { + return nil, err } - if mutableStateInfo, err := s.applyWorkflowIDReusePolicy(ctx, t, runID); err != nil { + if mutableStateInfo, err := s.applyWorkflowIDReusePolicy(ctx, workflowConditionFailed, creationCtx.runID); err != nil { switch err { case nil: - if !s.request.StartRequest.GetRequestEagerExecution() { + if !request.GetRequestEagerExecution() { return &historyservice.StartWorkflowExecutionResponse{ - RunId: runID, + RunId: creationCtx.runID, }, nil } events, err := s.getWorkflowHistory(ctx, mutableStateInfo) if err != nil { return nil, err } - return s.generateResponse(runID, mutableStateInfo.workflowTaskInfo, events) + return s.generateResponse(creationCtx.runID, mutableStateInfo.workflowTaskInfo, events) case consts.ErrWorkflowCompleted: // previous workflow already closed // fallthough to the logic for only creating the new workflow below @@ -198,20 +235,42 @@ func (s *Starter) Invoke( return nil, err } } + if err := s.updateCurrentExecution(ctx, creationCtx, workflowConditionFailed); err != nil { + return nil, err + } + return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEvents)) +} - if err = workflowContext.GetContext().CreateWorkflowExecution( +// updateCurrentExecution creates a new workflow execution and sets it to "current". +func (s *Starter) updateCurrentExecution( + ctx context.Context, + creationCtx *creationContext, + workflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, +) error { + now := s.shardCtx.GetTimeSource().Now() + return creationCtx.workflowContext.GetContext().CreateWorkflowExecution( ctx, now, persistence.CreateWorkflowModeUpdateCurrent, - t.RunID, - t.LastWriteVersion, - workflowContext.GetMutableState(), - newWorkflow, - newWorkflowEventsSeq, - ); err != nil { - return nil, err + workflowConditionFailed.RunID, + workflowConditionFailed.LastWriteVersion, + creationCtx.workflowContext.GetMutableState(), + creationCtx.workflowSnapshot, + creationCtx.workflowEvents, + ) +} + +func (s *Starter) verifyNamespaceActive(creationCtx *creationContext, workflowConditionFailed *persistence.CurrentWorkflowConditionFailedError) error { + if creationCtx.workflowContext.GetMutableState().GetCurrentVersion() < workflowConditionFailed.LastWriteVersion { + clusterMetadata := s.shardCtx.GetClusterMetadata() + clusterName := clusterMetadata.ClusterNameForFailoverVersion(s.namespace.IsGlobalNamespace(), workflowConditionFailed.LastWriteVersion) + return serviceerror.NewNamespaceNotActive( + s.namespace.Name().String(), + clusterMetadata.GetCurrentClusterName(), + clusterName, + ) } - return s.generateResponse(runID, workflowTaskInfo, extractHistoryEvents(newWorkflowEventsSeq)) + return nil } // applyWorkflowIDReusePolicy applies the workflow ID reuse policy in case a workflow start requests fails with a @@ -222,15 +281,15 @@ func (s *Starter) Invoke( // execution. func (s *Starter) applyWorkflowIDReusePolicy( ctx context.Context, - t *persistence.CurrentWorkflowConditionFailedError, + workflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, runID string, -) (*MutableStateInfo, error) { +) (*mutableStateInfo, error) { workflowID := s.request.StartRequest.WorkflowId prevExecutionUpdateAction, err := api.ApplyWorkflowIDReusePolicy( - t.RequestID, - t.RunID, - t.State, - t.Status, + workflowConditionFailed.RequestID, + workflowConditionFailed.RunID, + workflowConditionFailed.State, + workflowConditionFailed.Status, workflowID, runID, s.request.StartRequest.GetWorkflowIdReusePolicy(), @@ -242,7 +301,7 @@ func (s *Starter) applyWorkflowIDReusePolicy( if prevExecutionUpdateAction == nil { return nil, nil } - var mutableStateInfo *MutableStateInfo + var mutableStateInfo *mutableStateInfo // update prev execution and create new execution in one transaction err = api.GetAndUpdateWorkflowWithNew( ctx, @@ -251,7 +310,7 @@ func (s *Starter) applyWorkflowIDReusePolicy( definition.NewWorkflowKey( s.namespace.ID().String(), workflowID, - t.RunID, + workflowConditionFailed.RunID, ), prevExecutionUpdateAction, func() (workflow.Context, workflow.MutableState, error) { @@ -318,7 +377,7 @@ func (s *Starter) respondToRetriedRequest( // getMutableStateInfo gets the relevant mutable state information while getting the state for the given run from the // workflow cache and managing the cache lease. -func (s *Starter) getMutableStateInfo(ctx context.Context, runID string) (*MutableStateInfo, error) { +func (s *Starter) getMutableStateInfo(ctx context.Context, runID string) (*mutableStateInfo, error) { // We techincally never want to create a new execution but in practice this should not happen. workflowContext, releaseFn, err := s.workflowConsistencyChecker.GetWorkflowCache().GetOrCreateWorkflowExecution( ctx, @@ -341,7 +400,7 @@ func (s *Starter) getMutableStateInfo(ctx context.Context, runID string) (*Mutab } // extractMutableStateInfo extracts the relevant information to generate a start response with an eager workflow task. -func extractMutableStateInfo(mutableState workflow.MutableState) (*MutableStateInfo, error) { +func extractMutableStateInfo(mutableState workflow.MutableState) (*mutableStateInfo, error) { branchToken, err := mutableState.GetCurrentBranchToken() if err != nil { return nil, err @@ -350,7 +409,7 @@ func extractMutableStateInfo(mutableState workflow.MutableState) (*MutableStateI // Future work for the request retry path: extend the task timeout (by failing / timing out the current task). workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() - return &MutableStateInfo{ + return &mutableStateInfo{ branchToken: branchToken, lastEventID: mutableState.GetNextEventID() - 1, workflowTaskInfo: workflowTaskInfo, @@ -359,7 +418,7 @@ func extractMutableStateInfo(mutableState workflow.MutableState) (*MutableStateI } // getWorkflowHistory loads the workflow history based on given mutable state information from the DB. -func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState *MutableStateInfo) ([]*history.HistoryEvent, error) { +func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState *mutableStateInfo) ([]*history.HistoryEvent, error) { var events []*history.HistoryEvent // Future optimization: generate the task from mutable state to save the extra DB read. // NOTE: While unlikely that there'll be more than one page, it's safer to make less assumptions From f32e716ca1e4be28a00966ac686f044e3a2471fc Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 25 Jan 2023 17:25:47 -0800 Subject: [PATCH 12/17] Fix task inflight condition --- service/history/api/startworkflow/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index f5f3da4e1c6..5f5e9314dc4 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -161,7 +161,7 @@ func (s *Starter) createNewMutableState(ctx context.Context, workflowID string, now := s.shardCtx.GetTimeSource().Now() mutableState := workflowContext.GetMutableState() workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() - if !hasInflight { + if s.request.StartRequest.GetRequestEagerExecution() && !hasInflight { return nil, serviceerror.NewInternal("unexpected error: mutable state did not have an inflight workflow task") } workflowSnapshot, workflowEvents, err := mutableState.CloseTransactionAsSnapshot( From 593a669b1384be43f896e8c7c9774402ac17b5cc Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Fri, 27 Jan 2023 17:40:05 -0800 Subject: [PATCH 13/17] Address review comments --- common/dynamicconfig/constants.go | 5 +- config/dynamicconfig/development-sql.yaml | 2 + service/history/api/create_workflow_util.go | 3 +- service/history/api/startworkflow/api.go | 164 ++++++++++-------- service/history/configs/config.go | 2 + service/history/handler.go | 2 - service/history/historyEngine_test.go | 37 +++- service/history/tests/vars.go | 1 + .../history/workflow/mutable_state_impl.go | 2 + 9 files changed, 140 insertions(+), 78 deletions(-) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index f3dbb2e77f0..b921e277d59 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -87,8 +87,11 @@ const ( EnableParentClosePolicyWorker = "system.enableParentClosePolicyWorker" // EnableStickyQuery indicates if sticky query should be enabled per namespace EnableStickyQuery = "system.enableStickyQuery" - // EnableActivityEagerExecution indicates if acitivty eager execution is enabled per namespace + // EnableActivityEagerExecution indicates if activity eager execution is enabled per namespace EnableActivityEagerExecution = "system.enableActivityEagerExecution" + // EnableEagerWorkflowStart toggles "eager workflow start" - returning the first workflow task inline in the + // response to a StartWorkflowExecution request and skipping the trip through matching. + EnableEagerWorkflowStart = "system.enableEagerWorkflowStart" // NamespaceCacheRefreshInterval is the key for namespace cache refresh interval dynamic config NamespaceCacheRefreshInterval = "system.namespaceCacheRefreshInterval" diff --git a/config/dynamicconfig/development-sql.yaml b/config/dynamicconfig/development-sql.yaml index b0ead9ed2da..6c98f155d2d 100644 --- a/config/dynamicconfig/development-sql.yaml +++ b/config/dynamicconfig/development-sql.yaml @@ -36,6 +36,8 @@ # constraints: {} #system.enableParentClosePolicyWorker: # - value: true +system.enableEagerWorkflowStart: + - value: true limit.maxIDLength: - value: 255 constraints: {} diff --git a/service/history/api/create_workflow_util.go b/service/history/api/create_workflow_util.go index 56539aeb94b..1b6d37995a0 100644 --- a/service/history/api/create_workflow_util.go +++ b/service/history/api/create_workflow_util.go @@ -111,7 +111,8 @@ func NewWorkflowWithSignal( return nil, err } - if requestEagerExecution { + // If first workflow task should back off (e.g. cron or workflow retry) a workflow task will not be scheduled. + if requestEagerExecution && scheduledEventID != 0 { _, _, err = newMutableState.AddWorkflowTaskStartedEvent( scheduledEventID, startRequest.StartRequest.RequestId, diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 5f5e9314dc4..e45de546cb8 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -28,14 +28,14 @@ import ( "context" "github.com/google/uuid" - "go.temporal.io/api/common/v1" - "go.temporal.io/api/history/v1" + commonpb "go.temporal.io/api/common/v1" + historypb "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/api/historyservice/v1" - serverCommon "go.temporal.io/server/common" + "go.temporal.io/server/common" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -50,7 +50,7 @@ import ( type Starter struct { shardCtx shard.Context workflowConsistencyChecker api.WorkflowConsistencyChecker - tokenSerializer serverCommon.TaskTokenSerializer + tokenSerializer common.TaskTokenSerializer request *historyservice.StartWorkflowExecutionRequest namespace *namespace.Namespace } @@ -79,7 +79,7 @@ type mutableStateInfo struct { func NewStarter( shardCtx shard.Context, workflowConsistencyChecker api.WorkflowConsistencyChecker, - tokenSerializer serverCommon.TaskTokenSerializer, + tokenSerializer common.TaskTokenSerializer, request *historyservice.StartWorkflowExecutionRequest, ) (*Starter, error) { namespaceEntry, err := api.GetActiveNamespace(shardCtx, namespace.ID(request.GetNamespaceId())) @@ -106,7 +106,11 @@ func (s *Starter) prepare(ctx context.Context) error { return err } - if request.GetRequestEagerExecution() { + // Override to false to avoid having to look up the dynamic config throughout the diffrent code paths. + if !s.shardCtx.GetConfig().EnableEagerWorkflowStart(s.namespace.Name().String()) { + request.RequestEagerExecution = false + } + if s.requestEagerStart() { metricsHandler.Counter(metrics.WorkflowEagerExecutionCounter.GetMetricName()).Record( 1, metrics.NamespaceTag(s.namespace.Name().String()), @@ -116,6 +120,10 @@ func (s *Starter) prepare(ctx context.Context) error { return nil } +func (s *Starter) requestEagerStart() bool { + return s.request.StartRequest.GetRequestEagerExecution() && *s.request.FirstWorkflowTaskBackoff == 0 +} + // Invoke starts a new workflow execution func (s *Starter) Invoke( ctx context.Context, @@ -131,7 +139,7 @@ func (s *Starter) Invoke( if err != nil { return nil, err } - err = s.createNewExecution(ctx, creationCtx) + err = s.createBrandNew(ctx, creationCtx) if err == nil { return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEvents)) } @@ -161,7 +169,7 @@ func (s *Starter) createNewMutableState(ctx context.Context, workflowID string, now := s.shardCtx.GetTimeSource().Now() mutableState := workflowContext.GetMutableState() workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() - if s.request.StartRequest.GetRequestEagerExecution() && !hasInflight { + if s.requestEagerStart() && !hasInflight { return nil, serviceerror.NewInternal("unexpected error: mutable state did not have an inflight workflow task") } workflowSnapshot, workflowEvents, err := mutableState.CloseTransactionAsSnapshot( @@ -185,8 +193,8 @@ func (s *Starter) createNewMutableState(ctx context.Context, workflowID string, }, nil } -// createNewExecution creates a new "brand new" execution in the executions table. -func (s *Starter) createNewExecution(ctx context.Context, creationCtx *creationContext) error { +// createBrandNew creates a new "brand new" execution in the executions table. +func (s *Starter) createBrandNew(ctx context.Context, creationCtx *creationContext) error { now := s.shardCtx.GetTimeSource().Now() return creationCtx.workflowContext.GetContext().CreateWorkflowExecution( ctx, @@ -206,64 +214,50 @@ func (s *Starter) createNewExecution(ctx context.Context, creationCtx *creationC func (s *Starter) handleConflict( ctx context.Context, creationCtx *creationContext, - workflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, + currentWorkflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, ) (*historyservice.StartWorkflowExecutionResponse, error) { request := s.request.StartRequest - if workflowConditionFailed.RequestID == request.GetRequestId() { - return s.respondToRetriedRequest(ctx, workflowConditionFailed.RunID) + if currentWorkflowConditionFailed.RequestID == request.GetRequestId() { + return s.respondToRetriedRequest(ctx, currentWorkflowConditionFailed.RunID) } - if err := s.verifyNamespaceActive(creationCtx, workflowConditionFailed); err != nil { + if err := s.verifyNamespaceActive(creationCtx, currentWorkflowConditionFailed); err != nil { return nil, err } - if mutableStateInfo, err := s.applyWorkflowIDReusePolicy(ctx, workflowConditionFailed, creationCtx.runID); err != nil { - switch err { - case nil: - if !request.GetRequestEagerExecution() { - return &historyservice.StartWorkflowExecutionResponse{ - RunId: creationCtx.runID, - }, nil - } - events, err := s.getWorkflowHistory(ctx, mutableStateInfo) - if err != nil { - return nil, err - } - return s.generateResponse(creationCtx.runID, mutableStateInfo.workflowTaskInfo, events) - case consts.ErrWorkflowCompleted: - // previous workflow already closed - // fallthough to the logic for only creating the new workflow below - default: - return nil, err - } + response, err := s.applyWorkflowIDReusePolicy(ctx, currentWorkflowConditionFailed, creationCtx) + if err != nil { + return nil, err + } else if response != nil { + return response, nil } - if err := s.updateCurrentExecution(ctx, creationCtx, workflowConditionFailed); err != nil { + if err := s.createAsCurrent(ctx, creationCtx, currentWorkflowConditionFailed); err != nil { return nil, err } return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEvents)) } -// updateCurrentExecution creates a new workflow execution and sets it to "current". -func (s *Starter) updateCurrentExecution( +// createAsCurrent creates a new workflow execution and sets it to "current". +func (s *Starter) createAsCurrent( ctx context.Context, creationCtx *creationContext, - workflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, + currentWorkflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, ) error { now := s.shardCtx.GetTimeSource().Now() return creationCtx.workflowContext.GetContext().CreateWorkflowExecution( ctx, now, persistence.CreateWorkflowModeUpdateCurrent, - workflowConditionFailed.RunID, - workflowConditionFailed.LastWriteVersion, + currentWorkflowConditionFailed.RunID, + currentWorkflowConditionFailed.LastWriteVersion, creationCtx.workflowContext.GetMutableState(), creationCtx.workflowSnapshot, creationCtx.workflowEvents, ) } -func (s *Starter) verifyNamespaceActive(creationCtx *creationContext, workflowConditionFailed *persistence.CurrentWorkflowConditionFailedError) error { - if creationCtx.workflowContext.GetMutableState().GetCurrentVersion() < workflowConditionFailed.LastWriteVersion { +func (s *Starter) verifyNamespaceActive(creationCtx *creationContext, currentWorkflowConditionFailed *persistence.CurrentWorkflowConditionFailedError) error { + if creationCtx.workflowContext.GetMutableState().GetCurrentVersion() < currentWorkflowConditionFailed.LastWriteVersion { clusterMetadata := s.shardCtx.GetClusterMetadata() - clusterName := clusterMetadata.ClusterNameForFailoverVersion(s.namespace.IsGlobalNamespace(), workflowConditionFailed.LastWriteVersion) + clusterName := clusterMetadata.ClusterNameForFailoverVersion(s.namespace.IsGlobalNamespace(), currentWorkflowConditionFailed.LastWriteVersion) return serviceerror.NewNamespaceNotActive( s.namespace.Name().String(), clusterMetadata.GetCurrentClusterName(), @@ -277,21 +271,20 @@ func (s *Starter) verifyNamespaceActive(creationCtx *creationContext, workflowCo // duplicate execution. // At the time of this writing, the only possible action here is to terminate the current execution in case the start // request's ID reuse policy is TERMINATE_IF_RUNNING. -// Returns non-nil MutableStateInfo if an action was required and completed successfully resulting in a newly created -// execution. +// Returns non-nil response if an action was required and completed successfully resulting in a newly created execution. func (s *Starter) applyWorkflowIDReusePolicy( ctx context.Context, - workflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, - runID string, -) (*mutableStateInfo, error) { + currentWorkflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, + creationCtx *creationContext, +) (*historyservice.StartWorkflowExecutionResponse, error) { workflowID := s.request.StartRequest.WorkflowId prevExecutionUpdateAction, err := api.ApplyWorkflowIDReusePolicy( - workflowConditionFailed.RequestID, - workflowConditionFailed.RunID, - workflowConditionFailed.State, - workflowConditionFailed.Status, + currentWorkflowConditionFailed.RequestID, + currentWorkflowConditionFailed.RunID, + currentWorkflowConditionFailed.State, + currentWorkflowConditionFailed.Status, workflowID, - runID, + creationCtx.runID, s.request.StartRequest.GetWorkflowIdReusePolicy(), ) if err != nil { @@ -310,7 +303,7 @@ func (s *Starter) applyWorkflowIDReusePolicy( definition.NewWorkflowKey( s.namespace.ID().String(), workflowID, - workflowConditionFailed.RunID, + currentWorkflowConditionFailed.RunID, ), prevExecutionUpdateAction, func() (workflow.Context, workflow.MutableState, error) { @@ -319,7 +312,7 @@ func (s *Starter) applyWorkflowIDReusePolicy( s.shardCtx, s.namespace, workflowID, - runID, + creationCtx.runID, s.request, nil) if err != nil { @@ -335,7 +328,25 @@ func (s *Starter) applyWorkflowIDReusePolicy( s.shardCtx, s.workflowConsistencyChecker, ) - return mutableStateInfo, err + switch err { + case nil: + if !s.requestEagerStart() { + return &historyservice.StartWorkflowExecutionResponse{ + RunId: creationCtx.runID, + }, nil + } + events, err := s.getWorkflowHistory(ctx, mutableStateInfo) + if err != nil { + return nil, err + } + return s.generateResponse(creationCtx.runID, mutableStateInfo.workflowTaskInfo, events) + case consts.ErrWorkflowCompleted: + // previous workflow already closed + // fallthough to the logic for only creating the new workflow below + return nil, nil + default: + return nil, err + } } // respondToRetriedRequest provides a response in case a start request is retried. @@ -345,7 +356,7 @@ func (s *Starter) respondToRetriedRequest( ) (*historyservice.StartWorkflowExecutionResponse, error) { request := s.request.StartRequest - if !request.GetRequestEagerExecution() { + if !s.requestEagerStart() { return &historyservice.StartWorkflowExecutionResponse{ RunId: runID, }, nil @@ -382,18 +393,18 @@ func (s *Starter) getMutableStateInfo(ctx context.Context, runID string) (*mutab workflowContext, releaseFn, err := s.workflowConsistencyChecker.GetWorkflowCache().GetOrCreateWorkflowExecution( ctx, s.namespace.ID(), - common.WorkflowExecution{WorkflowId: s.request.StartRequest.WorkflowId, RunId: runID}, + commonpb.WorkflowExecution{WorkflowId: s.request.StartRequest.WorkflowId, RunId: runID}, workflow.CallerTypeAPI, ) + if err != nil { + return nil, err + } + var releaseErr error defer func() { releaseFn(releaseErr) }() - if err != nil { - return nil, err - } - var mutableState workflow.MutableState mutableState, releaseErr = workflowContext.LoadMutableState(ctx) return extractMutableStateInfo(mutableState) @@ -407,21 +418,28 @@ func extractMutableStateInfo(mutableState workflow.MutableState) (*mutableStateI } // Future work for the request retry path: extend the task timeout (by failing / timing out the current task). - workflowTaskInfo, hasInflight := mutableState.GetInFlightWorkflowTask() + workflowTaskInfoSource, hasInflight := mutableState.GetInFlightWorkflowTask() + // The workflowTaskInfo returned from the mutable state call is generated on the fly and technically doesn't require + // cloning. We clone here just in case that changes. + var workflowTaskInfo workflow.WorkflowTaskInfo + if hasInflight { + workflowTaskInfo = *workflowTaskInfoSource + } return &mutableStateInfo{ branchToken: branchToken, lastEventID: mutableState.GetNextEventID() - 1, - workflowTaskInfo: workflowTaskInfo, + workflowTaskInfo: &workflowTaskInfo, hasInflight: hasInflight, }, nil } // getWorkflowHistory loads the workflow history based on given mutable state information from the DB. -func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState *mutableStateInfo) ([]*history.HistoryEvent, error) { - var events []*history.HistoryEvent +func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState *mutableStateInfo) ([]*historypb.HistoryEvent, error) { + var events []*historypb.HistoryEvent // Future optimization: generate the task from mutable state to save the extra DB read. - // NOTE: While unlikely that there'll be more than one page, it's safer to make less assumptions + // NOTE: While unlikely that there'll be more than one page, it's safer to make less assumptions. + // TODO: Frontend also supports returning raw history and it's controlled by a feature flag (yycptt thinks). for { response, err := s.shardCtx.GetExecutionManager().ReadHistoryBranch(ctx, &persistence.ReadHistoryBranchRequest{ ShardID: s.shardCtx.GetShardID(), @@ -434,7 +452,7 @@ func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState *mutableS return nil, err } events = append(events, response.HistoryEvents...) - if response.NextPageToken == nil || len(response.NextPageToken) == 0 { + if len(response.NextPageToken) == 0 { break } } @@ -444,11 +462,11 @@ func (s *Starter) getWorkflowHistory(ctx context.Context, mutableState *mutableS // extractHistoryEvents extracts all history events from a batch of events sent to persistence. // It's unlikely that persistence events would span multiple batches but better safe than sorry. -func extractHistoryEvents(persistenceEvents []*persistence.WorkflowEvents) []*history.HistoryEvent { +func extractHistoryEvents(persistenceEvents []*persistence.WorkflowEvents) []*historypb.HistoryEvent { if len(persistenceEvents) == 1 { return persistenceEvents[0].Events } - var events []*history.HistoryEvent + var events []*historypb.HistoryEvent for _, page := range persistenceEvents { events = append(events, page.Events...) } @@ -460,14 +478,14 @@ func extractHistoryEvents(persistenceEvents []*persistence.WorkflowEvents) []*hi func (s *Starter) generateResponse( runID string, workflowTaskInfo *workflow.WorkflowTaskInfo, - historyEvents []*history.HistoryEvent, + historyEvents []*historypb.HistoryEvent, ) (*historyservice.StartWorkflowExecutionResponse, error) { shardCtx := s.shardCtx tokenSerializer := s.tokenSerializer request := s.request.StartRequest workflowID := request.WorkflowId - if !request.GetRequestEagerExecution() { + if !s.requestEagerStart() { return &historyservice.StartWorkflowExecutionResponse{ RunId: runID, }, nil @@ -493,12 +511,12 @@ func (s *Starter) generateResponse( Clock: clock, EagerWorkflowTask: &workflowservice.PollWorkflowTaskQueueResponse{ TaskToken: serializedToken, - WorkflowExecution: &common.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, + WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, WorkflowType: request.GetWorkflowType(), PreviousStartedEventId: 0, StartedEventId: workflowTaskInfo.StartedEventID, Attempt: 1, - History: &history.History{Events: historyEvents}, + History: &historypb.History{Events: historyEvents}, NextPageToken: nil, WorkflowExecutionTaskQueue: workflowTaskInfo.TaskQueue, ScheduledTime: workflowTaskInfo.ScheduledTime, diff --git a/service/history/configs/config.go b/service/history/configs/config.go index ec9c14f276f..8f4c4a7cb4b 100644 --- a/service/history/configs/config.go +++ b/service/history/configs/config.go @@ -273,6 +273,7 @@ type Config struct { EnableCrossNamespaceCommands dynamicconfig.BoolPropertyFn EnableActivityEagerExecution dynamicconfig.BoolPropertyFnWithNamespaceFilter + EnableEagerWorkflowStart dynamicconfig.BoolPropertyFnWithNamespaceFilter NamespaceCacheRefreshInterval dynamicconfig.DurationPropertyFn // ArchivalQueueProcessor settings @@ -490,6 +491,7 @@ func NewConfig(dc *dynamicconfig.Collection, numberOfShards int32, isAdvancedVis EnableCrossNamespaceCommands: dc.GetBoolProperty(dynamicconfig.EnableCrossNamespaceCommands, true), EnableActivityEagerExecution: dc.GetBoolPropertyFnWithNamespaceFilter(dynamicconfig.EnableActivityEagerExecution, false), + EnableEagerWorkflowStart: dc.GetBoolPropertyFnWithNamespaceFilter(dynamicconfig.EnableEagerWorkflowStart, false), NamespaceCacheRefreshInterval: dc.GetDurationProperty(dynamicconfig.NamespaceCacheRefreshInterval, 10*time.Second), // Archival related diff --git a/service/history/handler.go b/service/history/handler.go index 9d9ab8b8453..d3bf05fedae 100644 --- a/service/history/handler.go +++ b/service/history/handler.go @@ -531,8 +531,6 @@ func (h *Handler) StartWorkflowExecution(ctx context.Context, request *historyse return nil, h.convertError(err) } if response.Clock == nil { - // TODO: do we still need to create a new clock with "eager" execution? - // We already put the clock in the task token and the response in that case. response.Clock, err = shardContext.NewVectorClock() } if err != nil { diff --git a/service/history/historyEngine_test.go b/service/history/historyEngine_test.go index 67b29454e59..63ed363b8b6 100644 --- a/service/history/historyEngine_test.go +++ b/service/history/historyEngine_test.go @@ -5067,7 +5067,7 @@ func (s *engineSuite) TestReapplyEvents_ResetWorkflow() { s.NoError(err) } -func (s *engineSuite) TestEagerWorkflowDispatch_DoesNotCreateTransferTask() { +func (s *engineSuite) TestEagerWorkflowStart_DoesNotCreateTransferTask() { var recordedTasks []tasks.Task s.mockExecutionMgr.EXPECT().CreateWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, request *persistence.CreateWorkflowExecutionRequest) (*persistence.CreateWorkflowExecutionResponse, error) { @@ -5101,6 +5101,41 @@ func (s *engineSuite) TestEagerWorkflowDispatch_DoesNotCreateTransferTask() { s.Equal(len(recordedTasks), 0) } +func (s *engineSuite) TestEagerWorkflowStart_FromCron_SkipsEager() { + var recordedTasks []tasks.Task + + s.mockExecutionMgr.EXPECT().CreateWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, request *persistence.CreateWorkflowExecutionRequest) (*persistence.CreateWorkflowExecutionResponse, error) { + recordedTasks = request.NewWorkflowSnapshot.Tasks[tasks.CategoryTransfer] + persistenceResponse := persistence.CreateWorkflowExecutionResponse{NewMutableStateStats: tests.CreateWorkflowExecutionResponse.NewMutableStateStats} + return &persistenceResponse, nil + }) + + i := interceptor.NewTelemetryInterceptor(s.mockShard.GetNamespaceRegistry(), s.mockShard.GetMetricsHandler(), s.mockShard.Resource.Logger) + response, err := i.Intercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req interface{}) (interface{}, error) { + firstWorkflowTaskBackoff := time.Second + response, err := s.mockHistoryEngine.StartWorkflowExecution(ctx, &historyservice.StartWorkflowExecutionRequest{ + NamespaceId: tests.NamespaceID.String(), + Attempt: 1, + ContinueAsNewInitiator: enumspb.CONTINUE_AS_NEW_INITIATOR_CRON_SCHEDULE, + FirstWorkflowTaskBackoff: &firstWorkflowTaskBackoff, + StartRequest: &workflowservice.StartWorkflowExecutionRequest{ + WorkflowId: "test", + Namespace: tests.Namespace.String(), + WorkflowType: &commonpb.WorkflowType{Name: "test"}, + TaskQueue: &taskqueuepb.TaskQueue{Kind: enumspb.TASK_QUEUE_KIND_NORMAL, Name: "test"}, + Identity: "test", + RequestId: "test", + CronSchedule: "* * * * *", + RequestEagerExecution: true, + }, + }) + return response, err + }) + s.NoError(err) + s.Nil(response.(*historyservice.StartWorkflowExecutionResponse).EagerWorkflowTask) + s.Equal(len(recordedTasks), 0) +} + func (s *engineSuite) getMutableState(testNamespaceID namespace.ID, we commonpb.WorkflowExecution) workflow.MutableState { context, release, err := s.workflowCache.GetOrCreateWorkflowExecution( context.Background(), diff --git a/service/history/tests/vars.go b/service/history/tests/vars.go index 8a8bce1476b..2ba316227d3 100644 --- a/service/history/tests/vars.go +++ b/service/history/tests/vars.go @@ -183,6 +183,7 @@ func NewDynamicConfig() *configs.Config { // reduce the duration of long poll to increase test speed config.LongPollExpirationInterval = dc.GetDurationPropertyFilteredByNamespace(dynamicconfig.HistoryLongPollExpirationInterval, 10*time.Second) config.EnableActivityEagerExecution = dynamicconfig.GetBoolPropertyFnFilteredByNamespace(true) + config.EnableEagerWorkflowStart = dynamicconfig.GetBoolPropertyFnFilteredByNamespace(true) config.NamespaceCacheRefreshInterval = dynamicconfig.GetDurationPropertyFn(time.Second) config.DurableArchivalEnabled = dynamicconfig.GetBoolPropertyFn(true) return config diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index bee1fbb1d8a..be29613f532 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -1472,6 +1472,8 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( CronSchedule: command.CronSchedule, Memo: command.Memo, SearchAttributes: command.SearchAttributes, + // No need to request eager execution here (for now) + RequestEagerExecution: false, } enums.SetDefaultContinueAsNewInitiator(&command.Initiator) From 9a3a16ed7f0f8cde10a24f2c1e62e59cf0b1b70d Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Tue, 31 Jan 2023 17:12:16 -0800 Subject: [PATCH 14/17] Address review comments --- common/metrics/metric_defs.go | 216 ++++++++++---------- common/metrics/tags.go | 8 + config/dynamicconfig/development-cass.yaml | 2 + service/history/api/create_workflow_util.go | 2 +- service/history/api/startworkflow/api.go | 104 ++++++---- 5 files changed, 187 insertions(+), 145 deletions(-) diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 1ba3234e2c7..06a02a121c8 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1461,111 +1461,117 @@ var ( CommandTypeModifyWorkflowPropertiesCounter = NewCounterDef("modify_workflow_properties_command") CommandTypeChildWorkflowCounter = NewCounterDef("child_workflow_command") ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution") - WorkflowEagerExecutionCounter = NewCounterDef("workflow_eager_execution") - EmptyCompletionCommandsCounter = NewCounterDef("empty_completion_commands") - MultipleCompletionCommandsCounter = NewCounterDef("multiple_completion_commands") - FailedWorkflowTasksCounter = NewCounterDef("failed_workflow_tasks") - WorkflowTaskAttempt = NewDimensionlessHistogramDef("workflow_task_attempt") - StaleMutableStateCounter = NewCounterDef("stale_mutable_state") - AutoResetPointsLimitExceededCounter = NewCounterDef("auto_reset_points_exceed_limit") - AutoResetPointCorruptionCounter = NewCounterDef("auto_reset_point_corruption") - ConcurrencyUpdateFailureCounter = NewCounterDef("concurrency_update_failure") - ServiceErrShardOwnershipLostCounter = NewCounterDef("service_errors_shard_ownership_lost") - ServiceErrTaskAlreadyStartedCounter = NewCounterDef("service_errors_task_already_started") - HeartbeatTimeoutCounter = NewCounterDef("heartbeat_timeout") - ScheduleToStartTimeoutCounter = NewCounterDef("schedule_to_start_timeout") - StartToCloseTimeoutCounter = NewCounterDef("start_to_close_timeout") - ScheduleToCloseTimeoutCounter = NewCounterDef("schedule_to_close_timeout") - NewTimerNotifyCounter = NewCounterDef("new_timer_notifications") - AcquireShardsCounter = NewCounterDef("acquire_shards_count") - AcquireShardsLatency = NewTimerDef("acquire_shards_latency") - MembershipChangedCounter = NewCounterDef("membership_changed_count") - NumShardsGauge = NewGaugeDef("numshards_gauge") - GetEngineForShardErrorCounter = NewCounterDef("get_engine_for_shard_errors") - GetEngineForShardLatency = NewTimerDef("get_engine_for_shard_latency") - RemoveEngineForShardLatency = NewTimerDef("remove_engine_for_shard_latency") - CompleteWorkflowTaskWithStickyEnabledCounter = NewCounterDef("complete_workflow_task_sticky_enabled_count") - CompleteWorkflowTaskWithStickyDisabledCounter = NewCounterDef("complete_workflow_task_sticky_disabled_count") - WorkflowTaskHeartbeatTimeoutCounter = NewCounterDef("workflow_task_heartbeat_timeout_count") - EmptyReplicationEventsCounter = NewCounterDef("empty_replication_events") - DuplicateReplicationEventsCounter = NewCounterDef("duplicate_replication_events") - StaleReplicationEventsCounter = NewCounterDef("stale_replication_events") - ReplicationEventsSizeTimer = NewTimerDef("replication_events_size") - BufferReplicationTaskTimer = NewTimerDef("buffer_replication_tasks") - UnbufferReplicationTaskTimer = NewTimerDef("unbuffer_replication_tasks") - HistoryConflictsCounter = NewCounterDef("history_conflicts") - CompleteTaskFailedCounter = NewCounterDef("complete_task_fail_count") - AcquireLockFailedCounter = NewCounterDef("acquire_lock_failed") - WorkflowContextCleared = NewCounterDef("workflow_context_cleared") - MutableStateSize = NewBytesHistogramDef("mutable_state_size") - ExecutionInfoSize = NewBytesHistogramDef("execution_info_size") - ExecutionStateSize = NewBytesHistogramDef("execution_state_size") - ActivityInfoSize = NewBytesHistogramDef("activity_info_size") - TimerInfoSize = NewBytesHistogramDef("timer_info_size") - ChildInfoSize = NewBytesHistogramDef("child_info_size") - RequestCancelInfoSize = NewBytesHistogramDef("request_cancel_info_size") - SignalInfoSize = NewBytesHistogramDef("signal_info_size") - BufferedEventsSize = NewBytesHistogramDef("buffered_events_size") - ActivityInfoCount = NewDimensionlessHistogramDef("activity_info_count") - TimerInfoCount = NewDimensionlessHistogramDef("timer_info_count") - ChildInfoCount = NewDimensionlessHistogramDef("child_info_count") - SignalInfoCount = NewDimensionlessHistogramDef("signal_info_count") - RequestCancelInfoCount = NewDimensionlessHistogramDef("request_cancel_info_count") - BufferedEventsCount = NewDimensionlessHistogramDef("buffered_events_count") - TaskCount = NewDimensionlessHistogramDef("task_count") - WorkflowRetryBackoffTimerCount = NewCounterDef("workflow_retry_backoff_timer") - WorkflowCronBackoffTimerCount = NewCounterDef("workflow_cron_backoff_timer") - WorkflowCleanupDeleteCount = NewCounterDef("workflow_cleanup_delete") - WorkflowCleanupArchiveCount = NewCounterDef("workflow_cleanup_archive") - WorkflowCleanupNopCount = NewCounterDef("workflow_cleanup_nop") - WorkflowCleanupDeleteHistoryInlineCount = NewCounterDef("workflow_cleanup_delete_history_inline") - WorkflowSuccessCount = NewCounterDef("workflow_success") - WorkflowCancelCount = NewCounterDef("workflow_cancel") - WorkflowFailedCount = NewCounterDef("workflow_failed") - WorkflowTimeoutCount = NewCounterDef("workflow_timeout") - WorkflowTerminateCount = NewCounterDef("workflow_terminate") - WorkflowContinuedAsNewCount = NewCounterDef("workflow_continued_as_new") - LastRetrievedMessageID = NewGaugeDef("last_retrieved_message_id") - LastProcessedMessageID = NewGaugeDef("last_processed_message_id") - ReplicationTasksApplied = NewCounterDef("replication_tasks_applied") - ReplicationTasksFailed = NewCounterDef("replication_tasks_failed") - ReplicationTasksLag = NewTimerDef("replication_tasks_lag") - ReplicationLatency = NewTimerDef("replication_latency") - ReplicationTasksFetched = NewTimerDef("replication_tasks_fetched") - ReplicationTasksReturned = NewTimerDef("replication_tasks_returned") - ReplicationTasksAppliedLatency = NewTimerDef("replication_tasks_applied_latency") - ReplicationDLQFailed = NewCounterDef("replication_dlq_enqueue_failed") - ReplicationDLQMaxLevelGauge = NewGaugeDef("replication_dlq_max_level") - ReplicationDLQAckLevelGauge = NewGaugeDef("replication_dlq_ack_level") - GetReplicationMessagesForShardLatency = NewTimerDef("get_replication_messages_for_shard") - GetDLQReplicationMessagesLatency = NewTimerDef("get_dlq_replication_messages") - EventReapplySkippedCount = NewCounterDef("event_reapply_skipped_count") - DirectQueryDispatchLatency = NewTimerDef("direct_query_dispatch_latency") - DirectQueryDispatchStickyLatency = NewTimerDef("direct_query_dispatch_sticky_latency") - DirectQueryDispatchNonStickyLatency = NewTimerDef("direct_query_dispatch_non_sticky_latency") - DirectQueryDispatchStickySuccessCount = NewCounterDef("direct_query_dispatch_sticky_success") - DirectQueryDispatchNonStickySuccessCount = NewCounterDef("direct_query_dispatch_non_sticky_success") - DirectQueryDispatchClearStickinessLatency = NewTimerDef("direct_query_dispatch_clear_stickiness_latency") - DirectQueryDispatchClearStickinessSuccessCount = NewCounterDef("direct_query_dispatch_clear_stickiness_success") - DirectQueryDispatchTimeoutBeforeNonStickyCount = NewCounterDef("direct_query_dispatch_timeout_before_non_sticky") - WorkflowTaskQueryLatency = NewTimerDef("workflow_task_query_latency") - ConsistentQueryTimeoutCount = NewCounterDef("consistent_query_timeout") - QueryBeforeFirstWorkflowTaskCount = NewCounterDef("query_before_first_workflow_task") - QueryBufferExceededCount = NewCounterDef("query_buffer_exceeded") - QueryRegistryInvalidStateCount = NewCounterDef("query_registry_invalid_state") - WorkerNotSupportsConsistentQueryCount = NewCounterDef("worker_not_supports_consistent_query") - WorkflowTaskTimeoutOverrideCount = NewCounterDef("workflow_task_timeout_overrides") - WorkflowRunTimeoutOverrideCount = NewCounterDef("workflow_run_timeout_overrides") - ReplicationTaskCleanupCount = NewCounterDef("replication_task_cleanup_count") - ReplicationTaskCleanupFailure = NewCounterDef("replication_task_cleanup_failed") - MutableStateChecksumMismatch = NewCounterDef("mutable_state_checksum_mismatch") - MutableStateChecksumInvalidated = NewCounterDef("mutable_state_checksum_invalidated") - ClusterMetadataLockLatency = NewTimerDef("cluster_metadata_lock_latency") - ClusterMetadataCallbackLockLatency = NewTimerDef("cluster_metadata_callback_lock_latency") - ShardControllerLockLatency = NewTimerDef("shard_controller_lock_latency") - ShardLockLatency = NewTimerDef("shard_lock_latency") - NamespaceRegistryLockLatency = NewTimerDef("namespace_registry_lock_latency") - ClosedWorkflowBufferEventCount = NewCounterDef("closed_workflow_buffer_event_counter") + // WorkflowEagerExecutionCounter is emitted any time eager workflow start is requested. + WorkflowEagerExecutionCounter = NewCounterDef("workflow_eager_execution") + // WorkflowEagerExecutionDeniedCounter is emitted any time eager workflow start is requested and the serer fell back + // to standard dispatch. + // Timeouts and failures are not counted in this metric. + // This metric has a "reason" tag attached to it to understand why eager start was denied. + WorkflowEagerExecutionDeniedCounter = NewCounterDef("workflow_eager_execution_denied") + EmptyCompletionCommandsCounter = NewCounterDef("empty_completion_commands") + MultipleCompletionCommandsCounter = NewCounterDef("multiple_completion_commands") + FailedWorkflowTasksCounter = NewCounterDef("failed_workflow_tasks") + WorkflowTaskAttempt = NewDimensionlessHistogramDef("workflow_task_attempt") + StaleMutableStateCounter = NewCounterDef("stale_mutable_state") + AutoResetPointsLimitExceededCounter = NewCounterDef("auto_reset_points_exceed_limit") + AutoResetPointCorruptionCounter = NewCounterDef("auto_reset_point_corruption") + ConcurrencyUpdateFailureCounter = NewCounterDef("concurrency_update_failure") + ServiceErrShardOwnershipLostCounter = NewCounterDef("service_errors_shard_ownership_lost") + ServiceErrTaskAlreadyStartedCounter = NewCounterDef("service_errors_task_already_started") + HeartbeatTimeoutCounter = NewCounterDef("heartbeat_timeout") + ScheduleToStartTimeoutCounter = NewCounterDef("schedule_to_start_timeout") + StartToCloseTimeoutCounter = NewCounterDef("start_to_close_timeout") + ScheduleToCloseTimeoutCounter = NewCounterDef("schedule_to_close_timeout") + NewTimerNotifyCounter = NewCounterDef("new_timer_notifications") + AcquireShardsCounter = NewCounterDef("acquire_shards_count") + AcquireShardsLatency = NewTimerDef("acquire_shards_latency") + MembershipChangedCounter = NewCounterDef("membership_changed_count") + NumShardsGauge = NewGaugeDef("numshards_gauge") + GetEngineForShardErrorCounter = NewCounterDef("get_engine_for_shard_errors") + GetEngineForShardLatency = NewTimerDef("get_engine_for_shard_latency") + RemoveEngineForShardLatency = NewTimerDef("remove_engine_for_shard_latency") + CompleteWorkflowTaskWithStickyEnabledCounter = NewCounterDef("complete_workflow_task_sticky_enabled_count") + CompleteWorkflowTaskWithStickyDisabledCounter = NewCounterDef("complete_workflow_task_sticky_disabled_count") + WorkflowTaskHeartbeatTimeoutCounter = NewCounterDef("workflow_task_heartbeat_timeout_count") + EmptyReplicationEventsCounter = NewCounterDef("empty_replication_events") + DuplicateReplicationEventsCounter = NewCounterDef("duplicate_replication_events") + StaleReplicationEventsCounter = NewCounterDef("stale_replication_events") + ReplicationEventsSizeTimer = NewTimerDef("replication_events_size") + BufferReplicationTaskTimer = NewTimerDef("buffer_replication_tasks") + UnbufferReplicationTaskTimer = NewTimerDef("unbuffer_replication_tasks") + HistoryConflictsCounter = NewCounterDef("history_conflicts") + CompleteTaskFailedCounter = NewCounterDef("complete_task_fail_count") + AcquireLockFailedCounter = NewCounterDef("acquire_lock_failed") + WorkflowContextCleared = NewCounterDef("workflow_context_cleared") + MutableStateSize = NewBytesHistogramDef("mutable_state_size") + ExecutionInfoSize = NewBytesHistogramDef("execution_info_size") + ExecutionStateSize = NewBytesHistogramDef("execution_state_size") + ActivityInfoSize = NewBytesHistogramDef("activity_info_size") + TimerInfoSize = NewBytesHistogramDef("timer_info_size") + ChildInfoSize = NewBytesHistogramDef("child_info_size") + RequestCancelInfoSize = NewBytesHistogramDef("request_cancel_info_size") + SignalInfoSize = NewBytesHistogramDef("signal_info_size") + BufferedEventsSize = NewBytesHistogramDef("buffered_events_size") + ActivityInfoCount = NewDimensionlessHistogramDef("activity_info_count") + TimerInfoCount = NewDimensionlessHistogramDef("timer_info_count") + ChildInfoCount = NewDimensionlessHistogramDef("child_info_count") + SignalInfoCount = NewDimensionlessHistogramDef("signal_info_count") + RequestCancelInfoCount = NewDimensionlessHistogramDef("request_cancel_info_count") + BufferedEventsCount = NewDimensionlessHistogramDef("buffered_events_count") + TaskCount = NewDimensionlessHistogramDef("task_count") + WorkflowRetryBackoffTimerCount = NewCounterDef("workflow_retry_backoff_timer") + WorkflowCronBackoffTimerCount = NewCounterDef("workflow_cron_backoff_timer") + WorkflowCleanupDeleteCount = NewCounterDef("workflow_cleanup_delete") + WorkflowCleanupArchiveCount = NewCounterDef("workflow_cleanup_archive") + WorkflowCleanupNopCount = NewCounterDef("workflow_cleanup_nop") + WorkflowCleanupDeleteHistoryInlineCount = NewCounterDef("workflow_cleanup_delete_history_inline") + WorkflowSuccessCount = NewCounterDef("workflow_success") + WorkflowCancelCount = NewCounterDef("workflow_cancel") + WorkflowFailedCount = NewCounterDef("workflow_failed") + WorkflowTimeoutCount = NewCounterDef("workflow_timeout") + WorkflowTerminateCount = NewCounterDef("workflow_terminate") + WorkflowContinuedAsNewCount = NewCounterDef("workflow_continued_as_new") + LastRetrievedMessageID = NewGaugeDef("last_retrieved_message_id") + LastProcessedMessageID = NewGaugeDef("last_processed_message_id") + ReplicationTasksApplied = NewCounterDef("replication_tasks_applied") + ReplicationTasksFailed = NewCounterDef("replication_tasks_failed") + ReplicationTasksLag = NewTimerDef("replication_tasks_lag") + ReplicationLatency = NewTimerDef("replication_latency") + ReplicationTasksFetched = NewTimerDef("replication_tasks_fetched") + ReplicationTasksReturned = NewTimerDef("replication_tasks_returned") + ReplicationTasksAppliedLatency = NewTimerDef("replication_tasks_applied_latency") + ReplicationDLQFailed = NewCounterDef("replication_dlq_enqueue_failed") + ReplicationDLQMaxLevelGauge = NewGaugeDef("replication_dlq_max_level") + ReplicationDLQAckLevelGauge = NewGaugeDef("replication_dlq_ack_level") + GetReplicationMessagesForShardLatency = NewTimerDef("get_replication_messages_for_shard") + GetDLQReplicationMessagesLatency = NewTimerDef("get_dlq_replication_messages") + EventReapplySkippedCount = NewCounterDef("event_reapply_skipped_count") + DirectQueryDispatchLatency = NewTimerDef("direct_query_dispatch_latency") + DirectQueryDispatchStickyLatency = NewTimerDef("direct_query_dispatch_sticky_latency") + DirectQueryDispatchNonStickyLatency = NewTimerDef("direct_query_dispatch_non_sticky_latency") + DirectQueryDispatchStickySuccessCount = NewCounterDef("direct_query_dispatch_sticky_success") + DirectQueryDispatchNonStickySuccessCount = NewCounterDef("direct_query_dispatch_non_sticky_success") + DirectQueryDispatchClearStickinessLatency = NewTimerDef("direct_query_dispatch_clear_stickiness_latency") + DirectQueryDispatchClearStickinessSuccessCount = NewCounterDef("direct_query_dispatch_clear_stickiness_success") + DirectQueryDispatchTimeoutBeforeNonStickyCount = NewCounterDef("direct_query_dispatch_timeout_before_non_sticky") + WorkflowTaskQueryLatency = NewTimerDef("workflow_task_query_latency") + ConsistentQueryTimeoutCount = NewCounterDef("consistent_query_timeout") + QueryBeforeFirstWorkflowTaskCount = NewCounterDef("query_before_first_workflow_task") + QueryBufferExceededCount = NewCounterDef("query_buffer_exceeded") + QueryRegistryInvalidStateCount = NewCounterDef("query_registry_invalid_state") + WorkerNotSupportsConsistentQueryCount = NewCounterDef("worker_not_supports_consistent_query") + WorkflowTaskTimeoutOverrideCount = NewCounterDef("workflow_task_timeout_overrides") + WorkflowRunTimeoutOverrideCount = NewCounterDef("workflow_run_timeout_overrides") + ReplicationTaskCleanupCount = NewCounterDef("replication_task_cleanup_count") + ReplicationTaskCleanupFailure = NewCounterDef("replication_task_cleanup_failed") + MutableStateChecksumMismatch = NewCounterDef("mutable_state_checksum_mismatch") + MutableStateChecksumInvalidated = NewCounterDef("mutable_state_checksum_invalidated") + ClusterMetadataLockLatency = NewTimerDef("cluster_metadata_lock_latency") + ClusterMetadataCallbackLockLatency = NewTimerDef("cluster_metadata_callback_lock_latency") + ShardControllerLockLatency = NewTimerDef("shard_controller_lock_latency") + ShardLockLatency = NewTimerDef("shard_lock_latency") + NamespaceRegistryLockLatency = NewTimerDef("namespace_registry_lock_latency") + ClosedWorkflowBufferEventCount = NewCounterDef("closed_workflow_buffer_event_counter") // Matching MatchingClientForwardedCounter = NewCounterDef("forwarded") diff --git a/common/metrics/tags.go b/common/metrics/tags.go index 5f8bc591ccf..a77f2dc7c01 100644 --- a/common/metrics/tags.go +++ b/common/metrics/tags.go @@ -51,6 +51,8 @@ const ( commandType = "commandType" serviceName = "service_name" actionType = "action_type" + // Generic reason tag can be used anywhere a reason is needed. + reason = "reason" namespaceAllValue = "all" unknownValue = "_unknown_" @@ -278,3 +280,9 @@ func StringTag(key string, value string) Tag { func CacheTypeTag(value string) Tag { return &tagImpl{key: CacheTypeTagName, value: value} } + +// ReasonTag is a generic tag can be used anywhere a reason is needed. +// Make sure that the value is of limited cardinality. +func ReasonTag(value string) Tag { + return &tagImpl{key: reason, value: value} +} diff --git a/config/dynamicconfig/development-cass.yaml b/config/dynamicconfig/development-cass.yaml index 6e77991e252..33835160e94 100644 --- a/config/dynamicconfig/development-cass.yaml +++ b/config/dynamicconfig/development-cass.yaml @@ -36,3 +36,5 @@ # constraints: {} #system.enableParentClosePolicyWorker: # - value: true +system.enableEagerWorkflowStart: + - value: true diff --git a/service/history/api/create_workflow_util.go b/service/history/api/create_workflow_util.go index 1b6d37995a0..b8d25d2317a 100644 --- a/service/history/api/create_workflow_util.go +++ b/service/history/api/create_workflow_util.go @@ -112,7 +112,7 @@ func NewWorkflowWithSignal( } // If first workflow task should back off (e.g. cron or workflow retry) a workflow task will not be scheduled. - if requestEagerExecution && scheduledEventID != 0 { + if requestEagerExecution && newMutableState.HasPendingWorkflowTask() { _, _, err = newMutableState.AddWorkflowTaskStartedEvent( scheduledEventID, startRequest.StartRequest.RequestId, diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index e45de546cb8..5eceb3c1c96 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -46,6 +46,14 @@ import ( "go.temporal.io/server/service/history/workflow" ) +type eagerStartDeniedReason string + +const ( + eagerStartDeniedReasonDynamicConfigDisabled eagerStartDeniedReason = "dynamic_config_disabled" + eagerStartDeniedReasonFirstWorkflowTaskBackoff eagerStartDeniedReason = "first_workflow_task_backoff" + eagerStartDeniedReasonTaskAlreadyDispatched eagerStartDeniedReason = "task_already_dispatched" +) + // Starter starts a new workflow execution. type Starter struct { shardCtx shard.Context @@ -58,12 +66,12 @@ type Starter struct { // creationContext is a container for all information obtained from creating the uncommitted execution. // The information is later used to create a new execution and handle conflicts. type creationContext struct { - workflowID string - runID string - workflowContext api.WorkflowContext - workflowTaskInfo *workflow.WorkflowTaskInfo - workflowSnapshot *persistence.WorkflowSnapshot - workflowEvents []*persistence.WorkflowEvents + workflowID string + runID string + workflowContext api.WorkflowContext + workflowTaskInfo *workflow.WorkflowTaskInfo + workflowSnapshot *persistence.WorkflowSnapshot + workflowEventBatches []*persistence.WorkflowEvents } // mutableStateInfo is a container for the relevant mutable state information to generate a start response with an eager @@ -106,22 +114,40 @@ func (s *Starter) prepare(ctx context.Context) error { return err } - // Override to false to avoid having to look up the dynamic config throughout the diffrent code paths. - if !s.shardCtx.GetConfig().EnableEagerWorkflowStart(s.namespace.Name().String()) { - request.RequestEagerExecution = false - } - if s.requestEagerStart() { + if request.RequestEagerExecution { metricsHandler.Counter(metrics.WorkflowEagerExecutionCounter.GetMetricName()).Record( 1, metrics.NamespaceTag(s.namespace.Name().String()), metrics.TaskQueueTag(request.TaskQueue.Name), + metrics.WorkflowTypeTag(request.WorkflowType.Name), ) } + + // Override to false to avoid having to look up the dynamic config throughout the diffrent code paths. + if !s.shardCtx.GetConfig().EnableEagerWorkflowStart(s.namespace.Name().String()) { + s.recordEagerDenied(eagerStartDeniedReasonDynamicConfigDisabled) + request.RequestEagerExecution = false + } + if *s.request.FirstWorkflowTaskBackoff > 0 { + s.recordEagerDenied(eagerStartDeniedReasonFirstWorkflowTaskBackoff) + request.RequestEagerExecution = false + } return nil } +func (s *Starter) recordEagerDenied(reason eagerStartDeniedReason) { + metricsHandler := s.shardCtx.GetMetricsHandler() + metricsHandler.Counter(metrics.WorkflowEagerExecutionDeniedCounter.GetMetricName()).Record( + 1, + metrics.NamespaceTag(s.namespace.Name().String()), + metrics.TaskQueueTag(s.request.StartRequest.TaskQueue.Name), + metrics.WorkflowTypeTag(s.request.StartRequest.WorkflowType.Name), + metrics.ReasonTag(""), + ) +} + func (s *Starter) requestEagerStart() bool { - return s.request.StartRequest.GetRequestEagerExecution() && *s.request.FirstWorkflowTaskBackoff == 0 + return s.request.StartRequest.GetRequestEagerExecution() } // Invoke starts a new workflow execution @@ -141,12 +167,13 @@ func (s *Starter) Invoke( } err = s.createBrandNew(ctx, creationCtx) if err == nil { - return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEvents)) + return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEventBatches)) } t, ok := err.(*persistence.CurrentWorkflowConditionFailedError) if !ok { return nil, err } + // The history and mutable state we generated above should be deleted by a background process. return s.handleConflict(ctx, creationCtx, t) } @@ -172,24 +199,24 @@ func (s *Starter) createNewMutableState(ctx context.Context, workflowID string, if s.requestEagerStart() && !hasInflight { return nil, serviceerror.NewInternal("unexpected error: mutable state did not have an inflight workflow task") } - workflowSnapshot, workflowEvents, err := mutableState.CloseTransactionAsSnapshot( + workflowSnapshot, eventBatches, err := mutableState.CloseTransactionAsSnapshot( now, workflow.TransactionPolicyActive, ) if err != nil { return nil, err } - if len(workflowEvents) != 1 { + if len(eventBatches) != 1 { return nil, serviceerror.NewInternal("unable to create 1st event batch") } return &creationContext{ - workflowID: workflowID, - runID: runID, - workflowContext: workflowContext, - workflowTaskInfo: workflowTaskInfo, - workflowSnapshot: workflowSnapshot, - workflowEvents: workflowEvents, + workflowID: workflowID, + runID: runID, + workflowContext: workflowContext, + workflowTaskInfo: workflowTaskInfo, + workflowSnapshot: workflowSnapshot, + workflowEventBatches: eventBatches, }, nil } @@ -204,13 +231,13 @@ func (s *Starter) createBrandNew(ctx context.Context, creationCtx *creationConte 0, // prevLastWriteVersion creationCtx.workflowContext.GetMutableState(), creationCtx.workflowSnapshot, - creationCtx.workflowEvents, + creationCtx.workflowEventBatches, ) } -// handleConflict handles CurrentWorkflowConditionFailedError where a previous request with the same request ID already -// created a workflow execution with a different run ID. -// The history we generated above should be deleted by a background process. +// handleConflict handles CurrentWorkflowConditionFailedError where there's a workflow with the same workflowID. +// This may happen either when the currently handled request is a retry of a previous attempt (identified by the +// RequestID) or simply because a different run exists for the same workflow. func (s *Starter) handleConflict( ctx context.Context, creationCtx *creationContext, @@ -232,7 +259,7 @@ func (s *Starter) handleConflict( if err := s.createAsCurrent(ctx, creationCtx, currentWorkflowConditionFailed); err != nil { return nil, err } - return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEvents)) + return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEventBatches)) } // createAsCurrent creates a new workflow execution and sets it to "current". @@ -250,7 +277,7 @@ func (s *Starter) createAsCurrent( currentWorkflowConditionFailed.LastWriteVersion, creationCtx.workflowContext.GetMutableState(), creationCtx.workflowSnapshot, - creationCtx.workflowEvents, + creationCtx.workflowEventBatches, ) } @@ -354,8 +381,6 @@ func (s *Starter) respondToRetriedRequest( ctx context.Context, runID string, ) (*historyservice.StartWorkflowExecutionResponse, error) { - request := s.request.StartRequest - if !s.requestEagerStart() { return &historyservice.StartWorkflowExecutionResponse{ RunId: runID, @@ -371,11 +396,10 @@ func (s *Starter) respondToRetriedRequest( // The current workflow task is not inflight or not the first task or we exceeded the first attempt and fell back to // matching based dispatch. if !mutableStateInfo.hasInflight || mutableStateInfo.workflowTaskInfo.StartedEventID != 3 || mutableStateInfo.workflowTaskInfo.Attempt > 1 { - return nil, serviceerror.NewWorkflowExecutionAlreadyStarted( - "First workflow task is no longer inflight, retried request came in too late", - request.RequestId, - runID, - ) + s.recordEagerDenied(eagerStartDeniedReasonTaskAlreadyDispatched) + return &historyservice.StartWorkflowExecutionResponse{ + RunId: runID, + }, nil } events, err := s.getWorkflowHistory(ctx, mutableStateInfo) @@ -499,7 +523,7 @@ func (s *Starter) generateResponse( WorkflowId: workflowID, RunId: runID, ScheduledEventId: workflowTaskInfo.ScheduledEventID, - Attempt: 1, + Attempt: workflowTaskInfo.Attempt, Clock: clock, } serializedToken, err := tokenSerializer.Serialize(taskToken) @@ -510,12 +534,14 @@ func (s *Starter) generateResponse( RunId: runID, Clock: clock, EagerWorkflowTask: &workflowservice.PollWorkflowTaskQueueResponse{ - TaskToken: serializedToken, - WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, - WorkflowType: request.GetWorkflowType(), + TaskToken: serializedToken, + WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, + WorkflowType: request.GetWorkflowType(), + // TODO: consider getting the ID from mutable state, this was not done to avoid adding more complexity to + // the code to plumb that value through. PreviousStartedEventId: 0, StartedEventId: workflowTaskInfo.StartedEventID, - Attempt: 1, + Attempt: workflowTaskInfo.Attempt, History: &historypb.History{Events: historyEvents}, NextPageToken: nil, WorkflowExecutionTaskQueue: workflowTaskInfo.TaskQueue, From 41296fcd9b3821b461c1aebfe8c401d930dff0a5 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Tue, 31 Jan 2023 19:59:17 -0800 Subject: [PATCH 15/17] Fix missing reason tag value --- service/history/api/startworkflow/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 5eceb3c1c96..bb3ff405c70 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -142,7 +142,7 @@ func (s *Starter) recordEagerDenied(reason eagerStartDeniedReason) { metrics.NamespaceTag(s.namespace.Name().String()), metrics.TaskQueueTag(s.request.StartRequest.TaskQueue.Name), metrics.WorkflowTypeTag(s.request.StartRequest.WorkflowType.Name), - metrics.ReasonTag(""), + metrics.ReasonTag(string(reason)), ) } From efece494989ad19cd7aee6ac969469df5a79db51 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 1 Feb 2023 12:30:16 -0800 Subject: [PATCH 16/17] Add missing nil check --- service/history/api/startworkflow/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index bb3ff405c70..9d1591ca1da 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -128,7 +128,7 @@ func (s *Starter) prepare(ctx context.Context) error { s.recordEagerDenied(eagerStartDeniedReasonDynamicConfigDisabled) request.RequestEagerExecution = false } - if *s.request.FirstWorkflowTaskBackoff > 0 { + if s.request.FirstWorkflowTaskBackoff != nil && *s.request.FirstWorkflowTaskBackoff > 0 { s.recordEagerDenied(eagerStartDeniedReasonFirstWorkflowTaskBackoff) request.RequestEagerExecution = false } From e0a69c2284a704159408838d0e2e0edc470c8bcf Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 1 Feb 2023 16:53:17 -0800 Subject: [PATCH 17/17] Address review comments --- common/metrics/tags.go | 8 ++- service/history/api/startworkflow/api.go | 69 ++++++++++--------- service/history/handler.go | 6 +- .../history/workflow/mutable_state_impl.go | 2 +- 4 files changed, 45 insertions(+), 40 deletions(-) diff --git a/common/metrics/tags.go b/common/metrics/tags.go index a77f2dc7c01..e812333c8f9 100644 --- a/common/metrics/tags.go +++ b/common/metrics/tags.go @@ -281,8 +281,12 @@ func CacheTypeTag(value string) Tag { return &tagImpl{key: CacheTypeTagName, value: value} } +// ReasonString is just a string but the special type is defined here to remind callers of ReasonTag to limit the +// cardinality of possible reasons. +type ReasonString string + // ReasonTag is a generic tag can be used anywhere a reason is needed. // Make sure that the value is of limited cardinality. -func ReasonTag(value string) Tag { - return &tagImpl{key: reason, value: value} +func ReasonTag(value ReasonString) Tag { + return &tagImpl{key: reason, value: string(value)} } diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 9d1591ca1da..f5f1e7006ec 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -26,6 +26,7 @@ package startworkflow import ( "context" + "errors" "github.com/google/uuid" commonpb "go.temporal.io/api/common/v1" @@ -46,7 +47,7 @@ import ( "go.temporal.io/server/service/history/workflow" ) -type eagerStartDeniedReason string +type eagerStartDeniedReason metrics.ReasonString const ( eagerStartDeniedReasonDynamicConfigDisabled eagerStartDeniedReason = "dynamic_config_disabled" @@ -63,9 +64,9 @@ type Starter struct { namespace *namespace.Namespace } -// creationContext is a container for all information obtained from creating the uncommitted execution. +// creationParams is a container for all information obtained from creating the uncommitted execution. // The information is later used to create a new execution and handle conflicts. -type creationContext struct { +type creationParams struct { workflowID string runID string workflowContext api.WorkflowContext @@ -142,7 +143,7 @@ func (s *Starter) recordEagerDenied(reason eagerStartDeniedReason) { metrics.NamespaceTag(s.namespace.Name().String()), metrics.TaskQueueTag(s.request.StartRequest.TaskQueue.Name), metrics.WorkflowTypeTag(s.request.StartRequest.WorkflowType.Name), - metrics.ReasonTag(string(reason)), + metrics.ReasonTag(metrics.ReasonString(reason)), ) } @@ -161,25 +162,25 @@ func (s *Starter) Invoke( runID := uuid.NewString() - creationCtx, err := s.createNewMutableState(ctx, request.GetWorkflowId(), runID) + creationParams, err := s.createNewMutableState(ctx, request.GetWorkflowId(), runID) if err != nil { return nil, err } - err = s.createBrandNew(ctx, creationCtx) + err = s.createBrandNew(ctx, creationParams) if err == nil { - return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEventBatches)) + return s.generateResponse(creationParams.runID, creationParams.workflowTaskInfo, extractHistoryEvents(creationParams.workflowEventBatches)) } - t, ok := err.(*persistence.CurrentWorkflowConditionFailedError) - if !ok { + var currentWorkflowConditionFailedError *persistence.CurrentWorkflowConditionFailedError + if !errors.As(err, ¤tWorkflowConditionFailedError) { return nil, err } // The history and mutable state we generated above should be deleted by a background process. - return s.handleConflict(ctx, creationCtx, t) + return s.handleConflict(ctx, creationParams, currentWorkflowConditionFailedError) } // createNewMutableState creates a new workflow context, and closes its mutable state transaction as snapshot. // It returns the creationContext which can later be used to insert into the executions table. -func (s *Starter) createNewMutableState(ctx context.Context, workflowID string, runID string) (*creationContext, error) { +func (s *Starter) createNewMutableState(ctx context.Context, workflowID string, runID string) (*creationParams, error) { workflowContext, err := api.NewWorkflowWithSignal( ctx, s.shardCtx, @@ -210,7 +211,7 @@ func (s *Starter) createNewMutableState(ctx context.Context, workflowID string, return nil, serviceerror.NewInternal("unable to create 1st event batch") } - return &creationContext{ + return &creationParams{ workflowID: workflowID, runID: runID, workflowContext: workflowContext, @@ -221,17 +222,17 @@ func (s *Starter) createNewMutableState(ctx context.Context, workflowID string, } // createBrandNew creates a new "brand new" execution in the executions table. -func (s *Starter) createBrandNew(ctx context.Context, creationCtx *creationContext) error { +func (s *Starter) createBrandNew(ctx context.Context, creationParams *creationParams) error { now := s.shardCtx.GetTimeSource().Now() - return creationCtx.workflowContext.GetContext().CreateWorkflowExecution( + return creationParams.workflowContext.GetContext().CreateWorkflowExecution( ctx, now, persistence.CreateWorkflowModeBrandNew, "", // prevRunID 0, // prevLastWriteVersion - creationCtx.workflowContext.GetMutableState(), - creationCtx.workflowSnapshot, - creationCtx.workflowEventBatches, + creationParams.workflowContext.GetMutableState(), + creationParams.workflowSnapshot, + creationParams.workflowEventBatches, ) } @@ -240,49 +241,49 @@ func (s *Starter) createBrandNew(ctx context.Context, creationCtx *creationConte // RequestID) or simply because a different run exists for the same workflow. func (s *Starter) handleConflict( ctx context.Context, - creationCtx *creationContext, + creationParams *creationParams, currentWorkflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, ) (*historyservice.StartWorkflowExecutionResponse, error) { request := s.request.StartRequest if currentWorkflowConditionFailed.RequestID == request.GetRequestId() { return s.respondToRetriedRequest(ctx, currentWorkflowConditionFailed.RunID) } - if err := s.verifyNamespaceActive(creationCtx, currentWorkflowConditionFailed); err != nil { + if err := s.verifyNamespaceActive(creationParams, currentWorkflowConditionFailed); err != nil { return nil, err } - response, err := s.applyWorkflowIDReusePolicy(ctx, currentWorkflowConditionFailed, creationCtx) + response, err := s.applyWorkflowIDReusePolicy(ctx, currentWorkflowConditionFailed, creationParams) if err != nil { return nil, err } else if response != nil { return response, nil } - if err := s.createAsCurrent(ctx, creationCtx, currentWorkflowConditionFailed); err != nil { + if err := s.createAsCurrent(ctx, creationParams, currentWorkflowConditionFailed); err != nil { return nil, err } - return s.generateResponse(creationCtx.runID, creationCtx.workflowTaskInfo, extractHistoryEvents(creationCtx.workflowEventBatches)) + return s.generateResponse(creationParams.runID, creationParams.workflowTaskInfo, extractHistoryEvents(creationParams.workflowEventBatches)) } // createAsCurrent creates a new workflow execution and sets it to "current". func (s *Starter) createAsCurrent( ctx context.Context, - creationCtx *creationContext, + creationParams *creationParams, currentWorkflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, ) error { now := s.shardCtx.GetTimeSource().Now() - return creationCtx.workflowContext.GetContext().CreateWorkflowExecution( + return creationParams.workflowContext.GetContext().CreateWorkflowExecution( ctx, now, persistence.CreateWorkflowModeUpdateCurrent, currentWorkflowConditionFailed.RunID, currentWorkflowConditionFailed.LastWriteVersion, - creationCtx.workflowContext.GetMutableState(), - creationCtx.workflowSnapshot, - creationCtx.workflowEventBatches, + creationParams.workflowContext.GetMutableState(), + creationParams.workflowSnapshot, + creationParams.workflowEventBatches, ) } -func (s *Starter) verifyNamespaceActive(creationCtx *creationContext, currentWorkflowConditionFailed *persistence.CurrentWorkflowConditionFailedError) error { - if creationCtx.workflowContext.GetMutableState().GetCurrentVersion() < currentWorkflowConditionFailed.LastWriteVersion { +func (s *Starter) verifyNamespaceActive(creationParams *creationParams, currentWorkflowConditionFailed *persistence.CurrentWorkflowConditionFailedError) error { + if creationParams.workflowContext.GetMutableState().GetCurrentVersion() < currentWorkflowConditionFailed.LastWriteVersion { clusterMetadata := s.shardCtx.GetClusterMetadata() clusterName := clusterMetadata.ClusterNameForFailoverVersion(s.namespace.IsGlobalNamespace(), currentWorkflowConditionFailed.LastWriteVersion) return serviceerror.NewNamespaceNotActive( @@ -302,7 +303,7 @@ func (s *Starter) verifyNamespaceActive(creationCtx *creationContext, currentWor func (s *Starter) applyWorkflowIDReusePolicy( ctx context.Context, currentWorkflowConditionFailed *persistence.CurrentWorkflowConditionFailedError, - creationCtx *creationContext, + creationParams *creationParams, ) (*historyservice.StartWorkflowExecutionResponse, error) { workflowID := s.request.StartRequest.WorkflowId prevExecutionUpdateAction, err := api.ApplyWorkflowIDReusePolicy( @@ -311,7 +312,7 @@ func (s *Starter) applyWorkflowIDReusePolicy( currentWorkflowConditionFailed.State, currentWorkflowConditionFailed.Status, workflowID, - creationCtx.runID, + creationParams.runID, s.request.StartRequest.GetWorkflowIdReusePolicy(), ) if err != nil { @@ -339,7 +340,7 @@ func (s *Starter) applyWorkflowIDReusePolicy( s.shardCtx, s.namespace, workflowID, - creationCtx.runID, + creationParams.runID, s.request, nil) if err != nil { @@ -359,14 +360,14 @@ func (s *Starter) applyWorkflowIDReusePolicy( case nil: if !s.requestEagerStart() { return &historyservice.StartWorkflowExecutionResponse{ - RunId: creationCtx.runID, + RunId: creationParams.runID, }, nil } events, err := s.getWorkflowHistory(ctx, mutableStateInfo) if err != nil { return nil, err } - return s.generateResponse(creationCtx.runID, mutableStateInfo.workflowTaskInfo, events) + return s.generateResponse(creationParams.runID, mutableStateInfo.workflowTaskInfo, events) case consts.ErrWorkflowCompleted: // previous workflow already closed // fallthough to the logic for only creating the new workflow below diff --git a/service/history/handler.go b/service/history/handler.go index 31ddbbc60a9..d74a73c3160 100644 --- a/service/history/handler.go +++ b/service/history/handler.go @@ -530,9 +530,9 @@ func (h *Handler) StartWorkflowExecution(ctx context.Context, request *historyse } if response.Clock == nil { response.Clock, err = shardContext.NewVectorClock() - } - if err != nil { - return nil, h.convertError(err) + if err != nil { + return nil, h.convertError(err) + } } return response, nil } diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index f3ab37e5521..90762b5ced0 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -1726,7 +1726,7 @@ func (ms *MutableStateImpl) AddFirstWorkflowTaskScheduled( ) (int64, error) { opTag := tag.WorkflowActionWorkflowTaskScheduled if err := ms.checkMutability(opTag); err != nil { - return 0, err + return common.EmptyEventID, err } return ms.workflowTaskManager.AddFirstWorkflowTaskScheduled(startEvent, bypassTaskGeneration) }