Skip to content

Commit

Permalink
Fix MySQL visibility indexes with close_time column (#3927)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rodrigozhou
rodrigozhou authored Feb 10, 2023
1 parent 75c05a2 commit 77a4487
Show file tree
Hide file tree
Showing 7 changed files with 74 additions and 51 deletions.
4 changes: 3 additions & 1 deletion common/persistence/visibility/store/sql/query_converter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ type (
) (string, []any)

getDatetimeFormat() string

getCoalesceCloseTimeExpr() sqlparser.Expr
}

QueryConverter struct {
Expand Down Expand Up @@ -347,7 +349,7 @@ func (c *QueryConverter) convertColName(
}
var newExpr sqlparser.Expr = newColName(searchattribute.GetSqlDbColName(saFieldName))
if saAlias == searchattribute.CloseTime {
newExpr = getCoalesceCloseTimeExpr(c.getDatetimeFormat())
newExpr = c.getCoalesceCloseTimeExpr()
}
*exprRef = newExpr
return saAlias, saFieldName, nil
Expand Down
24 changes: 19 additions & 5 deletions common/persistence/visibility/store/sql/query_converter_mysql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,10 @@ type (
mysqlQueryConverter struct{}
)

var convertTypeJSON = &sqlparser.ConvertType{Type: "json"}
var (
convertTypeDatetime = &sqlparser.ConvertType{Type: "datetime"}
convertTypeJSON = &sqlparser.ConvertType{Type: "json"}
)

var _ sqlparser.Expr = (*castExpr)(nil)
var _ sqlparser.Expr = (*memberOfExpr)(nil)
Expand Down Expand Up @@ -96,6 +99,17 @@ func (c *mysqlQueryConverter) getDatetimeFormat() string {
return "2006-01-02 15:04:05.999999"
}

func (c *mysqlQueryConverter) getCoalesceCloseTimeExpr() sqlparser.Expr {
return newFuncExpr(
coalesceFuncName,
newColName(searchattribute.GetSqlDbColName(searchattribute.CloseTime)),
&castExpr{
Value: newUnsafeSQLString(maxDatetimeValue.Format(c.getDatetimeFormat())),
Type: convertTypeDatetime,
},
)
}

func (c *mysqlQueryConverter) convertKeywordListComparisonExpr(
expr *sqlparser.ComparisonExpr,
) (sqlparser.Expr, error) {
Expand Down Expand Up @@ -201,12 +215,12 @@ func (c *mysqlQueryConverter) buildSelectStmt(
whereClauses,
fmt.Sprintf(
"((%s = ? AND %s = ? AND %s > ?) OR (%s = ? AND %s < ?) OR %s < ?)",
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
searchattribute.GetSqlDbColName(searchattribute.StartTime),
searchattribute.GetSqlDbColName(searchattribute.RunID),
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
searchattribute.GetSqlDbColName(searchattribute.StartTime),
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
),
)
queryArgs = append(
Expand Down Expand Up @@ -234,7 +248,7 @@ func (c *mysqlQueryConverter) buildSelectStmt(
searchattribute.GetSqlDbColName(searchattribute.NamespaceID),
searchattribute.GetSqlDbColName(searchattribute.RunID),
strings.Join(whereClauses, " AND "),
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
searchattribute.GetSqlDbColName(searchattribute.StartTime),
searchattribute.GetSqlDbColName(searchattribute.RunID),
), queryArgs
Expand Down
16 changes: 12 additions & 4 deletions common/persistence/visibility/store/sql/query_converter_postgresql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,14 @@ func (c *pgQueryConverter) getDatetimeFormat() string {
return "2006-01-02 15:04:05.999999"
}

func (c *pgQueryConverter) getCoalesceCloseTimeExpr() sqlparser.Expr {
return newFuncExpr(
coalesceFuncName,
newColName(searchattribute.GetSqlDbColName(searchattribute.CloseTime)),
newUnsafeSQLString(maxDatetimeValue.Format(c.getDatetimeFormat())),
)
}

func (c *pgQueryConverter) convertKeywordListComparisonExpr(
expr *sqlparser.ComparisonExpr,
) (sqlparser.Expr, error) {
Expand Down Expand Up @@ -197,12 +205,12 @@ func (c *pgQueryConverter) buildSelectStmt(
whereClauses,
fmt.Sprintf(
"((%s = ? AND %s = ? AND %s > ?) OR (%s = ? AND %s < ?) OR %s < ?)",
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
searchattribute.GetSqlDbColName(searchattribute.StartTime),
searchattribute.GetSqlDbColName(searchattribute.RunID),
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
searchattribute.GetSqlDbColName(searchattribute.StartTime),
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
),
)
queryArgs = append(
Expand All @@ -226,7 +234,7 @@ func (c *pgQueryConverter) buildSelectStmt(
LIMIT ?`,
strings.Join(sqlplugin.DbFields, ", "),
strings.Join(whereClauses, " AND "),
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
searchattribute.GetSqlDbColName(searchattribute.StartTime),
searchattribute.GetSqlDbColName(searchattribute.RunID),
), queryArgs
Expand Down
16 changes: 12 additions & 4 deletions common/persistence/visibility/store/sql/query_converter_sqlite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,14 @@ func (c *sqliteQueryConverter) getDatetimeFormat() string {
return "2006-01-02 15:04:05.999999-07:00"
}

func (c *sqliteQueryConverter) getCoalesceCloseTimeExpr() sqlparser.Expr {
return newFuncExpr(
coalesceFuncName,
newColName(searchattribute.GetSqlDbColName(searchattribute.CloseTime)),
newUnsafeSQLString(maxDatetimeValue.Format(c.getDatetimeFormat())),
)
}

//nolint:revive // cyclomatic complexity 17 (> 15)
func (c *sqliteQueryConverter) convertKeywordListComparisonExpr(
expr *sqlparser.ComparisonExpr,
Expand Down Expand Up @@ -233,12 +241,12 @@ func (c *sqliteQueryConverter) buildSelectStmt(
whereClauses,
fmt.Sprintf(
"((%s = ? AND %s = ? AND %s > ?) OR (%s = ? AND %s < ?) OR %s < ?)",
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
searchattribute.GetSqlDbColName(searchattribute.StartTime),
searchattribute.GetSqlDbColName(searchattribute.RunID),
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
searchattribute.GetSqlDbColName(searchattribute.StartTime),
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
),
)
queryArgs = append(
Expand All @@ -262,7 +270,7 @@ func (c *sqliteQueryConverter) buildSelectStmt(
LIMIT ?`,
strings.Join(sqlplugin.DbFields, ", "),
strings.Join(whereClauses, " AND "),
sqlparser.String(getCoalesceCloseTimeExpr(c.getDatetimeFormat())),
sqlparser.String(c.getCoalesceCloseTimeExpr()),
searchattribute.GetSqlDbColName(searchattribute.StartTime),
searchattribute.GetSqlDbColName(searchattribute.RunID),
), queryArgs
Expand Down
9 changes: 0 additions & 9 deletions common/persistence/visibility/store/sql/query_converter_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import (
"time"

"github.com/xwb1989/sqlparser"
"go.temporal.io/server/common/searchattribute"
)

type (
Expand Down Expand Up @@ -95,11 +94,3 @@ func getMaxDatetimeValue() time.Time {
t, _ := time.Parse(time.RFC3339, "9999-12-31T23:59:59Z")
return t
}

func getCoalesceCloseTimeExpr(format string) sqlparser.Expr {
return newFuncExpr(
coalesceFuncName,
newColName(searchattribute.GetSqlDbColName(searchattribute.CloseTime)),
newUnsafeSQLString(maxDatetimeValue.Format(format)),
)
}
28 changes: 14 additions & 14 deletions schema/mysql/v8/visibility/schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,22 +40,22 @@ CREATE TABLE executions_visibility (
PRIMARY KEY (namespace_id, run_id)
);

CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);

-- Indexes for the predefined search attributes
CREATE INDEX by_temporal_change_version ON executions_visibility (namespace_id, (CAST(TemporalChangeVersion AS CHAR(255) ARRAY)), (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_binary_checksums ON executions_visibility (namespace_id, (CAST(BinaryChecksums AS CHAR(255) ARRAY)), (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_batcher_user ON executions_visibility (namespace_id, BatcherUser, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_scheduled_start_time ON executions_visibility (namespace_id, TemporalScheduledStartTime, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_scheduled_by_id ON executions_visibility (namespace_id, TemporalScheduledById, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_schedule_paused ON executions_visibility (namespace_id, TemporalSchedulePaused, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_namespace_division ON executions_visibility (namespace_id, TemporalNamespaceDivision, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_change_version ON executions_visibility (namespace_id, (CAST(TemporalChangeVersion AS CHAR(255) ARRAY)), (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_binary_checksums ON executions_visibility (namespace_id, (CAST(BinaryChecksums AS CHAR(255) ARRAY)), (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_batcher_user ON executions_visibility (namespace_id, BatcherUser, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_scheduled_start_time ON executions_visibility (namespace_id, TemporalScheduledStartTime, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_scheduled_by_id ON executions_visibility (namespace_id, TemporalScheduledById, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_schedule_paused ON executions_visibility (namespace_id, TemporalSchedulePaused, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_namespace_division ON executions_visibility (namespace_id, TemporalNamespaceDivision, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);


CREATE TABLE custom_search_attributes (
Expand Down
28 changes: 14 additions & 14 deletions schema/mysql/v8/visibility/versioned/v1.2/advanced_visibility.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,22 +25,22 @@ DROP INDEX by_status_by_close_time ON executions_visibility;
DROP INDEX by_close_time_by_status ON executions_visibility;

-- Create new indexes
CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);

-- Indexes for the predefined search attributes
CREATE INDEX by_temporal_change_version ON executions_visibility (namespace_id, (CAST(TemporalChangeVersion AS CHAR(255) ARRAY)), (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_binary_checksums ON executions_visibility (namespace_id, (CAST(BinaryChecksums AS CHAR(255) ARRAY)), (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_batcher_user ON executions_visibility (namespace_id, BatcherUser, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_scheduled_start_time ON executions_visibility (namespace_id, TemporalScheduledStartTime, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_scheduled_by_id ON executions_visibility (namespace_id, TemporalScheduledById, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_schedule_paused ON executions_visibility (namespace_id, TemporalSchedulePaused, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_namespace_division ON executions_visibility (namespace_id, TemporalNamespaceDivision, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_change_version ON executions_visibility (namespace_id, (CAST(TemporalChangeVersion AS CHAR(255) ARRAY)), (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_binary_checksums ON executions_visibility (namespace_id, (CAST(BinaryChecksums AS CHAR(255) ARRAY)), (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_batcher_user ON executions_visibility (namespace_id, BatcherUser, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_scheduled_start_time ON executions_visibility (namespace_id, TemporalScheduledStartTime, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_scheduled_by_id ON executions_visibility (namespace_id, TemporalScheduledById, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_schedule_paused ON executions_visibility (namespace_id, TemporalSchedulePaused, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
CREATE INDEX by_temporal_namespace_division ON executions_visibility (namespace_id, TemporalNamespaceDivision, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);


-- Custom search attributes
Expand Down

0 comments on commit 77a4487

Please sign in to comment.