From de6ca9c90cd70eaf1a3a669ac6860b7d3d897208 Mon Sep 17 00:00:00 2001
From: Rodrigo Zhou <rodrigozhou@users.noreply.github.com>
Date: Thu, 9 Feb 2023 21:28:09 -0800
Subject: [PATCH] Tokenize and build FTS query string for postgres and sqlite
 (#3933)

---
 .../store/sql/query_converter_postgresql.go   | 17 ++++++++++++++
 .../store/sql/query_converter_sqlite.go       | 23 ++++++++++---------
 .../store/sql/query_converter_util.go         | 14 +++++++++++
 3 files changed, 43 insertions(+), 11 deletions(-)

diff --git a/common/persistence/visibility/store/sql/query_converter_postgresql.go b/common/persistence/visibility/store/sql/query_converter_postgresql.go
index 6eb11b510dc..9a7f15025c1 100644
--- a/common/persistence/visibility/store/sql/query_converter_postgresql.go
+++ b/common/persistence/visibility/store/sql/query_converter_postgresql.go
@@ -156,6 +156,23 @@ func (c *pgQueryConverter) convertTextComparisonExpr(
 	if !isSupportedTextOperator(expr.Operator) {
 		return nil, query.NewConverterError("invalid query")
 	}
+	valueExpr, ok := expr.Right.(*unsafeSQLString)
+	if !ok {
+		return nil, query.NewConverterError(
+			"%s: unexpected value type (expected string, got %s)",
+			query.InvalidExpressionErrMessage,
+			sqlparser.String(expr.Right),
+		)
+	}
+	tokens := tokenizeTextQueryString(valueExpr.Val)
+	if len(tokens) == 0 {
+		return nil, query.NewConverterError(
+			"%s: unexpected value for Text type search attribute (no tokens found in %s)",
+			query.InvalidExpressionErrMessage,
+			sqlparser.String(expr.Right),
+		)
+	}
+	valueExpr.Val = strings.Join(tokens, " | ")
 	var newExpr sqlparser.Expr = &sqlparser.ComparisonExpr{
 		Operator: ftsMatchOp,
 		Left:     expr.Left,
diff --git a/common/persistence/visibility/store/sql/query_converter_sqlite.go b/common/persistence/visibility/store/sql/query_converter_sqlite.go
index 6f3a94f9760..c94ee41b056 100644
--- a/common/persistence/visibility/store/sql/query_converter_sqlite.go
+++ b/common/persistence/visibility/store/sql/query_converter_sqlite.go
@@ -179,22 +179,23 @@ func (c *sqliteQueryConverter) convertTextComparisonExpr(
 	}
 	colNameStr := sqlparser.String(colName)
 
-	value, err := query.ParseSqlValue(sqlparser.String(expr.Right))
-	if err != nil {
-		return nil, err
+	valueExpr, ok := expr.Right.(*unsafeSQLString)
+	if !ok {
+		return nil, query.NewConverterError(
+			"%s: unexpected value type (expected string, got %s)",
+			query.InvalidExpressionErrMessage,
+			sqlparser.String(expr.Right),
+		)
 	}
-
-	var ftsQuery string
-	switch v := value.(type) {
-	case string:
-		ftsQuery = fmt.Sprintf(`%s:(%s)`, colNameStr, v)
-	default:
+	tokens := tokenizeTextQueryString(valueExpr.Val)
+	if len(tokens) == 0 {
 		return nil, query.NewConverterError(
-			"%s: unexpected value type %T",
+			"%s: unexpected value for Text type search attribute (no tokens found in %s)",
 			query.InvalidExpressionErrMessage,
-			expr,
+			sqlparser.String(expr.Right),
 		)
 	}
+	ftsQuery := fmt.Sprintf("%s:(%s)", colNameStr, strings.Join(tokens, " OR "))
 
 	var oper string
 	switch expr.Operator {
diff --git a/common/persistence/visibility/store/sql/query_converter_util.go b/common/persistence/visibility/store/sql/query_converter_util.go
index 56ce50038cf..291d52268a4 100644
--- a/common/persistence/visibility/store/sql/query_converter_util.go
+++ b/common/persistence/visibility/store/sql/query_converter_util.go
@@ -25,6 +25,7 @@
 package sql
 
 import (
+	"strings"
 	"time"
 
 	"github.com/xwb1989/sqlparser"
@@ -94,3 +95,16 @@ func getMaxDatetimeValue() time.Time {
 	t, _ := time.Parse(time.RFC3339, "9999-12-31T23:59:59Z")
 	return t
 }
+
+// Simple tokenizer by spaces. It's a temporary solution as it doesn't cover tokenizer used by
+// PostgreSQL or SQLite.
+func tokenizeTextQueryString(s string) []string {
+	tokens := strings.Split(s, " ")
+	nonEmptyTokens := make([]string, 0, len(tokens))
+	for _, token := range tokens {
+		if token != "" {
+			nonEmptyTokens = append(nonEmptyTokens, token)
+		}
+	}
+	return nonEmptyTokens
+}