statement: support export (#778)

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd
+	github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
 	github.com/appleboy/gin-jwt/v2 v2.6.3
 	github.com/cenkalti/backoff/v4 v4.0.2
@@ -19,6 +20,7 @@ require (
 	github.com/jinzhu/gorm v1.9.12
 	github.com/joho/godotenv v1.3.0
 	github.com/joomcode/errorx v1.0.1
+	github.com/oleiade/reflections v1.0.0 // indirect
 	github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12
 	github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
 	github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c
@@ -42,4 +44,5 @@ require (
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
 	golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 // indirect
 	google.golang.org/grpc v1.25.1
+	gopkg.in/oleiade/reflections.v1 v1.0.0
 )

go.sum

@@ -12,6 +12,8 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd h1:59Whn6shj5MTVjTf2OX6+7iMcmY6h5CK0kTWwRaplL4=
 github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0=
+github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o=
+github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -226,6 +228,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=
+github.com/oleiade/reflections v1.0.0 h1:0ir4pc6v8/PJ0yw5AEtMddfXpWBXg9cnG7SgSoJuCgY=
+github.com/oleiade/reflections v1.0.0/go.mod h1:RbATFBbKYkVdqmSFtx13Bb/tVhR0lgOBXunWTZKeL4w=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
@@ -503,6 +507,8 @@ gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvR
 gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/oleiade/reflections.v1 v1.0.0 h1:nV9NFaFd5bXKjilVvPvA+/V/tNQk1pOEEc9gGWDkj+s=
+gopkg.in/oleiade/reflections.v1 v1.0.0/go.mod h1:SpA8pv+LUnF0FbB2hyRxc8XSng78D6iLBZ11PDb8Z5g=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/apiserver/statement/models.go

@@ -34,7 +34,8 @@ type TimeRange struct {
 }
 
 type Model struct {
-	AggPlanCount             int    `json:"plan_count" agg:"COUNT(DISTINCT plan_digest)"`
+	AggDigestText            string `json:"digest_text" agg:"ANY_VALUE(digest_text)"`
+	AggDigest                string `json:"digest" agg:"ANY_VALUE(digest)"`
 	AggExecCount             int    `json:"exec_count" agg:"SUM(exec_count)"`
 	AggSumErrors             int    `json:"sum_errors" agg:"SUM(sum_errors)"`
 	AggSumWarnings           int    `json:"sum_warnings" agg:"SUM(sum_warnings)"`
@@ -95,10 +96,9 @@ type Model struct {
 	AggSchemaName            string `json:"schema_name" agg:"ANY_VALUE(schema_name)"`
 	AggTableNames            string `json:"table_names" agg:"ANY_VALUE(table_names)"`
 	AggIndexNames            string `json:"index_names" agg:"ANY_VALUE(index_names)"`
-	AggDigestText            string `json:"digest_text" agg:"ANY_VALUE(digest_text)"`
-	AggDigest                string `json:"digest" agg:"ANY_VALUE(digest)"`
-	AggPlanDigest            string `json:"plan_digest" agg:"ANY_VALUE(plan_digest)"`
+	AggPlanCount             int    `json:"plan_count" agg:"COUNT(DISTINCT plan_digest)"`
 	AggPlan                  string `json:"plan" agg:"ANY_VALUE(plan)"`
+	AggPlanDigest            string `json:"plan_digest" agg:"ANY_VALUE(plan_digest)"`
 	// Computed fields
 	RelatedSchemas string `json:"related_schemas"`
 }

pkg/apiserver/statement/queries.go

@@ -130,15 +130,20 @@ func QueryStmtTypes(db *gorm.DB) (result []string, err error) {
 // schemas: ["tpcc", "test"]
 // stmtTypes: ["select", "update"]
 // fields: ["digest_text", "sum_latency"]
-func QueryStatementsOverview(
+func QueryStatements(
 	db *gorm.DB,
 	beginTime, endTime int,
 	schemas, stmtTypes []string,
 	text string,
 	fields []string,
 ) (result []Model, err error) {
-	fields = funk.UniqString(append(fields, "schema_name", "digest", "sum_latency")) // "schema_name", "digest" for group, "sum_latency" for order
-	aggrFields := getAggrFields(fields...)
+	var aggrFields []string
+	if len(fields) == 1 && fields[0] == "*" {
+		aggrFields = getAllAggrFields()
+	} else {
+		fields = funk.UniqString(append(fields, "schema_name", "digest", "sum_latency")) // "schema_name", "digest" for group, "sum_latency" for order
+		aggrFields = getAggrFields(fields...)
+	}
 
 	query := db.
 		Select(strings.Join(aggrFields, ", ")).

pkg/apiserver/statement/statement.go

@@ -14,11 +14,27 @@
 package statement
 
 import (
+	"encoding/base64"
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"reflect"
 	"strings"
+	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/gtank/cryptopasta"
+	"github.com/pingcap/log"
 	"go.uber.org/fx"
+	"go.uber.org/zap"
+
+	aesctr "github.com/Xeoncross/go-aesctr-with-hmac"
+
+	"gopkg.in/oleiade/reflections.v1"
 
 	"github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/user"
 	"github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/utils"
@@ -40,15 +56,23 @@ func NewService(p ServiceParams) *Service {
 
 func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 	endpoint := r.Group("/statements")
-	endpoint.Use(auth.MWAuthRequired())
-	endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient))
-	endpoint.GET("/config", s.configHandler)
-	endpoint.POST("/config", s.modifyConfigHandler)
-	endpoint.GET("/time_ranges", s.timeRangesHandler)
-	endpoint.GET("/stmt_types", s.stmtTypesHandler)
-	endpoint.GET("/overviews", s.overviewsHandler)
-	endpoint.GET("/plans", s.getPlansHandler)
-	endpoint.GET("/plan/detail", s.getPlanDetailHandler)
+	{
+		endpoint.GET("/download", s.downloadHandler)
+
+		endpoint.Use(auth.MWAuthRequired())
+		endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient))
+		{
+			endpoint.GET("/config", s.configHandler)
+			endpoint.POST("/config", s.modifyConfigHandler)
+			endpoint.GET("/time_ranges", s.timeRangesHandler)
+			endpoint.GET("/stmt_types", s.stmtTypesHandler)
+			endpoint.GET("/list", s.listHandler)
+			endpoint.GET("/plans", s.plansHandler)
+			endpoint.GET("/plan/detail", s.planDetailHandler)
+
+			endpoint.POST("/download/token", s.downloadTokenHandler)
+		}
+	}
 }
 
 // @Summary Get statement configurations
@@ -126,13 +150,13 @@ type GetStatementsRequest struct {
 	Fields    string   `json:"fields" form:"fields"`
 }
 
-// @Summary Get a list of statement overviews
+// @Summary Get a list of statements
 // @Param q query GetStatementsRequest true "Query"
 // @Success 200 {array} Model
-// @Router /statements/overviews [get]
+// @Router /statements/list [get]
 // @Security JwtAuth
 // @Failure 401 {object} utils.APIError "Unauthorized failure"
-func (s *Service) overviewsHandler(c *gin.Context) {
+func (s *Service) listHandler(c *gin.Context) {
 	var req GetStatementsRequest
 	if err := c.ShouldBindQuery(&req); err != nil {
 		utils.MakeInvalidRequestErrorFromError(c, err)
@@ -143,7 +167,7 @@ func (s *Service) overviewsHandler(c *gin.Context) {
 	if strings.TrimSpace(req.Fields) != "" {
 		fields = strings.Split(req.Fields, ",")
 	}
-	overviews, err := QueryStatementsOverview(
+	overviews, err := QueryStatements(
 		db,
 		req.BeginTime, req.EndTime,
 		req.Schemas,
@@ -170,7 +194,7 @@ type GetPlansRequest struct {
 // @Router /statements/plans [get]
 // @Security JwtAuth
 // @Failure 401 {object} utils.APIError "Unauthorized failure"
-func (s *Service) getPlansHandler(c *gin.Context) {
+func (s *Service) plansHandler(c *gin.Context) {
 	var req GetPlansRequest
 	if err := c.ShouldBindQuery(&req); err != nil {
 		utils.MakeInvalidRequestErrorFromError(c, err)
@@ -196,7 +220,7 @@ type GetPlanDetailRequest struct {
 // @Router /statements/plan/detail [get]
 // @Security JwtAuth
 // @Failure 401 {object} utils.APIError "Unauthorized failure"
-func (s *Service) getPlanDetailHandler(c *gin.Context) {
+func (s *Service) planDetailHandler(c *gin.Context) {
 	var req GetPlanDetailRequest
 	if err := c.ShouldBindQuery(&req); err != nil {
 		utils.MakeInvalidRequestErrorFromError(c, err)
@@ -210,3 +234,157 @@ func (s *Service) getPlanDetailHandler(c *gin.Context) {
 	}
 	c.JSON(http.StatusOK, result)
 }
+
+// @Router /statements/download/token [post]
+// @Summary Generate a download token for exported statements
+// @Produce plain
+// @Param request body GetStatementsRequest true "Request body"
+// @Success 200 {string} string "xxx"
+// @Security JwtAuth
+// @Failure 401 {object} utils.APIError "Unauthorized failure"
+func (s *Service) downloadTokenHandler(c *gin.Context) {
+	var req GetStatementsRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		utils.MakeInvalidRequestErrorFromError(c, err)
+		return
+	}
+	db := utils.GetTiDBConnection(c)
+	fields := []string{}
+	if strings.TrimSpace(req.Fields) != "" {
+		fields = strings.Split(req.Fields, ",")
+	}
+	overviews, err := QueryStatements(
+		db,
+		req.BeginTime, req.EndTime,
+		req.Schemas,
+		req.StmtTypes,
+		req.Text,
+		fields)
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	if len(overviews) == 0 {
+		utils.MakeInvalidRequestErrorFromError(c, errors.New("no data to export"))
+		return
+	}
+
+	// convert data
+	fieldsMap := make(map[string]string)
+	t := reflect.TypeOf(overviews[0])
+	fieldsNum := t.NumField()
+	allFields := make([]string, fieldsNum)
+	for i := 0; i < fieldsNum; i++ {
+		field := t.Field(i)
+		allFields[i] = strings.ToLower(field.Tag.Get("json"))
+		fieldsMap[allFields[i]] = field.Name
+	}
+	if len(fields) == 1 && fields[0] == "*" {
+		fields = allFields
+	}
+
+	csvData := [][]string{fields}
+	timeLayout := "01-02 15:04:05"
+	for _, overview := range overviews {
+		row := []string{}
+		for _, field := range fields {
+			filedName := fieldsMap[field]
+			s, _ := reflections.GetField(overview, filedName)
+			var val string
+			switch t := s.(type) {
+			case int:
+				if field == "first_seen" || field == "last_seen" {
+					val = time.Unix(int64(t), 0).Format(timeLayout)
+				} else {
+					val = fmt.Sprintf("%d", t)
+				}
+			default:
+				val = fmt.Sprintf("%s", t)
+			}
+			row = append(row, val)
+		}
+		csvData = append(csvData, row)
+	}
+
+	// generate temp file that persist encrypted data
+	timeLayout = "01021504"
+	beginTime := time.Unix(int64(req.BeginTime), 0).Format(timeLayout)
+	endTime := time.Unix(int64(req.EndTime), 0).Format(timeLayout)
+	csvFile, err := ioutil.TempFile("", fmt.Sprintf("statements_%s_%s_*.csv", beginTime, endTime))
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	defer csvFile.Close()
+
+	// generate encryption key
+	secretKey := *cryptopasta.NewEncryptionKey()
+
+	pr, pw := io.Pipe()
+	go func() {
+		csvwriter := csv.NewWriter(pw)
+		_ = csvwriter.WriteAll(csvData)
+		pw.Close()
+	}()
+	err = aesctr.Encrypt(pr, csvFile, secretKey[0:16], secretKey[16:])
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+
+	// generate token by filepath and secretKey
+	secretKeyStr := base64.StdEncoding.EncodeToString(secretKey[:])
+	token, err := utils.NewJWTString("statements/download", secretKeyStr+" "+csvFile.Name())
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	c.String(http.StatusOK, token)
+}
+
+// @Router /statements/download [get]
+// @Summary Download statements
+// @Produce text/csv
+// @Param token query string true "download token"
+// @Failure 400 {object} utils.APIError
+// @Failure 401 {object} utils.APIError "Unauthorized failure"
+func (s *Service) downloadHandler(c *gin.Context) {
+	token := c.Query("token")
+	tokenPlain, err := utils.ParseJWTString("statements/download", token)
+	if err != nil {
+		utils.MakeInvalidRequestErrorFromError(c, err)
+		return
+	}
+	arr := strings.Fields(tokenPlain)
+	if len(arr) != 2 {
+		utils.MakeInvalidRequestErrorFromError(c, errors.New("invalid token"))
+		return
+	}
+	secretKey, err := base64.StdEncoding.DecodeString(arr[0])
+	if err != nil {
+		utils.MakeInvalidRequestErrorFromError(c, err)
+		return
+	}
+
+	filePath := arr[1]
+	fileInfo, err := os.Stat(filePath)
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	f, err := os.Open(filePath)
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+
+	c.Writer.Header().Set("Content-type", "text/csv")
+	c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileInfo.Name()))
+	err = aesctr.Decrypt(f, c.Writer, secretKey[0:16], secretKey[16:])
+	if err != nil {
+		log.Error("decrypt csv failed", zap.Error(err))
+	}
+	// delete it anyway
+	f.Close()
+	_ = os.Remove(filePath)
+}

ui/lib/apps/SearchLogs/components/SearchProgress.tsx

@@ -154,7 +154,7 @@ export default function SearchProgress({
       return
     }
     const url = `${client.getBasePath()}/logs/download?token=${token}`
-    window.open(url)
+    window.location.href = url
   }
 
   async function handleCancel() {