From 40a2ebeeb785af1793500b2ca3f6e5a92ac1ddf1 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Thu, 22 Oct 2020 21:51:06 +0800 Subject: [PATCH 01/20] wip: export statements --- pkg/apiserver/statement/statement.go | 91 +++++++++++++++++++--- ui/lib/apps/Statement/pages/List/index.tsx | 3 + ui/lib/apps/Statement/translations/en.yaml | 1 + ui/lib/apps/Statement/translations/zh.yaml | 1 + 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index 2290dff902..d647ca6df4 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -14,6 +14,7 @@ package statement import ( + "errors" "net/http" "strings" @@ -40,15 +41,23 @@ func NewService(p ServiceParams) *Service { func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) { endpoint := r.Group("/statements") - endpoint.Use(auth.MWAuthRequired()) - endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient)) - endpoint.GET("/config", s.configHandler) - endpoint.POST("/config", s.modifyConfigHandler) - endpoint.GET("/time_ranges", s.timeRangesHandler) - endpoint.GET("/stmt_types", s.stmtTypesHandler) - endpoint.GET("/overviews", s.overviewsHandler) - endpoint.GET("/plans", s.getPlansHandler) - endpoint.GET("/plan/detail", s.getPlanDetailHandler) + { + endpoint.GET("/download", s.downloadHandler) + + endpoint.Use(auth.MWAuthRequired()) + endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient)) + { + endpoint.GET("/config", s.configHandler) + endpoint.POST("/config", s.modifyConfigHandler) + endpoint.GET("/time_ranges", s.timeRangesHandler) + endpoint.GET("/stmt_types", s.stmtTypesHandler) + endpoint.GET("/overviews", s.overviewsHandler) + endpoint.GET("/plans", s.getPlansHandler) + endpoint.GET("/plan/detail", s.getPlanDetailHandler) + + endpoint.GET("/download/acquire_token", s.downloadTokenHandler) + } + } } // @Summary Get statement configurations @@ -210,3 +219,67 @@ func (s *Service) getPlanDetailHandler(c *gin.Context) { } c.JSON(http.StatusOK, result) } + +// @Router /statements/download/acquire_token [get] +// @Summary Generate a download token for downloading statements +// @Produce plain +// @Param time query string true "Query" +// @Success 200 {string} string "xxx" +// @Security JwtAuth +// @Failure 401 {object} utils.APIError "Unauthorized failure" +func (s *Service) downloadTokenHandler(c *gin.Context) { + time := c.Query("time") + token, err := utils.NewJWTString("statements/download", time) + if err != nil { + _ = c.Error(err) + return + } + c.String(http.StatusOK, token) +} + +type DownloadStatementsRequest struct { + GetStatementsRequest + Token string `json:"token" form:"token"` +} + +// @Router /statements/download [get] +// @Summary Download statements +// @Produce application/x-tar,application/zip +// @Param q query DownloadStatementsRequest true "Query" +// @Failure 400 {object} utils.APIError +// @Failure 401 {object} utils.APIError "Unauthorized failure" +func (s *Service) downloadHandler(c *gin.Context) { + var req DownloadStatementsRequest + if err := c.ShouldBindQuery(&req); err != nil { + utils.MakeInvalidRequestErrorFromError(c, err) + return + } + str, err := utils.ParseJWTString("statements/download", req.Token) + if err != nil { + utils.MakeInvalidRequestErrorFromError(c, err) + return + } + time := string(req.BeginTime) + "-" + string(req.EndTime) + if str != time { + utils.MakeInvalidRequestErrorFromError(c, errors.New("invalid token")) + return + } + + db := utils.GetTiDBConnection(c) + fields := []string{} + if strings.TrimSpace(req.Fields) != "" { + fields = strings.Split(req.Fields, ",") + } + overviews, err := QueryStatementsOverview( + db, + req.BeginTime, req.EndTime, + req.Schemas, + req.StmtTypes, + req.Text, + fields) + if err != nil { + _ = c.Error(err) + return + } + // TODO +} diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index 7590375686..b01b6082fa 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -106,6 +106,9 @@ export default function StatementsOverview() { setQueryOptions({ ...queryOptions, searchText }) } /> + diff --git a/ui/lib/apps/Statement/translations/en.yaml b/ui/lib/apps/Statement/translations/en.yaml index f91128ac54..ebb2086698 100644 --- a/ui/lib/apps/Statement/translations/en.yaml +++ b/ui/lib/apps/Statement/translations/en.yaml @@ -37,6 +37,7 @@ statement: recent: Recent usual_time_ranges: Common custom_time_ranges: Custom + export: export settings: title: Settings disabled_result: diff --git a/ui/lib/apps/Statement/translations/zh.yaml b/ui/lib/apps/Statement/translations/zh.yaml index 2602746f16..9af4a985cf 100644 --- a/ui/lib/apps/Statement/translations/zh.yaml +++ b/ui/lib/apps/Statement/translations/zh.yaml @@ -37,6 +37,7 @@ statement: recent: 最近 usual_time_ranges: 常用时间范围 custom_time_ranges: 自定义时间范围 + export: 导出 settings: title: 设置 disabled_result: From 4f1de359a021e274db6ac0a27389fb573508fea2 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Fri, 23 Oct 2020 17:52:41 +0800 Subject: [PATCH 02/20] generate csv file --- pkg/apiserver/statement/statement.go | 110 ++++++++++++------ ui/lib/apps/Statement/pages/List/index.tsx | 14 ++- .../utils/useStatementTableController.ts | 17 +++ 3 files changed, 106 insertions(+), 35 deletions(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index d647ca6df4..b5621bcf3c 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -14,8 +14,13 @@ package statement import ( + "encoding/csv" "errors" + "fmt" + "io/ioutil" "net/http" + "reflect" + "strconv" "strings" "github.com/gin-gonic/gin" @@ -223,48 +228,16 @@ func (s *Service) getPlanDetailHandler(c *gin.Context) { // @Router /statements/download/acquire_token [get] // @Summary Generate a download token for downloading statements // @Produce plain -// @Param time query string true "Query" +// @Param q query GetStatementsRequest true "Query" // @Success 200 {string} string "xxx" // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" func (s *Service) downloadTokenHandler(c *gin.Context) { - time := c.Query("time") - token, err := utils.NewJWTString("statements/download", time) - if err != nil { - _ = c.Error(err) - return - } - c.String(http.StatusOK, token) -} - -type DownloadStatementsRequest struct { - GetStatementsRequest - Token string `json:"token" form:"token"` -} - -// @Router /statements/download [get] -// @Summary Download statements -// @Produce application/x-tar,application/zip -// @Param q query DownloadStatementsRequest true "Query" -// @Failure 400 {object} utils.APIError -// @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) downloadHandler(c *gin.Context) { - var req DownloadStatementsRequest + var req GetStatementsRequest if err := c.ShouldBindQuery(&req); err != nil { utils.MakeInvalidRequestErrorFromError(c, err) return } - str, err := utils.ParseJWTString("statements/download", req.Token) - if err != nil { - utils.MakeInvalidRequestErrorFromError(c, err) - return - } - time := string(req.BeginTime) + "-" + string(req.EndTime) - if str != time { - utils.MakeInvalidRequestErrorFromError(c, errors.New("invalid token")) - return - } - db := utils.GetTiDBConnection(c) fields := []string{} if strings.TrimSpace(req.Fields) != "" { @@ -281,5 +254,74 @@ func (s *Service) downloadHandler(c *gin.Context) { _ = c.Error(err) return } + if len(overviews) == 0 { + utils.MakeInvalidRequestErrorFromError(c, errors.New("no data to export")) + return + } + // TODO + // convert data + fieldsMap := make(map[string]string) + t := reflect.TypeOf(overviews[0]) + fieldsNum := t.NumField() + for i := 0; i < fieldsNum; i++ { + field := t.Field(i) + fieldsMap[strings.ToLower(field.Tag.Get("json"))] = field.Name + } + + csvData := [][]string{fields} + for _, overview := range overviews { + row := []string{} + v := reflect.ValueOf(overview) + for _, field := range fields { + filedName := fieldsMap[field] + s := v.FieldByName(filedName) + var val string + switch s.Interface().(type) { + case int: + val = strconv.FormatInt(s.Int(), 10) + case string: + val = s.String() + } + row = append(row, val) + } + csvData = append(csvData, row) + } + + // generate csv + // get token by filename + tmpfile, err := ioutil.TempFile("", "statements") + if err != nil { + _ = c.Error(err) + return + } + defer tmpfile.Close() + csvwriter := csv.NewWriter(tmpfile) + for _, csvRow := range csvData { + _ = csvwriter.Write(csvRow) + } + csvwriter.Flush() + fmt.Println("name:", tmpfile.Name()) + + // generate token + token, err := utils.NewJWTString("statements/download", tmpfile.Name()) + c.String(http.StatusOK, token) +} + +// @Router /statements/download [get] +// @Summary Download statements +// @Produce application/x-tar,application/zip +// @Param token query string true "download token" +// @Failure 400 {object} utils.APIError +// @Failure 401 {object} utils.APIError "Unauthorized failure" +func (s *Service) downloadHandler(c *gin.Context) { + token := c.Query("token") + str, err := utils.ParseJWTString("statements/download", token) + if err != nil { + utils.MakeInvalidRequestErrorFromError(c, err) + return + } + // str is filename + // send back this file + c.String(http.StatusOK, str) } diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index b01b6082fa..b7ea2ed2f0 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -17,6 +17,7 @@ import TimeRangeSelector from './TimeRangeSelector' import useStatementTableController, { DEF_STMT_COLUMN_KEYS, } from '../../utils/useStatementTableController' +import client from '@lib/client' const { Search } = Input @@ -47,8 +48,19 @@ export default function StatementsOverview() { allStmtTypes, loadingStatements, tableColumns, + getDownloadToken, } = controller + async function exportCSV() { + const res = await getDownloadToken() + const token = res.data + if (!token) { + return + } + const url = `${client.getBasePath()}/statements/download?token=${token}` + window.open(url) + } + return (
@@ -106,7 +118,7 @@ export default function StatementsOverview() { setQueryOptions({ ...queryOptions, searchText }) } /> - diff --git a/ui/lib/apps/Statement/utils/useStatementTableController.ts b/ui/lib/apps/Statement/utils/useStatementTableController.ts index ddfbf41b0d..7d56777f26 100644 --- a/ui/lib/apps/Statement/utils/useStatementTableController.ts +++ b/ui/lib/apps/Statement/utils/useStatementTableController.ts @@ -67,6 +67,8 @@ export interface IStatementTableController { tableColumns: IColumn[] visibleColumnKeys: IColumnKeys + + getDownloadToken: () => Promise } export default function useStatementTableController( @@ -218,6 +220,19 @@ export default function useStatementTableController( queryStatementList() }, [queryOptions, allTimeRanges, validTimeRange, selectedFields]) + function getDownloadToken() { + return client + .getInstance() + .statementsDownloadAcquireTokenGet( + validTimeRange.begin_time!, + validTimeRange.end_time!, + selectedFields, + queryOptions.schemas, + queryOptions.stmtTypes, + queryOptions.searchText + ) + } + return { queryOptions, setQueryOptions, @@ -237,5 +252,7 @@ export default function useStatementTableController( tableColumns, visibleColumnKeys, + + getDownloadToken, } } From 208725a1a42d0a0a8f7b5a1e80afeab618f07eb1 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Mon, 26 Oct 2020 11:21:44 +0800 Subject: [PATCH 03/20] implement download --- pkg/apiserver/statement/statement.go | 25 +++++++------- ui/lib/apps/Statement/pages/List/index.tsx | 14 ++++---- .../utils/useStatementTableController.ts | 33 ++++++++++++------- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index b5621bcf3c..d31a8256bc 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -24,7 +24,9 @@ import ( "strings" "github.com/gin-gonic/gin" + "github.com/pingcap/log" "go.uber.org/fx" + "go.uber.org/zap" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/user" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/utils" @@ -259,7 +261,6 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { return } - // TODO // convert data fieldsMap := make(map[string]string) t := reflect.TypeOf(overviews[0]) @@ -280,7 +281,7 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { switch s.Interface().(type) { case int: val = strconv.FormatInt(s.Int(), 10) - case string: + default: val = s.String() } row = append(row, val) @@ -289,8 +290,7 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { } // generate csv - // get token by filename - tmpfile, err := ioutil.TempFile("", "statements") + tmpfile, err := ioutil.TempFile("", fmt.Sprintf("statements_%d_%d_*.csv", req.BeginTime, req.EndTime)) if err != nil { _ = c.Error(err) return @@ -301,27 +301,30 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { _ = csvwriter.Write(csvRow) } csvwriter.Flush() - fmt.Println("name:", tmpfile.Name()) - // generate token + // generate token by filepath token, err := utils.NewJWTString("statements/download", tmpfile.Name()) c.String(http.StatusOK, token) } // @Router /statements/download [get] // @Summary Download statements -// @Produce application/x-tar,application/zip +// @Produce application/zip // @Param token query string true "download token" // @Failure 400 {object} utils.APIError // @Failure 401 {object} utils.APIError "Unauthorized failure" func (s *Service) downloadHandler(c *gin.Context) { token := c.Query("token") - str, err := utils.ParseJWTString("statements/download", token) + fielPath, err := utils.ParseJWTString("statements/download", token) if err != nil { utils.MakeInvalidRequestErrorFromError(c, err) return } - // str is filename - // send back this file - c.String(http.StatusOK, str) + + c.Writer.Header().Set("Content-type", "application/octet-stream") + c.Writer.Header().Set("Content-Disposition", "attachment; filename=\"statements.zip\"") + err = utils.StreamZipPack(c.Writer, []string{fielPath}, true) + if err != nil { + log.Error("Stream zip pack failed", zap.Error(err)) + } } diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index b7ea2ed2f0..b234547d7c 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -48,17 +48,17 @@ export default function StatementsOverview() { allStmtTypes, loadingStatements, tableColumns, + getDownloadToken, + downloading, } = controller async function exportCSV() { - const res = await getDownloadToken() - const token = res.data - if (!token) { - return + const token = await getDownloadToken() + if (token) { + const url = `${client.getBasePath()}/statements/download?token=${token}` + window.open(url) } - const url = `${client.getBasePath()}/statements/download?token=${token}` - window.open(url) } return ( @@ -118,7 +118,7 @@ export default function StatementsOverview() { setQueryOptions({ ...queryOptions, searchText }) } /> - diff --git a/ui/lib/apps/Statement/utils/useStatementTableController.ts b/ui/lib/apps/Statement/utils/useStatementTableController.ts index 7d56777f26..55c91222cd 100644 --- a/ui/lib/apps/Statement/utils/useStatementTableController.ts +++ b/ui/lib/apps/Statement/utils/useStatementTableController.ts @@ -69,6 +69,7 @@ export interface IStatementTableController { visibleColumnKeys: IColumnKeys getDownloadToken: () => Promise + downloading: boolean } export default function useStatementTableController( @@ -220,17 +221,26 @@ export default function useStatementTableController( queryStatementList() }, [queryOptions, allTimeRanges, validTimeRange, selectedFields]) - function getDownloadToken() { - return client - .getInstance() - .statementsDownloadAcquireTokenGet( - validTimeRange.begin_time!, - validTimeRange.end_time!, - selectedFields, - queryOptions.schemas, - queryOptions.stmtTypes, - queryOptions.searchText - ) + const [downloading, setDownloading] = useState(false) + async function getDownloadToken() { + let token = '' + try { + setDownloading(true) + const res = await client + .getInstance() + .statementsDownloadAcquireTokenGet( + validTimeRange.begin_time!, + validTimeRange.end_time!, + selectedFields, + queryOptions.schemas, + queryOptions.stmtTypes, + queryOptions.searchText + ) + token = res.data + } finally { + setDownloading(false) + } + return token } return { @@ -254,5 +264,6 @@ export default function useStatementTableController( visibleColumnKeys, getDownloadToken, + downloading, } } From 7d8d9149d2e52fa2dc3bc236064f50b19f79e0ac Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Mon, 26 Oct 2020 11:42:20 +0800 Subject: [PATCH 04/20] refine --- pkg/apiserver/statement/statement.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index d31a8256bc..dc026e598c 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -19,6 +19,7 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "reflect" "strconv" "strings" @@ -321,6 +322,12 @@ func (s *Service) downloadHandler(c *gin.Context) { return } + _, err = os.Stat(fielPath) + if err != nil { + _ = c.Error(err) + return + } + c.Writer.Header().Set("Content-type", "application/octet-stream") c.Writer.Header().Set("Content-Disposition", "attachment; filename=\"statements.zip\"") err = utils.StreamZipPack(c.Writer, []string{fielPath}, true) From 0023c63d55958f80e936d09e3bfa0bddf8fc0898 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Mon, 26 Oct 2020 13:09:22 +0800 Subject: [PATCH 05/20] fix CI --- pkg/apiserver/statement/statement.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index dc026e598c..38acdb1191 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -305,6 +305,10 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { // generate token by filepath token, err := utils.NewJWTString("statements/download", tmpfile.Name()) + if err != nil { + _ = c.Error(err) + return + } c.String(http.StatusOK, token) } From 87c31774d6a836ef47837405d899ec30c62589d5 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Tue, 27 Oct 2020 10:19:15 +0800 Subject: [PATCH 06/20] refine, address comments --- pkg/apiserver/statement/statement.go | 6 +++++- ui/lib/apps/Statement/utils/useStatementTableController.ts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index 38acdb1191..4b3984bc89 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -23,6 +23,7 @@ import ( "reflect" "strconv" "strings" + "time" "github.com/gin-gonic/gin" "github.com/pingcap/log" @@ -291,7 +292,10 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { } // generate csv - tmpfile, err := ioutil.TempFile("", fmt.Sprintf("statements_%d_%d_*.csv", req.BeginTime, req.EndTime)) + timeLayout := "01021504" + beginTime := time.Unix(int64(req.BeginTime), 0).Format(timeLayout) + endTime := time.Unix(int64(req.EndTime), 0).Format(timeLayout) + tmpfile, err := ioutil.TempFile("", fmt.Sprintf("statements_%s_%s_*.csv", beginTime, endTime)) if err != nil { _ = c.Error(err) return diff --git a/ui/lib/apps/Statement/utils/useStatementTableController.ts b/ui/lib/apps/Statement/utils/useStatementTableController.ts index 55c91222cd..f12e01e9d6 100644 --- a/ui/lib/apps/Statement/utils/useStatementTableController.ts +++ b/ui/lib/apps/Statement/utils/useStatementTableController.ts @@ -68,7 +68,7 @@ export interface IStatementTableController { tableColumns: IColumn[] visibleColumnKeys: IColumnKeys - getDownloadToken: () => Promise + getDownloadToken: () => Promise downloading: boolean } From b679283fe2efea4e8076ee9f0ffeed723316be4d Mon Sep 17 00:00:00 2001 From: Sparkle <1284531+baurine@users.noreply.github.com> Date: Tue, 27 Oct 2020 17:11:24 +0800 Subject: [PATCH 07/20] refine comment Co-authored-by: Wenxuan --- pkg/apiserver/statement/statement.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index 4b3984bc89..6dcaee39f6 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -230,7 +230,7 @@ func (s *Service) getPlanDetailHandler(c *gin.Context) { } // @Router /statements/download/acquire_token [get] -// @Summary Generate a download token for downloading statements +// @Summary Generate a download token for exported statements // @Produce plain // @Param q query GetStatementsRequest true "Query" // @Success 200 {string} string "xxx" From 864e209b0f0d77d7d5d4fb921fbfadea86215683 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Wed, 28 Oct 2020 16:34:56 +0800 Subject: [PATCH 08/20] wip: encrypt csv --- pkg/apiserver/statement/statement.go | 41 ++++++++++++++----- .../utils/useStatementTableController.ts | 18 ++++---- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index 6dcaee39f6..20433482b9 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -14,6 +14,8 @@ package statement import ( + "bufio" + "bytes" "encoding/csv" "errors" "fmt" @@ -26,6 +28,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/gtank/cryptopasta" "github.com/pingcap/log" "go.uber.org/fx" "go.uber.org/zap" @@ -64,7 +67,7 @@ func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) { endpoint.GET("/plans", s.getPlansHandler) endpoint.GET("/plan/detail", s.getPlanDetailHandler) - endpoint.GET("/download/acquire_token", s.downloadTokenHandler) + endpoint.POST("/download/token", s.downloadTokenHandler) } } } @@ -229,16 +232,16 @@ func (s *Service) getPlanDetailHandler(c *gin.Context) { c.JSON(http.StatusOK, result) } -// @Router /statements/download/acquire_token [get] +// @Router /statements/download/token [post] // @Summary Generate a download token for exported statements // @Produce plain -// @Param q query GetStatementsRequest true "Query" +// @Param request body GetStatementsRequest true "Request body" // @Success 200 {string} string "xxx" // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" func (s *Service) downloadTokenHandler(c *gin.Context) { var req GetStatementsRequest - if err := c.ShouldBindQuery(&req); err != nil { + if err := c.ShouldBindJSON(&req); err != nil { utils.MakeInvalidRequestErrorFromError(c, err) return } @@ -291,24 +294,42 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { csvData = append(csvData, row) } - // generate csv + // generate temp file that persist encrypted data timeLayout := "01021504" beginTime := time.Unix(int64(req.BeginTime), 0).Format(timeLayout) endTime := time.Unix(int64(req.EndTime), 0).Format(timeLayout) - tmpfile, err := ioutil.TempFile("", fmt.Sprintf("statements_%s_%s_*.csv", beginTime, endTime)) + csvFile, err := ioutil.TempFile("", fmt.Sprintf("statements_%s_%s_*.csv", beginTime, endTime)) if err != nil { _ = c.Error(err) return } - defer tmpfile.Close() - csvwriter := csv.NewWriter(tmpfile) + defer csvFile.Close() + // generate buf writer for tmpfile + csvBuf := bufio.NewWriter(csvFile) + // generate encrypted key + secretKey := cryptopasta.NewEncryptionKey() + + rowBuf := bytes.NewBuffer(nil) + csvwriter := csv.NewWriter(rowBuf) for _, csvRow := range csvData { _ = csvwriter.Write(csvRow) + csvwriter.Flush() + row := make([]byte, len(rowBuf.String())) + _, _ = rowBuf.Read(row) + + encrypted, err := cryptopasta.Encrypt(row, secretKey) + if err != nil { + _ = c.Error(err) + return + } + csvBuf.Write(encrypted) } - csvwriter.Flush() + csvBuf.Flush() + + // zip file and encrypt // generate token by filepath - token, err := utils.NewJWTString("statements/download", tmpfile.Name()) + token, err := utils.NewJWTString("statements/download", csvFile.Name()) if err != nil { _ = c.Error(err) return diff --git a/ui/lib/apps/Statement/utils/useStatementTableController.ts b/ui/lib/apps/Statement/utils/useStatementTableController.ts index f12e01e9d6..e47fb9827f 100644 --- a/ui/lib/apps/Statement/utils/useStatementTableController.ts +++ b/ui/lib/apps/Statement/utils/useStatementTableController.ts @@ -226,16 +226,14 @@ export default function useStatementTableController( let token = '' try { setDownloading(true) - const res = await client - .getInstance() - .statementsDownloadAcquireTokenGet( - validTimeRange.begin_time!, - validTimeRange.end_time!, - selectedFields, - queryOptions.schemas, - queryOptions.stmtTypes, - queryOptions.searchText - ) + const res = await client.getInstance().statementsDownloadTokenPost({ + begin_time: validTimeRange.begin_time, + end_time: validTimeRange.end_time, + fields: selectedFields, + schemas: queryOptions.schemas, + stmt_types: queryOptions.stmtTypes, + text: queryOptions.searchText, + }) token = res.data } finally { setDownloading(false) From f96058f74d593196e6b2f6aa962ba900a9c0b39e Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Thu, 29 Oct 2020 11:09:26 +0800 Subject: [PATCH 09/20] refine the encrypt --- go.mod | 1 + go.sum | 2 ++ pkg/apiserver/statement/statement.go | 38 +++++++++++----------------- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 947eb421dd..c9704ff7e9 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd + github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/appleboy/gin-jwt/v2 v2.6.3 github.com/cenkalti/backoff/v4 v4.0.2 diff --git a/go.sum b/go.sum index 5cb1fd340d..7b6b60877d 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd h1:59Whn6shj5MTVjTf2OX6+7iMcmY6h5CK0kTWwRaplL4= github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0= +github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= +github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index 20433482b9..b3964a2de6 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -14,11 +14,10 @@ package statement import ( - "bufio" - "bytes" "encoding/csv" "errors" "fmt" + "io" "io/ioutil" "net/http" "os" @@ -33,6 +32,7 @@ import ( "go.uber.org/fx" "go.uber.org/zap" + aesctr "github.com/Xeoncross/go-aesctr-with-hmac" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/user" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/utils" "github.com/pingcap-incubator/tidb-dashboard/pkg/tidb" @@ -304,31 +304,23 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { return } defer csvFile.Close() - // generate buf writer for tmpfile - csvBuf := bufio.NewWriter(csvFile) - // generate encrypted key - secretKey := cryptopasta.NewEncryptionKey() - rowBuf := bytes.NewBuffer(nil) - csvwriter := csv.NewWriter(rowBuf) - for _, csvRow := range csvData { - _ = csvwriter.Write(csvRow) - csvwriter.Flush() - row := make([]byte, len(rowBuf.String())) - _, _ = rowBuf.Read(row) + // generate encryption key + secretKey := cryptopasta.NewEncryptionKey() - encrypted, err := cryptopasta.Encrypt(row, secretKey) - if err != nil { - _ = c.Error(err) - return - } - csvBuf.Write(encrypted) + pr, pw := io.Pipe() + go func() { + csvwriter := csv.NewWriter(pw) + _ = csvwriter.WriteAll(csvData) + pw.Close() + }() + err = aesctr.Encrypt(pr, csvFile, (*secretKey)[0:16], (*secretKey)[16:]) + if err != nil { + _ = c.Error(err) + return } - csvBuf.Flush() - - // zip file and encrypt - // generate token by filepath + // generate token by filepath and secretKey token, err := utils.NewJWTString("statements/download", csvFile.Name()) if err != nil { _ = c.Error(err) From 8335ae48e3d08944e76ab0acd3e2a0f9b65c5321 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Thu, 29 Oct 2020 11:50:33 +0800 Subject: [PATCH 10/20] decrypt when downloading --- pkg/apiserver/statement/statement.go | 37 ++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index b3964a2de6..445538787f 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -14,6 +14,7 @@ package statement import ( + "encoding/base64" "encoding/csv" "errors" "fmt" @@ -306,7 +307,7 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { defer csvFile.Close() // generate encryption key - secretKey := cryptopasta.NewEncryptionKey() + secretKey := *cryptopasta.NewEncryptionKey() pr, pw := io.Pipe() go func() { @@ -314,14 +315,15 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { _ = csvwriter.WriteAll(csvData) pw.Close() }() - err = aesctr.Encrypt(pr, csvFile, (*secretKey)[0:16], (*secretKey)[16:]) + err = aesctr.Encrypt(pr, csvFile, secretKey[0:16], secretKey[16:]) if err != nil { _ = c.Error(err) return } // generate token by filepath and secretKey - token, err := utils.NewJWTString("statements/download", csvFile.Name()) + secretKeyStr := base64.StdEncoding.EncodeToString(secretKey[:]) + token, err := utils.NewJWTString("statements/download", secretKeyStr+" "+csvFile.Name()) if err != nil { _ = c.Error(err) return @@ -337,22 +339,41 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { // @Failure 401 {object} utils.APIError "Unauthorized failure" func (s *Service) downloadHandler(c *gin.Context) { token := c.Query("token") - fielPath, err := utils.ParseJWTString("statements/download", token) + tokenPlain, err := utils.ParseJWTString("statements/download", token) + if err != nil { + utils.MakeInvalidRequestErrorFromError(c, err) + return + } + arr := strings.Fields(tokenPlain) + if len(arr) != 2 { + utils.MakeInvalidRequestErrorFromError(c, errors.New("invalid token")) + return + } + secretKey, err := base64.StdEncoding.DecodeString(arr[0]) if err != nil { utils.MakeInvalidRequestErrorFromError(c, err) return } - _, err = os.Stat(fielPath) + filePath := arr[1] + fileInfo, err := os.Stat(filePath) + if err != nil { + _ = c.Error(err) + return + } + f, err := os.Open(filePath) if err != nil { _ = c.Error(err) return } c.Writer.Header().Set("Content-type", "application/octet-stream") - c.Writer.Header().Set("Content-Disposition", "attachment; filename=\"statements.zip\"") - err = utils.StreamZipPack(c.Writer, []string{fielPath}, true) + c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileInfo.Name())) + err = aesctr.Decrypt(f, c.Writer, secretKey[0:16], secretKey[16:]) if err != nil { - log.Error("Stream zip pack failed", zap.Error(err)) + log.Error("decrypt csv failed", zap.Error(err)) } + // delete it anyway + f.Close() + _ = os.Remove(filePath) } From de6c3c8d546306b87ded97ea291de121ee3416b5 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Thu, 29 Oct 2020 13:18:39 +0800 Subject: [PATCH 11/20] use third-party reflection lib --- go.mod | 2 ++ go.sum | 4 ++++ pkg/apiserver/statement/statement.go | 15 ++++++++------- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index c9704ff7e9..84184fdcbc 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/jinzhu/gorm v1.9.12 github.com/joho/godotenv v1.3.0 github.com/joomcode/errorx v1.0.1 + github.com/oleiade/reflections v1.0.0 // indirect github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c @@ -43,4 +44,5 @@ require ( golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 // indirect google.golang.org/grpc v1.25.1 + gopkg.in/oleiade/reflections.v1 v1.0.0 ) diff --git a/go.sum b/go.sum index 7b6b60877d..36b1c7ab53 100644 --- a/go.sum +++ b/go.sum @@ -228,6 +228,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= +github.com/oleiade/reflections v1.0.0 h1:0ir4pc6v8/PJ0yw5AEtMddfXpWBXg9cnG7SgSoJuCgY= +github.com/oleiade/reflections v1.0.0/go.mod h1:RbATFBbKYkVdqmSFtx13Bb/tVhR0lgOBXunWTZKeL4w= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= @@ -505,6 +507,8 @@ gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvR gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/oleiade/reflections.v1 v1.0.0 h1:nV9NFaFd5bXKjilVvPvA+/V/tNQk1pOEEc9gGWDkj+s= +gopkg.in/oleiade/reflections.v1 v1.0.0/go.mod h1:SpA8pv+LUnF0FbB2hyRxc8XSng78D6iLBZ11PDb8Z5g= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index 445538787f..fa371ba146 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -23,7 +23,6 @@ import ( "net/http" "os" "reflect" - "strconv" "strings" "time" @@ -34,6 +33,9 @@ import ( "go.uber.org/zap" aesctr "github.com/Xeoncross/go-aesctr-with-hmac" + + "gopkg.in/oleiade/reflections.v1" + "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/user" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/utils" "github.com/pingcap-incubator/tidb-dashboard/pkg/tidb" @@ -279,16 +281,15 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { csvData := [][]string{fields} for _, overview := range overviews { row := []string{} - v := reflect.ValueOf(overview) for _, field := range fields { filedName := fieldsMap[field] - s := v.FieldByName(filedName) + s, _ := reflections.GetField(overview, filedName) var val string - switch s.Interface().(type) { + switch s.(type) { case int: - val = strconv.FormatInt(s.Int(), 10) + val = fmt.Sprintf("%d", s) default: - val = s.String() + val = fmt.Sprintf("%s", s) } row = append(row, val) } @@ -333,7 +334,7 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { // @Router /statements/download [get] // @Summary Download statements -// @Produce application/zip +// @Produce application/octet-stream // @Param token query string true "download token" // @Failure 400 {object} utils.APIError // @Failure 401 {object} utils.APIError "Unauthorized failure" From 0750c087f2e551592de982ff58a26261e8bdd3b9 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Thu, 29 Oct 2020 15:03:47 +0800 Subject: [PATCH 12/20] refine content-type --- pkg/apiserver/statement/statement.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index fa371ba146..fa7c2ae5f9 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -334,7 +334,7 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { // @Router /statements/download [get] // @Summary Download statements -// @Produce application/octet-stream +// @Produce text/csv // @Param token query string true "download token" // @Failure 400 {object} utils.APIError // @Failure 401 {object} utils.APIError "Unauthorized failure" @@ -368,7 +368,7 @@ func (s *Service) downloadHandler(c *gin.Context) { return } - c.Writer.Header().Set("Content-type", "application/octet-stream") + c.Writer.Header().Set("Content-type", "text/csv") c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileInfo.Name())) err = aesctr.Decrypt(f, c.Writer, secretKey[0:16], secretKey[16:]) if err != nil { From 51d27151bdd11a6870bee4a5edcce203574e8de7 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Thu, 29 Oct 2020 16:45:17 +0800 Subject: [PATCH 13/20] refine download progress in ui --- ui/lib/apps/Statement/pages/List/index.tsx | 58 ++++++++++++++++++---- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index b234547d7c..fde85d6dfe 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -1,10 +1,22 @@ import React, { useState } from 'react' -import { Space, Tooltip, Drawer, Button, Checkbox, Result, Input } from 'antd' +import { + Space, + Tooltip, + Drawer, + Button, + Checkbox, + Result, + Input, + Dropdown, + Menu, + message, + Modal, +} from 'antd' import { useLocalStorageState } from '@umijs/hooks' import { - SettingOutlined, ReloadOutlined, LoadingOutlined, + MenuOutlined, } from '@ant-design/icons' import { ScrollablePane } from 'office-ui-fabric-react/lib/ScrollablePane' import { useTranslation } from 'react-i18next' @@ -54,13 +66,42 @@ export default function StatementsOverview() { } = controller async function exportCSV() { + message.info('exporting...', 1) const token = await getDownloadToken() if (token) { const url = `${client.getBasePath()}/statements/download?token=${token}` - window.open(url) + Modal.success({ + title: 'Export finished', + okText: 'Download', + onOk: () => { + window.open(url) + return Promise.resolve('') + }, + }) + } + } + + function menuItemClick({ key }) { + switch (key) { + case 'settings': + setShowSettings(true) + break + case 'export': + exportCSV() + break } } + const dropdownMenu = ( + + {t('statement.settings.title')} + + {/* {t('statement.pages.overview.toolbar.export')} */} + {downloading ? 'Exporting' : 'Export'} + + + ) + return (
@@ -118,9 +159,6 @@ export default function StatementsOverview() { setQueryOptions({ ...queryOptions, searchText }) } /> - @@ -142,9 +180,6 @@ export default function StatementsOverview() { } /> )} - - setShowSettings(true)} /> - {loadingStatements ? ( @@ -152,6 +187,11 @@ export default function StatementsOverview() { )} + +
+ +
+ From 858301f5e1a1cea2527c940eb5065807b971f35d Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Thu, 29 Oct 2020 17:22:48 +0800 Subject: [PATCH 14/20] refine download csv experience in ui --- ui/lib/apps/Statement/pages/List/index.tsx | 25 +++++++++++----------- ui/lib/apps/Statement/translations/en.yaml | 3 ++- ui/lib/apps/Statement/translations/zh.yaml | 1 + 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index fde85d6dfe..cf883223f0 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -10,7 +10,6 @@ import { Dropdown, Menu, message, - Modal, } from 'antd' import { useLocalStorageState } from '@umijs/hooks' import { @@ -36,6 +35,12 @@ const { Search } = Input const STMT_VISIBLE_COLUMN_KEYS = 'statement.visible_column_keys' const STMT_SHOW_FULL_SQL = 'statement.show_full_sql' +function downloadByLink(url: string) { + const downloadLink = document.createElement('a') + downloadLink.href = url + downloadLink.click() +} + export default function StatementsOverview() { const { t } = useTranslation() @@ -66,18 +71,13 @@ export default function StatementsOverview() { } = controller async function exportCSV() { - message.info('exporting...', 1) + message.info(t('statement.pages.overview.toolbar.exporting') + '...', 2) const token = await getDownloadToken() if (token) { const url = `${client.getBasePath()}/statements/download?token=${token}` - Modal.success({ - title: 'Export finished', - okText: 'Download', - onOk: () => { - window.open(url) - return Promise.resolve('') - }, - }) + // `window.open(url)` would cause browser popup interception if getDownloadToken takes long time + // window.open(url) + downloadByLink(url) } } @@ -96,8 +96,9 @@ export default function StatementsOverview() { {t('statement.settings.title')} - {/* {t('statement.pages.overview.toolbar.export')} */} - {downloading ? 'Exporting' : 'Export'} + {downloading + ? t('statement.pages.overview.toolbar.exporting') + : t('statement.pages.overview.toolbar.export')} ) diff --git a/ui/lib/apps/Statement/translations/en.yaml b/ui/lib/apps/Statement/translations/en.yaml index 3f9f6b3179..030c4d1e21 100644 --- a/ui/lib/apps/Statement/translations/en.yaml +++ b/ui/lib/apps/Statement/translations/en.yaml @@ -37,7 +37,8 @@ statement: recent: Recent usual_time_ranges: Common custom_time_ranges: Custom - export: export + export: Export + exporting: Exporting settings: title: Settings disabled_result: diff --git a/ui/lib/apps/Statement/translations/zh.yaml b/ui/lib/apps/Statement/translations/zh.yaml index 61101a1a01..e2b5045a13 100644 --- a/ui/lib/apps/Statement/translations/zh.yaml +++ b/ui/lib/apps/Statement/translations/zh.yaml @@ -38,6 +38,7 @@ statement: usual_time_ranges: 常用时间范围 custom_time_ranges: 自定义时间范围 export: 导出 + exporting: 正在导出 settings: title: 设置 disabled_result: From c29ea221f206fc90dea3cc47616ffda01a6991b6 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Thu, 29 Oct 2020 17:25:51 +0800 Subject: [PATCH 15/20] rename getDownloadToken to genDownloadToken --- ui/lib/apps/Statement/pages/List/index.tsx | 6 +++--- ui/lib/apps/Statement/utils/useStatementTableController.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index cf883223f0..8787646f42 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -66,16 +66,16 @@ export default function StatementsOverview() { loadingStatements, tableColumns, - getDownloadToken, + genDownloadToken, downloading, } = controller async function exportCSV() { message.info(t('statement.pages.overview.toolbar.exporting') + '...', 2) - const token = await getDownloadToken() + const token = await genDownloadToken() if (token) { const url = `${client.getBasePath()}/statements/download?token=${token}` - // `window.open(url)` would cause browser popup interception if getDownloadToken takes long time + // `window.open(url)` would cause browser popup interception if genDownloadToken takes long time // window.open(url) downloadByLink(url) } diff --git a/ui/lib/apps/Statement/utils/useStatementTableController.ts b/ui/lib/apps/Statement/utils/useStatementTableController.ts index e47fb9827f..0d854dc4fb 100644 --- a/ui/lib/apps/Statement/utils/useStatementTableController.ts +++ b/ui/lib/apps/Statement/utils/useStatementTableController.ts @@ -68,7 +68,7 @@ export interface IStatementTableController { tableColumns: IColumn[] visibleColumnKeys: IColumnKeys - getDownloadToken: () => Promise + genDownloadToken: () => Promise downloading: boolean } @@ -222,7 +222,7 @@ export default function useStatementTableController( }, [queryOptions, allTimeRanges, validTimeRange, selectedFields]) const [downloading, setDownloading] = useState(false) - async function getDownloadToken() { + async function genDownloadToken() { let token = '' try { setDownloading(true) @@ -261,7 +261,7 @@ export default function useStatementTableController( tableColumns, visibleColumnKeys, - getDownloadToken, + genDownloadToken, downloading, } } From 7d803748d25224713dd3d7084fa27f1314b38be1 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Thu, 29 Oct 2020 22:29:36 +0800 Subject: [PATCH 16/20] refine ui --- ui/lib/apps/Statement/pages/List/index.tsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index 8787646f42..36d804fbde 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -16,6 +16,8 @@ import { ReloadOutlined, LoadingOutlined, MenuOutlined, + SettingOutlined, + ExportOutlined, } from '@ant-design/icons' import { ScrollablePane } from 'office-ui-fabric-react/lib/ScrollablePane' import { useTranslation } from 'react-i18next' @@ -71,13 +73,17 @@ export default function StatementsOverview() { } = controller async function exportCSV() { - message.info(t('statement.pages.overview.toolbar.exporting') + '...', 2) + const hide = message.loading( + t('statement.pages.overview.toolbar.exporting') + '...', + 0 + ) const token = await genDownloadToken() if (token) { const url = `${client.getBasePath()}/statements/download?token=${token}` // `window.open(url)` would cause browser popup interception if genDownloadToken takes long time // window.open(url) downloadByLink(url) + hide() } } @@ -94,8 +100,10 @@ export default function StatementsOverview() { const dropdownMenu = ( - {t('statement.settings.title')} - + }> + {t('statement.settings.title')} + + }> {downloading ? t('statement.pages.overview.toolbar.exporting') : t('statement.pages.overview.toolbar.export')} From 47109f67771967fab4dcb01d4091474c1a26dec5 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Fri, 30 Oct 2020 10:59:35 +0800 Subject: [PATCH 17/20] refine --- ui/lib/apps/SearchLogs/components/SearchProgress.tsx | 2 +- ui/lib/apps/Statement/pages/List/index.tsx | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ui/lib/apps/SearchLogs/components/SearchProgress.tsx b/ui/lib/apps/SearchLogs/components/SearchProgress.tsx index 40172f122b..4626eca1d4 100644 --- a/ui/lib/apps/SearchLogs/components/SearchProgress.tsx +++ b/ui/lib/apps/SearchLogs/components/SearchProgress.tsx @@ -154,7 +154,7 @@ export default function SearchProgress({ return } const url = `${client.getBasePath()}/logs/download?token=${token}` - window.open(url) + window.location.href = url } async function handleCancel() { diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index 36d804fbde..b9831b58ee 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -37,12 +37,6 @@ const { Search } = Input const STMT_VISIBLE_COLUMN_KEYS = 'statement.visible_column_keys' const STMT_SHOW_FULL_SQL = 'statement.show_full_sql' -function downloadByLink(url: string) { - const downloadLink = document.createElement('a') - downloadLink.href = url - downloadLink.click() -} - export default function StatementsOverview() { const { t } = useTranslation() @@ -82,7 +76,7 @@ export default function StatementsOverview() { const url = `${client.getBasePath()}/statements/download?token=${token}` // `window.open(url)` would cause browser popup interception if genDownloadToken takes long time // window.open(url) - downloadByLink(url) + window.location.href = url hide() } } From b6b076b8103555cfb6c0479687acf804b85f6251 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Fri, 30 Oct 2020 16:48:36 +0800 Subject: [PATCH 18/20] export all fields instead of selected fields --- pkg/apiserver/statement/models.go | 8 +++--- pkg/apiserver/statement/queries.go | 11 +++++--- pkg/apiserver/statement/statement.go | 27 +++++++++++-------- ui/lib/apps/Statement/pages/List/index.tsx | 17 +++++++----- .../utils/useStatementTableController.ts | 4 +-- 5 files changed, 40 insertions(+), 27 deletions(-) diff --git a/pkg/apiserver/statement/models.go b/pkg/apiserver/statement/models.go index 3352700056..e4120dcab9 100644 --- a/pkg/apiserver/statement/models.go +++ b/pkg/apiserver/statement/models.go @@ -34,7 +34,8 @@ type TimeRange struct { } type Model struct { - AggPlanCount int `json:"plan_count" agg:"COUNT(DISTINCT plan_digest)"` + AggDigestText string `json:"digest_text" agg:"ANY_VALUE(digest_text)"` + AggDigest string `json:"digest" agg:"ANY_VALUE(digest)"` AggExecCount int `json:"exec_count" agg:"SUM(exec_count)"` AggSumErrors int `json:"sum_errors" agg:"SUM(sum_errors)"` AggSumWarnings int `json:"sum_warnings" agg:"SUM(sum_warnings)"` @@ -95,10 +96,9 @@ type Model struct { AggSchemaName string `json:"schema_name" agg:"ANY_VALUE(schema_name)"` AggTableNames string `json:"table_names" agg:"ANY_VALUE(table_names)"` AggIndexNames string `json:"index_names" agg:"ANY_VALUE(index_names)"` - AggDigestText string `json:"digest_text" agg:"ANY_VALUE(digest_text)"` - AggDigest string `json:"digest" agg:"ANY_VALUE(digest)"` - AggPlanDigest string `json:"plan_digest" agg:"ANY_VALUE(plan_digest)"` + AggPlanCount int `json:"plan_count" agg:"COUNT(DISTINCT plan_digest)"` AggPlan string `json:"plan" agg:"ANY_VALUE(plan)"` + AggPlanDigest string `json:"plan_digest" agg:"ANY_VALUE(plan_digest)"` // Computed fields RelatedSchemas string `json:"related_schemas"` } diff --git a/pkg/apiserver/statement/queries.go b/pkg/apiserver/statement/queries.go index d8c07b8eb6..fc07ef7339 100644 --- a/pkg/apiserver/statement/queries.go +++ b/pkg/apiserver/statement/queries.go @@ -130,15 +130,20 @@ func QueryStmtTypes(db *gorm.DB) (result []string, err error) { // schemas: ["tpcc", "test"] // stmtTypes: ["select", "update"] // fields: ["digest_text", "sum_latency"] -func QueryStatementsOverview( +func QueryStatements( db *gorm.DB, beginTime, endTime int, schemas, stmtTypes []string, text string, fields []string, ) (result []Model, err error) { - fields = funk.UniqString(append(fields, "schema_name", "digest", "sum_latency")) // "schema_name", "digest" for group, "sum_latency" for order - aggrFields := getAggrFields(fields...) + var aggrFields []string + if len(fields) == 1 && fields[0] == "*" { + aggrFields = getAllAggrFields() + } else { + fields = funk.UniqString(append(fields, "schema_name", "digest", "sum_latency")) // "schema_name", "digest" for group, "sum_latency" for order + aggrFields = getAggrFields(fields...) + } query := db. Select(strings.Join(aggrFields, ", ")). diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index fa7c2ae5f9..8def73b1a7 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -66,9 +66,9 @@ func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) { endpoint.POST("/config", s.modifyConfigHandler) endpoint.GET("/time_ranges", s.timeRangesHandler) endpoint.GET("/stmt_types", s.stmtTypesHandler) - endpoint.GET("/overviews", s.overviewsHandler) - endpoint.GET("/plans", s.getPlansHandler) - endpoint.GET("/plan/detail", s.getPlanDetailHandler) + endpoint.GET("/list", s.listHandler) + endpoint.GET("/plans", s.plansHandler) + endpoint.GET("/plan/detail", s.planDetailHandler) endpoint.POST("/download/token", s.downloadTokenHandler) } @@ -150,13 +150,13 @@ type GetStatementsRequest struct { Fields string `json:"fields" form:"fields"` } -// @Summary Get a list of statement overviews +// @Summary Get a list of statements // @Param q query GetStatementsRequest true "Query" // @Success 200 {array} Model -// @Router /statements/overviews [get] +// @Router /statements/list [get] // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) overviewsHandler(c *gin.Context) { +func (s *Service) listHandler(c *gin.Context) { var req GetStatementsRequest if err := c.ShouldBindQuery(&req); err != nil { utils.MakeInvalidRequestErrorFromError(c, err) @@ -167,7 +167,7 @@ func (s *Service) overviewsHandler(c *gin.Context) { if strings.TrimSpace(req.Fields) != "" { fields = strings.Split(req.Fields, ",") } - overviews, err := QueryStatementsOverview( + overviews, err := QueryStatements( db, req.BeginTime, req.EndTime, req.Schemas, @@ -194,7 +194,7 @@ type GetPlansRequest struct { // @Router /statements/plans [get] // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) getPlansHandler(c *gin.Context) { +func (s *Service) plansHandler(c *gin.Context) { var req GetPlansRequest if err := c.ShouldBindQuery(&req); err != nil { utils.MakeInvalidRequestErrorFromError(c, err) @@ -220,7 +220,7 @@ type GetPlanDetailRequest struct { // @Router /statements/plan/detail [get] // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) getPlanDetailHandler(c *gin.Context) { +func (s *Service) planDetailHandler(c *gin.Context) { var req GetPlanDetailRequest if err := c.ShouldBindQuery(&req); err != nil { utils.MakeInvalidRequestErrorFromError(c, err) @@ -253,7 +253,7 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { if strings.TrimSpace(req.Fields) != "" { fields = strings.Split(req.Fields, ",") } - overviews, err := QueryStatementsOverview( + overviews, err := QueryStatements( db, req.BeginTime, req.EndTime, req.Schemas, @@ -273,9 +273,14 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { fieldsMap := make(map[string]string) t := reflect.TypeOf(overviews[0]) fieldsNum := t.NumField() + allFields := make([]string, fieldsNum) for i := 0; i < fieldsNum; i++ { field := t.Field(i) - fieldsMap[strings.ToLower(field.Tag.Get("json"))] = field.Name + allFields[i] = strings.ToLower(field.Tag.Get("json")) + fieldsMap[allFields[i]] = field.Name + } + if len(fields) == 1 && fields[0] == "*" { + fields = allFields } csvData := [][]string{fields} diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index b9831b58ee..79ca5d9cb7 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -22,6 +22,7 @@ import { import { ScrollablePane } from 'office-ui-fabric-react/lib/ScrollablePane' import { useTranslation } from 'react-i18next' +import client from '@lib/client' import { Card, ColumnsSelector, Toolbar, MultiSelect } from '@lib/components' import { StatementsTable } from '../../components' @@ -30,7 +31,6 @@ import TimeRangeSelector from './TimeRangeSelector' import useStatementTableController, { DEF_STMT_COLUMN_KEYS, } from '../../utils/useStatementTableController' -import client from '@lib/client' const { Search } = Input @@ -71,12 +71,15 @@ export default function StatementsOverview() { t('statement.pages.overview.toolbar.exporting') + '...', 0 ) - const token = await genDownloadToken() - if (token) { - const url = `${client.getBasePath()}/statements/download?token=${token}` - // `window.open(url)` would cause browser popup interception if genDownloadToken takes long time - // window.open(url) - window.location.href = url + try { + const token = await genDownloadToken() + if (token) { + const url = `${client.getBasePath()}/statements/download?token=${token}` + // `window.open(url)` would cause browser popup interception if genDownloadToken takes long time + // window.open(url) + window.location.href = url + } + } finally { hide() } } diff --git a/ui/lib/apps/Statement/utils/useStatementTableController.ts b/ui/lib/apps/Statement/utils/useStatementTableController.ts index 0d854dc4fb..5abcf2499b 100644 --- a/ui/lib/apps/Statement/utils/useStatementTableController.ts +++ b/ui/lib/apps/Statement/utils/useStatementTableController.ts @@ -199,7 +199,7 @@ export default function useStatementTableController( try { const res = await client .getInstance() - .statementsOverviewsGet( + .statementsListGet( validTimeRange.begin_time!, validTimeRange.end_time!, selectedFields, @@ -229,7 +229,7 @@ export default function useStatementTableController( const res = await client.getInstance().statementsDownloadTokenPost({ begin_time: validTimeRange.begin_time, end_time: validTimeRange.end_time, - fields: selectedFields, + fields: '*', schemas: queryOptions.schemas, stmt_types: queryOptions.stmtTypes, text: queryOptions.searchText, From e2561ce635b1df35b938c0c5fedfe00ffc46d616 Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Fri, 30 Oct 2020 17:17:50 +0800 Subject: [PATCH 19/20] format time type column --- pkg/apiserver/statement/statement.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index 8def73b1a7..d90d952190 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -284,6 +284,7 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { } csvData := [][]string{fields} + timeLayout := "01-02 15:04:05" for _, overview := range overviews { row := []string{} for _, field := range fields { @@ -292,7 +293,11 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { var val string switch s.(type) { case int: - val = fmt.Sprintf("%d", s) + if field == "first_seen" || field == "last_seen" { + val = time.Unix(int64(s.(int)), 0).Format(timeLayout) + } else { + val = fmt.Sprintf("%d", s) + } default: val = fmt.Sprintf("%s", s) } @@ -302,7 +307,7 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { } // generate temp file that persist encrypted data - timeLayout := "01021504" + timeLayout = "01021504" beginTime := time.Unix(int64(req.BeginTime), 0).Format(timeLayout) endTime := time.Unix(int64(req.EndTime), 0).Format(timeLayout) csvFile, err := ioutil.TempFile("", fmt.Sprintf("statements_%s_%s_*.csv", beginTime, endTime)) From 097b3685110a670a627457bfbf9f15a41dfbd3ef Mon Sep 17 00:00:00 2001 From: baurine <2008.hbl@gmail.com> Date: Fri, 30 Oct 2020 22:11:09 +0800 Subject: [PATCH 20/20] fix compile --- pkg/apiserver/statement/statement.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index d90d952190..951828a160 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -291,15 +291,15 @@ func (s *Service) downloadTokenHandler(c *gin.Context) { filedName := fieldsMap[field] s, _ := reflections.GetField(overview, filedName) var val string - switch s.(type) { + switch t := s.(type) { case int: if field == "first_seen" || field == "last_seen" { - val = time.Unix(int64(s.(int)), 0).Format(timeLayout) + val = time.Unix(int64(t), 0).Format(timeLayout) } else { - val = fmt.Sprintf("%d", s) + val = fmt.Sprintf("%d", t) } default: - val = fmt.Sprintf("%s", s) + val = fmt.Sprintf("%s", t) } row = append(row, val) }