diff --git a/go.mod b/go.mod index 947eb421dd..84184fdcbc 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd + github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/appleboy/gin-jwt/v2 v2.6.3 github.com/cenkalti/backoff/v4 v4.0.2 @@ -19,6 +20,7 @@ require ( github.com/jinzhu/gorm v1.9.12 github.com/joho/godotenv v1.3.0 github.com/joomcode/errorx v1.0.1 + github.com/oleiade/reflections v1.0.0 // indirect github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c @@ -42,4 +44,5 @@ require ( golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 // indirect google.golang.org/grpc v1.25.1 + gopkg.in/oleiade/reflections.v1 v1.0.0 ) diff --git a/go.sum b/go.sum index 5cb1fd340d..36b1c7ab53 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd h1:59Whn6shj5MTVjTf2OX6+7iMcmY6h5CK0kTWwRaplL4= github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0= +github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= +github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -226,6 +228,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= +github.com/oleiade/reflections v1.0.0 h1:0ir4pc6v8/PJ0yw5AEtMddfXpWBXg9cnG7SgSoJuCgY= +github.com/oleiade/reflections v1.0.0/go.mod h1:RbATFBbKYkVdqmSFtx13Bb/tVhR0lgOBXunWTZKeL4w= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= @@ -503,6 +507,8 @@ gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvR gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/oleiade/reflections.v1 v1.0.0 h1:nV9NFaFd5bXKjilVvPvA+/V/tNQk1pOEEc9gGWDkj+s= +gopkg.in/oleiade/reflections.v1 v1.0.0/go.mod h1:SpA8pv+LUnF0FbB2hyRxc8XSng78D6iLBZ11PDb8Z5g= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/apiserver/statement/models.go b/pkg/apiserver/statement/models.go index 3352700056..e4120dcab9 100644 --- a/pkg/apiserver/statement/models.go +++ b/pkg/apiserver/statement/models.go @@ -34,7 +34,8 @@ type TimeRange struct { } type Model struct { - AggPlanCount int `json:"plan_count" agg:"COUNT(DISTINCT plan_digest)"` + AggDigestText string `json:"digest_text" agg:"ANY_VALUE(digest_text)"` + AggDigest string `json:"digest" agg:"ANY_VALUE(digest)"` AggExecCount int `json:"exec_count" agg:"SUM(exec_count)"` AggSumErrors int `json:"sum_errors" agg:"SUM(sum_errors)"` AggSumWarnings int `json:"sum_warnings" agg:"SUM(sum_warnings)"` @@ -95,10 +96,9 @@ type Model struct { AggSchemaName string `json:"schema_name" agg:"ANY_VALUE(schema_name)"` AggTableNames string `json:"table_names" agg:"ANY_VALUE(table_names)"` AggIndexNames string `json:"index_names" agg:"ANY_VALUE(index_names)"` - AggDigestText string `json:"digest_text" agg:"ANY_VALUE(digest_text)"` - AggDigest string `json:"digest" agg:"ANY_VALUE(digest)"` - AggPlanDigest string `json:"plan_digest" agg:"ANY_VALUE(plan_digest)"` + AggPlanCount int `json:"plan_count" agg:"COUNT(DISTINCT plan_digest)"` AggPlan string `json:"plan" agg:"ANY_VALUE(plan)"` + AggPlanDigest string `json:"plan_digest" agg:"ANY_VALUE(plan_digest)"` // Computed fields RelatedSchemas string `json:"related_schemas"` } diff --git a/pkg/apiserver/statement/queries.go b/pkg/apiserver/statement/queries.go index d8c07b8eb6..fc07ef7339 100644 --- a/pkg/apiserver/statement/queries.go +++ b/pkg/apiserver/statement/queries.go @@ -130,15 +130,20 @@ func QueryStmtTypes(db *gorm.DB) (result []string, err error) { // schemas: ["tpcc", "test"] // stmtTypes: ["select", "update"] // fields: ["digest_text", "sum_latency"] -func QueryStatementsOverview( +func QueryStatements( db *gorm.DB, beginTime, endTime int, schemas, stmtTypes []string, text string, fields []string, ) (result []Model, err error) { - fields = funk.UniqString(append(fields, "schema_name", "digest", "sum_latency")) // "schema_name", "digest" for group, "sum_latency" for order - aggrFields := getAggrFields(fields...) + var aggrFields []string + if len(fields) == 1 && fields[0] == "*" { + aggrFields = getAllAggrFields() + } else { + fields = funk.UniqString(append(fields, "schema_name", "digest", "sum_latency")) // "schema_name", "digest" for group, "sum_latency" for order + aggrFields = getAggrFields(fields...) + } query := db. Select(strings.Join(aggrFields, ", ")). diff --git a/pkg/apiserver/statement/statement.go b/pkg/apiserver/statement/statement.go index 2290dff902..951828a160 100644 --- a/pkg/apiserver/statement/statement.go +++ b/pkg/apiserver/statement/statement.go @@ -14,11 +14,27 @@ package statement import ( + "encoding/base64" + "encoding/csv" + "errors" + "fmt" + "io" + "io/ioutil" "net/http" + "os" + "reflect" "strings" + "time" "github.com/gin-gonic/gin" + "github.com/gtank/cryptopasta" + "github.com/pingcap/log" "go.uber.org/fx" + "go.uber.org/zap" + + aesctr "github.com/Xeoncross/go-aesctr-with-hmac" + + "gopkg.in/oleiade/reflections.v1" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/user" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/utils" @@ -40,15 +56,23 @@ func NewService(p ServiceParams) *Service { func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) { endpoint := r.Group("/statements") - endpoint.Use(auth.MWAuthRequired()) - endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient)) - endpoint.GET("/config", s.configHandler) - endpoint.POST("/config", s.modifyConfigHandler) - endpoint.GET("/time_ranges", s.timeRangesHandler) - endpoint.GET("/stmt_types", s.stmtTypesHandler) - endpoint.GET("/overviews", s.overviewsHandler) - endpoint.GET("/plans", s.getPlansHandler) - endpoint.GET("/plan/detail", s.getPlanDetailHandler) + { + endpoint.GET("/download", s.downloadHandler) + + endpoint.Use(auth.MWAuthRequired()) + endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient)) + { + endpoint.GET("/config", s.configHandler) + endpoint.POST("/config", s.modifyConfigHandler) + endpoint.GET("/time_ranges", s.timeRangesHandler) + endpoint.GET("/stmt_types", s.stmtTypesHandler) + endpoint.GET("/list", s.listHandler) + endpoint.GET("/plans", s.plansHandler) + endpoint.GET("/plan/detail", s.planDetailHandler) + + endpoint.POST("/download/token", s.downloadTokenHandler) + } + } } // @Summary Get statement configurations @@ -126,13 +150,13 @@ type GetStatementsRequest struct { Fields string `json:"fields" form:"fields"` } -// @Summary Get a list of statement overviews +// @Summary Get a list of statements // @Param q query GetStatementsRequest true "Query" // @Success 200 {array} Model -// @Router /statements/overviews [get] +// @Router /statements/list [get] // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) overviewsHandler(c *gin.Context) { +func (s *Service) listHandler(c *gin.Context) { var req GetStatementsRequest if err := c.ShouldBindQuery(&req); err != nil { utils.MakeInvalidRequestErrorFromError(c, err) @@ -143,7 +167,7 @@ func (s *Service) overviewsHandler(c *gin.Context) { if strings.TrimSpace(req.Fields) != "" { fields = strings.Split(req.Fields, ",") } - overviews, err := QueryStatementsOverview( + overviews, err := QueryStatements( db, req.BeginTime, req.EndTime, req.Schemas, @@ -170,7 +194,7 @@ type GetPlansRequest struct { // @Router /statements/plans [get] // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) getPlansHandler(c *gin.Context) { +func (s *Service) plansHandler(c *gin.Context) { var req GetPlansRequest if err := c.ShouldBindQuery(&req); err != nil { utils.MakeInvalidRequestErrorFromError(c, err) @@ -196,7 +220,7 @@ type GetPlanDetailRequest struct { // @Router /statements/plan/detail [get] // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) getPlanDetailHandler(c *gin.Context) { +func (s *Service) planDetailHandler(c *gin.Context) { var req GetPlanDetailRequest if err := c.ShouldBindQuery(&req); err != nil { utils.MakeInvalidRequestErrorFromError(c, err) @@ -210,3 +234,157 @@ func (s *Service) getPlanDetailHandler(c *gin.Context) { } c.JSON(http.StatusOK, result) } + +// @Router /statements/download/token [post] +// @Summary Generate a download token for exported statements +// @Produce plain +// @Param request body GetStatementsRequest true "Request body" +// @Success 200 {string} string "xxx" +// @Security JwtAuth +// @Failure 401 {object} utils.APIError "Unauthorized failure" +func (s *Service) downloadTokenHandler(c *gin.Context) { + var req GetStatementsRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.MakeInvalidRequestErrorFromError(c, err) + return + } + db := utils.GetTiDBConnection(c) + fields := []string{} + if strings.TrimSpace(req.Fields) != "" { + fields = strings.Split(req.Fields, ",") + } + overviews, err := QueryStatements( + db, + req.BeginTime, req.EndTime, + req.Schemas, + req.StmtTypes, + req.Text, + fields) + if err != nil { + _ = c.Error(err) + return + } + if len(overviews) == 0 { + utils.MakeInvalidRequestErrorFromError(c, errors.New("no data to export")) + return + } + + // convert data + fieldsMap := make(map[string]string) + t := reflect.TypeOf(overviews[0]) + fieldsNum := t.NumField() + allFields := make([]string, fieldsNum) + for i := 0; i < fieldsNum; i++ { + field := t.Field(i) + allFields[i] = strings.ToLower(field.Tag.Get("json")) + fieldsMap[allFields[i]] = field.Name + } + if len(fields) == 1 && fields[0] == "*" { + fields = allFields + } + + csvData := [][]string{fields} + timeLayout := "01-02 15:04:05" + for _, overview := range overviews { + row := []string{} + for _, field := range fields { + filedName := fieldsMap[field] + s, _ := reflections.GetField(overview, filedName) + var val string + switch t := s.(type) { + case int: + if field == "first_seen" || field == "last_seen" { + val = time.Unix(int64(t), 0).Format(timeLayout) + } else { + val = fmt.Sprintf("%d", t) + } + default: + val = fmt.Sprintf("%s", t) + } + row = append(row, val) + } + csvData = append(csvData, row) + } + + // generate temp file that persist encrypted data + timeLayout = "01021504" + beginTime := time.Unix(int64(req.BeginTime), 0).Format(timeLayout) + endTime := time.Unix(int64(req.EndTime), 0).Format(timeLayout) + csvFile, err := ioutil.TempFile("", fmt.Sprintf("statements_%s_%s_*.csv", beginTime, endTime)) + if err != nil { + _ = c.Error(err) + return + } + defer csvFile.Close() + + // generate encryption key + secretKey := *cryptopasta.NewEncryptionKey() + + pr, pw := io.Pipe() + go func() { + csvwriter := csv.NewWriter(pw) + _ = csvwriter.WriteAll(csvData) + pw.Close() + }() + err = aesctr.Encrypt(pr, csvFile, secretKey[0:16], secretKey[16:]) + if err != nil { + _ = c.Error(err) + return + } + + // generate token by filepath and secretKey + secretKeyStr := base64.StdEncoding.EncodeToString(secretKey[:]) + token, err := utils.NewJWTString("statements/download", secretKeyStr+" "+csvFile.Name()) + if err != nil { + _ = c.Error(err) + return + } + c.String(http.StatusOK, token) +} + +// @Router /statements/download [get] +// @Summary Download statements +// @Produce text/csv +// @Param token query string true "download token" +// @Failure 400 {object} utils.APIError +// @Failure 401 {object} utils.APIError "Unauthorized failure" +func (s *Service) downloadHandler(c *gin.Context) { + token := c.Query("token") + tokenPlain, err := utils.ParseJWTString("statements/download", token) + if err != nil { + utils.MakeInvalidRequestErrorFromError(c, err) + return + } + arr := strings.Fields(tokenPlain) + if len(arr) != 2 { + utils.MakeInvalidRequestErrorFromError(c, errors.New("invalid token")) + return + } + secretKey, err := base64.StdEncoding.DecodeString(arr[0]) + if err != nil { + utils.MakeInvalidRequestErrorFromError(c, err) + return + } + + filePath := arr[1] + fileInfo, err := os.Stat(filePath) + if err != nil { + _ = c.Error(err) + return + } + f, err := os.Open(filePath) + if err != nil { + _ = c.Error(err) + return + } + + c.Writer.Header().Set("Content-type", "text/csv") + c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileInfo.Name())) + err = aesctr.Decrypt(f, c.Writer, secretKey[0:16], secretKey[16:]) + if err != nil { + log.Error("decrypt csv failed", zap.Error(err)) + } + // delete it anyway + f.Close() + _ = os.Remove(filePath) +} diff --git a/ui/lib/apps/SearchLogs/components/SearchProgress.tsx b/ui/lib/apps/SearchLogs/components/SearchProgress.tsx index 40172f122b..4626eca1d4 100644 --- a/ui/lib/apps/SearchLogs/components/SearchProgress.tsx +++ b/ui/lib/apps/SearchLogs/components/SearchProgress.tsx @@ -154,7 +154,7 @@ export default function SearchProgress({ return } const url = `${client.getBasePath()}/logs/download?token=${token}` - window.open(url) + window.location.href = url } async function handleCancel() { diff --git a/ui/lib/apps/Statement/pages/List/index.tsx b/ui/lib/apps/Statement/pages/List/index.tsx index 7590375686..79ca5d9cb7 100644 --- a/ui/lib/apps/Statement/pages/List/index.tsx +++ b/ui/lib/apps/Statement/pages/List/index.tsx @@ -1,14 +1,28 @@ import React, { useState } from 'react' -import { Space, Tooltip, Drawer, Button, Checkbox, Result, Input } from 'antd' +import { + Space, + Tooltip, + Drawer, + Button, + Checkbox, + Result, + Input, + Dropdown, + Menu, + message, +} from 'antd' import { useLocalStorageState } from '@umijs/hooks' import { - SettingOutlined, ReloadOutlined, LoadingOutlined, + MenuOutlined, + SettingOutlined, + ExportOutlined, } from '@ant-design/icons' import { ScrollablePane } from 'office-ui-fabric-react/lib/ScrollablePane' import { useTranslation } from 'react-i18next' +import client from '@lib/client' import { Card, ColumnsSelector, Toolbar, MultiSelect } from '@lib/components' import { StatementsTable } from '../../components' @@ -47,8 +61,53 @@ export default function StatementsOverview() { allStmtTypes, loadingStatements, tableColumns, + + genDownloadToken, + downloading, } = controller + async function exportCSV() { + const hide = message.loading( + t('statement.pages.overview.toolbar.exporting') + '...', + 0 + ) + try { + const token = await genDownloadToken() + if (token) { + const url = `${client.getBasePath()}/statements/download?token=${token}` + // `window.open(url)` would cause browser popup interception if genDownloadToken takes long time + // window.open(url) + window.location.href = url + } + } finally { + hide() + } + } + + function menuItemClick({ key }) { + switch (key) { + case 'settings': + setShowSettings(true) + break + case 'export': + exportCSV() + break + } + } + + const dropdownMenu = ( + + }> + {t('statement.settings.title')} + + }> + {downloading + ? t('statement.pages.overview.toolbar.exporting') + : t('statement.pages.overview.toolbar.export')} + + + ) + return (
@@ -127,9 +186,6 @@ export default function StatementsOverview() { } /> )} - - setShowSettings(true)} /> - {loadingStatements ? ( @@ -137,6 +193,11 @@ export default function StatementsOverview() { )} + +
+ +
+ diff --git a/ui/lib/apps/Statement/translations/en.yaml b/ui/lib/apps/Statement/translations/en.yaml index b4a44d1417..030c4d1e21 100644 --- a/ui/lib/apps/Statement/translations/en.yaml +++ b/ui/lib/apps/Statement/translations/en.yaml @@ -37,6 +37,8 @@ statement: recent: Recent usual_time_ranges: Common custom_time_ranges: Custom + export: Export + exporting: Exporting settings: title: Settings disabled_result: diff --git a/ui/lib/apps/Statement/translations/zh.yaml b/ui/lib/apps/Statement/translations/zh.yaml index f15541f201..e2b5045a13 100644 --- a/ui/lib/apps/Statement/translations/zh.yaml +++ b/ui/lib/apps/Statement/translations/zh.yaml @@ -37,6 +37,8 @@ statement: recent: 最近 usual_time_ranges: 常用时间范围 custom_time_ranges: 自定义时间范围 + export: 导出 + exporting: 正在导出 settings: title: 设置 disabled_result: diff --git a/ui/lib/apps/Statement/utils/useStatementTableController.ts b/ui/lib/apps/Statement/utils/useStatementTableController.ts index ddfbf41b0d..5abcf2499b 100644 --- a/ui/lib/apps/Statement/utils/useStatementTableController.ts +++ b/ui/lib/apps/Statement/utils/useStatementTableController.ts @@ -67,6 +67,9 @@ export interface IStatementTableController { tableColumns: IColumn[] visibleColumnKeys: IColumnKeys + + genDownloadToken: () => Promise + downloading: boolean } export default function useStatementTableController( @@ -196,7 +199,7 @@ export default function useStatementTableController( try { const res = await client .getInstance() - .statementsOverviewsGet( + .statementsListGet( validTimeRange.begin_time!, validTimeRange.end_time!, selectedFields, @@ -218,6 +221,26 @@ export default function useStatementTableController( queryStatementList() }, [queryOptions, allTimeRanges, validTimeRange, selectedFields]) + const [downloading, setDownloading] = useState(false) + async function genDownloadToken() { + let token = '' + try { + setDownloading(true) + const res = await client.getInstance().statementsDownloadTokenPost({ + begin_time: validTimeRange.begin_time, + end_time: validTimeRange.end_time, + fields: '*', + schemas: queryOptions.schemas, + stmt_types: queryOptions.stmtTypes, + text: queryOptions.searchText, + }) + token = res.data + } finally { + setDownloading(false) + } + return token + } + return { queryOptions, setQueryOptions, @@ -237,5 +260,8 @@ export default function useStatementTableController( tableColumns, visibleColumnKeys, + + genDownloadToken, + downloading, } }