export statements

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd
+	github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
 	github.com/appleboy/gin-jwt/v2 v2.6.3
 	github.com/cenkalti/backoff/v4 v4.0.2
@@ -19,6 +20,7 @@ require (
 	github.com/jinzhu/gorm v1.9.12
 	github.com/joho/godotenv v1.3.0
 	github.com/joomcode/errorx v1.0.1
+	github.com/oleiade/reflections v1.0.0 // indirect
 	github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12
 	github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
 	github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c
@@ -42,4 +44,5 @@ require (
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
 	golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 // indirect
 	google.golang.org/grpc v1.25.1
+	gopkg.in/oleiade/reflections.v1 v1.0.0
 )

go.sum

@@ -12,6 +12,8 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd h1:59Whn6shj5MTVjTf2OX6+7iMcmY6h5CK0kTWwRaplL4=
 github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0=
+github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o=
+github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -226,6 +228,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=
+github.com/oleiade/reflections v1.0.0 h1:0ir4pc6v8/PJ0yw5AEtMddfXpWBXg9cnG7SgSoJuCgY=
+github.com/oleiade/reflections v1.0.0/go.mod h1:RbATFBbKYkVdqmSFtx13Bb/tVhR0lgOBXunWTZKeL4w=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
@@ -503,6 +507,8 @@ gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvR
 gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/oleiade/reflections.v1 v1.0.0 h1:nV9NFaFd5bXKjilVvPvA+/V/tNQk1pOEEc9gGWDkj+s=
+gopkg.in/oleiade/reflections.v1 v1.0.0/go.mod h1:SpA8pv+LUnF0FbB2hyRxc8XSng78D6iLBZ11PDb8Z5g=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/apiserver/statement/statement.go

@@ -14,11 +14,27 @@
 package statement
 
 import (
+	"encoding/base64"
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"reflect"
 	"strings"
+	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/gtank/cryptopasta"
+	"github.com/pingcap/log"
 	"go.uber.org/fx"
+	"go.uber.org/zap"
+
+	aesctr "github.com/Xeoncross/go-aesctr-with-hmac"
+
+	"gopkg.in/oleiade/reflections.v1"
 
 	"github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/user"
 	"github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/utils"
@@ -40,15 +56,23 @@ func NewService(p ServiceParams) *Service {
 
 func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 	endpoint := r.Group("/statements")
-	endpoint.Use(auth.MWAuthRequired())
-	endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient))
-	endpoint.GET("/config", s.configHandler)
-	endpoint.POST("/config", s.modifyConfigHandler)
-	endpoint.GET("/time_ranges", s.timeRangesHandler)
-	endpoint.GET("/stmt_types", s.stmtTypesHandler)
-	endpoint.GET("/overviews", s.overviewsHandler)
-	endpoint.GET("/plans", s.getPlansHandler)
-	endpoint.GET("/plan/detail", s.getPlanDetailHandler)
+	{
+		endpoint.GET("/download", s.downloadHandler)
+
+		endpoint.Use(auth.MWAuthRequired())
+		endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient))
+		{
+			endpoint.GET("/config", s.configHandler)
+			endpoint.POST("/config", s.modifyConfigHandler)
+			endpoint.GET("/time_ranges", s.timeRangesHandler)
+			endpoint.GET("/stmt_types", s.stmtTypesHandler)
+			endpoint.GET("/overviews", s.overviewsHandler)
+			endpoint.GET("/plans", s.getPlansHandler)
+			endpoint.GET("/plan/detail", s.getPlanDetailHandler)
+
+			endpoint.POST("/download/token", s.downloadTokenHandler)
+		}
+	}
 }
 
 // @Summary Get statement configurations
@@ -210,3 +234,147 @@ func (s *Service) getPlanDetailHandler(c *gin.Context) {
 	}
 	c.JSON(http.StatusOK, result)
 }
+
+// @Router /statements/download/token [post]
+// @Summary Generate a download token for exported statements
+// @Produce plain
+// @Param request body GetStatementsRequest true "Request body"
+// @Success 200 {string} string "xxx"
+// @Security JwtAuth
+// @Failure 401 {object} utils.APIError "Unauthorized failure"
+func (s *Service) downloadTokenHandler(c *gin.Context) {
+	var req GetStatementsRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		utils.MakeInvalidRequestErrorFromError(c, err)
+		return
+	}
+	db := utils.GetTiDBConnection(c)
+	fields := []string{}
+	if strings.TrimSpace(req.Fields) != "" {
+		fields = strings.Split(req.Fields, ",")
+	}
+	overviews, err := QueryStatementsOverview(
+		db,
+		req.BeginTime, req.EndTime,
+		req.Schemas,
+		req.StmtTypes,
+		req.Text,
+		fields)
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	if len(overviews) == 0 {
+		utils.MakeInvalidRequestErrorFromError(c, errors.New("no data to export"))
+		return
+	}
+
+	// convert data
+	fieldsMap := make(map[string]string)
+	t := reflect.TypeOf(overviews[0])
+	fieldsNum := t.NumField()
+	for i := 0; i < fieldsNum; i++ {
+		field := t.Field(i)
+		fieldsMap[strings.ToLower(field.Tag.Get("json"))] = field.Name
+	}
+
+	csvData := [][]string{fields}
+	for _, overview := range overviews {
+		row := []string{}
+		for _, field := range fields {
+			filedName := fieldsMap[field]
+			s, _ := reflections.GetField(overview, filedName)
+			var val string
+			switch s.(type) {
+			case int:
+				val = fmt.Sprintf("%d", s)
+			default:
+				val = fmt.Sprintf("%s", s)
+			}
+			row = append(row, val)
+		}
+		csvData = append(csvData, row)
+	}
+
+	// generate temp file that persist encrypted data
+	timeLayout := "01021504"
+	beginTime := time.Unix(int64(req.BeginTime), 0).Format(timeLayout)
+	endTime := time.Unix(int64(req.EndTime), 0).Format(timeLayout)
+	csvFile, err := ioutil.TempFile("", fmt.Sprintf("statements_%s_%s_*.csv", beginTime, endTime))
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	defer csvFile.Close()
+
+	// generate encryption key
+	secretKey := *cryptopasta.NewEncryptionKey()
+
+	pr, pw := io.Pipe()
+	go func() {
+		csvwriter := csv.NewWriter(pw)
+		_ = csvwriter.WriteAll(csvData)
+		pw.Close()
+	}()
+	err = aesctr.Encrypt(pr, csvFile, secretKey[0:16], secretKey[16:])
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+
+	// generate token by filepath and secretKey
+	secretKeyStr := base64.StdEncoding.EncodeToString(secretKey[:])
+	token, err := utils.NewJWTString("statements/download", secretKeyStr+" "+csvFile.Name())
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	c.String(http.StatusOK, token)
+}
+
+// @Router /statements/download [get]
+// @Summary Download statements
+// @Produce text/csv
+// @Param token query string true "download token"
+// @Failure 400 {object} utils.APIError
+// @Failure 401 {object} utils.APIError "Unauthorized failure"
+func (s *Service) downloadHandler(c *gin.Context) {
+	token := c.Query("token")
+	tokenPlain, err := utils.ParseJWTString("statements/download", token)
+	if err != nil {
+		utils.MakeInvalidRequestErrorFromError(c, err)
+		return
+	}
+	arr := strings.Fields(tokenPlain)
+	if len(arr) != 2 {
+		utils.MakeInvalidRequestErrorFromError(c, errors.New("invalid token"))
+		return
+	}
+	secretKey, err := base64.StdEncoding.DecodeString(arr[0])
+	if err != nil {
+		utils.MakeInvalidRequestErrorFromError(c, err)
+		return
+	}
+
+	filePath := arr[1]
+	fileInfo, err := os.Stat(filePath)
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	f, err := os.Open(filePath)
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+
+	c.Writer.Header().Set("Content-type", "text/csv")
+	c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileInfo.Name()))
+	err = aesctr.Decrypt(f, c.Writer, secretKey[0:16], secretKey[16:])
+	if err != nil {
+		log.Error("decrypt csv failed", zap.Error(err))
+	}
+	// delete it anyway
+	f.Close()
+	_ = os.Remove(filePath)
+}

ui/lib/apps/Statement/pages/List/index.tsx

@@ -1,10 +1,21 @@
 import React, { useState } from 'react'
-import { Space, Tooltip, Drawer, Button, Checkbox, Result, Input } from 'antd'
+import {
+  Space,
+  Tooltip,
+  Drawer,
+  Button,
+  Checkbox,
+  Result,
+  Input,
+  Dropdown,
+  Menu,
+  message,
+} from 'antd'
 import { useLocalStorageState } from '@umijs/hooks'
 import {
-  SettingOutlined,
   ReloadOutlined,
   LoadingOutlined,
+  MenuOutlined,
 } from '@ant-design/icons'
 import { ScrollablePane } from 'office-ui-fabric-react/lib/ScrollablePane'
 import { useTranslation } from 'react-i18next'
@@ -17,12 +28,19 @@ import TimeRangeSelector from './TimeRangeSelector'
 import useStatementTableController, {
   DEF_STMT_COLUMN_KEYS,
 } from '../../utils/useStatementTableController'
+import client from '@lib/client'
 
 const { Search } = Input
 
 const STMT_VISIBLE_COLUMN_KEYS = 'statement.visible_column_keys'
 const STMT_SHOW_FULL_SQL = 'statement.show_full_sql'
 
+function downloadByLink(url: string) {
+  const downloadLink = document.createElement('a')
+  downloadLink.href = url
+  downloadLink.click()
+}
+
 export default function StatementsOverview() {
   const { t } = useTranslation()
 
@@ -47,8 +65,44 @@ export default function StatementsOverview() {
     allStmtTypes,
     loadingStatements,
     tableColumns,
+
+    genDownloadToken,
+    downloading,
   } = controller
 
+  async function exportCSV() {
+    message.info(t('statement.pages.overview.toolbar.exporting') + '...', 2)
+    const token = await genDownloadToken()
+    if (token) {
+      const url = `${client.getBasePath()}/statements/download?token=${token}`
+      // `window.open(url)` would cause browser popup interception if genDownloadToken takes long time
+      // window.open(url)
+      downloadByLink(url)
+    }
+  }
+
+  function menuItemClick({ key }) {
+    switch (key) {
+      case 'settings':
+        setShowSettings(true)
+        break
+      case 'export':
+        exportCSV()
+        break
+    }
+  }
+
+  const dropdownMenu = (
+    <Menu onClick={menuItemClick}>
+      <Menu.Item key="settings">{t('statement.settings.title')}</Menu.Item>
+      <Menu.Item key="export" disabled={downloading}>
+        {downloading
+          ? t('statement.pages.overview.toolbar.exporting')
+          : t('statement.pages.overview.toolbar.export')}
+      </Menu.Item>
+    </Menu>
+  )
+
   return (
     <div style={{ height: '100vh', display: 'flex', flexDirection: 'column' }}>
       <Card>
@@ -127,16 +181,18 @@ export default function StatementsOverview() {
                 }
               />
             )}
-            <Tooltip title={t('statement.settings.title')}>
-              <SettingOutlined onClick={() => setShowSettings(true)} />
-            </Tooltip>
             <Tooltip title={t('statement.pages.overview.toolbar.refresh')}>
               {loadingStatements ? (
                 <LoadingOutlined />
               ) : (
                 <ReloadOutlined onClick={refresh} />
               )}
             </Tooltip>
+            <Dropdown overlay={dropdownMenu} placement="bottomRight">
+              <div style={{ cursor: 'pointer' }}>
+                <MenuOutlined />
+              </div>
+            </Dropdown>
           </Space>
         </Toolbar>
       </Card>

ui/lib/apps/Statement/translations/en.yaml

@@ -37,6 +37,8 @@ statement:
           recent: Recent
           usual_time_ranges: Common
           custom_time_ranges: Custom
+        export: Export
+        exporting: Exporting
   settings:
     title: Settings
     disabled_result:

ui/lib/apps/Statement/translations/zh.yaml

@@ -37,6 +37,8 @@ statement:
           recent: 最近
           usual_time_ranges: 常用时间范围
           custom_time_ranges: 自定义时间范围
+        export: 导出
+        exporting: 正在导出
   settings:
     title: 设置
     disabled_result: