From 7a9da9e7c3259cebfa28ef9f20174092d0df57f9 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 6 Feb 2020 20:20:28 +0800 Subject: [PATCH 1/5] infoschema: add metric_detail table to query all detail metrics Signed-off-by: crazycs --- executor/builder.go | 8 ++++ executor/metric_reader.go | 69 ++++++++++++++++++++++++++++ infoschema/tables.go | 13 ++++++ planner/core/logical_plan_builder.go | 2 +- session/session.go | 4 ++ 5 files changed, 95 insertions(+), 1 deletion(-) diff --git a/executor/builder.go b/executor/builder.go index 30b64b64fe9b2..cffe8378edc62 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1327,6 +1327,14 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo extractor: v.Extractor.(*plannercore.MetricTableExtractor), }, } + case strings.ToLower(infoschema.TableMetricDetail): + return &ClusterReaderExec{ + baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), + retriever: &MetricDetailRetriever{ + table: v.Table, + extractor: v.Extractor.(*plannercore.MetricTableExtractor), + }, + } } } tb, _ := b.is.TableByID(v.Table.ID) diff --git a/executor/metric_reader.go b/executor/metric_reader.go index cea4206048e24..49d03479754a4 100644 --- a/executor/metric_reader.go +++ b/executor/metric_reader.go @@ -243,3 +243,72 @@ func (e *MetricSummaryRetriever) genMetricQuerySQLS(name, startTime, endTime str } return sqls } + +// MetricDetailRetriever uses to read metric detail data. +type MetricDetailRetriever struct { + dummyCloser + table *model.TableInfo + extractor *plannercore.MetricTableExtractor + retrieved bool +} + +func (e *MetricDetailRetriever) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) { + if e.retrieved || e.extractor.SkipRequest { + return nil, nil + } + e.retrieved = true + totalRows := make([][]types.Datum, 0, len(infoschema.MetricTableMap)) + quantiles := []float64{1, 0.999, 0.99, 0.90, 0.80} + tps := make([]*types.FieldType, 0, len(e.table.Columns)) + for _, col := range e.table.Columns { + tps = append(tps, &col.FieldType) + } + startTime := e.extractor.StartTime.Format(plannercore.MetricTableTimeFormat) + endTime := e.extractor.EndTime.Format(plannercore.MetricTableTimeFormat) + for name, def := range infoschema.MetricTableMap { + sqls := e.genMetricQuerySQLS(name, startTime, endTime, def.Quantile, quantiles, def) + for _, sql := range sqls { + rows, _, err := sctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(sql) + if err != nil { + return nil, errors.Trace(err) + } + for _, row := range rows { + totalRows = append(totalRows, row.GetDatumRow(tps)) + } + } + } + return totalRows, nil +} + +func (e *MetricDetailRetriever) genMetricQuerySQLS(name, startTime, endTime string, quantile float64, quantiles []float64, def infoschema.MetricTableDef) []string { + //labels := strings.Join(def.Labels, ",") + labels := "" + labelsColumn := `""` + for i, label := range def.Labels { + if i > 0 { + labels = labels + "," + } + labels = labels + "`" + label + "`" + } + if len(labels) > 0 { + labelsColumn = fmt.Sprintf("concat_ws(' = ','%s',concat_ws(', ',%s))", strings.Join(def.Labels, ", "), labels) + } + if quantile == 0 { + sql := fmt.Sprintf(`select "%[1]s", %[2]s as label,min(time),sum(value),avg(value),min(value),max(value) from metric_schema.%[1]s where time > '%[3]s' and time < '%[4]s'`, + name, labelsColumn, startTime, endTime) + if len(def.Labels) > 0 { + sql += " group by " + labels + } + return []string{sql} + } + sqls := []string{} + for _, quantile := range quantiles { + sql := fmt.Sprintf(`select "%[1]s_%[5]v", %[2]s as label,min(time),sum(value),avg(value),min(value),max(value) from metric_schema.%[1]s where time > '%[3]s' and time < '%[4]s' and quantile=%[5]v`, + name, labelsColumn, startTime, endTime, quantile) + if len(def.Labels) > 0 { + sql += " group by " + labels + } + sqls = append(sqls, sql) + } + return sqls +} diff --git a/infoschema/tables.go b/infoschema/tables.go index 18e9e7c8bdfcc..dedc6baaea6f5 100755 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -108,6 +108,8 @@ const ( TableInspectionResult = "INSPECTION_RESULT" // TableMetricSummary is a summary table that contains all metrics. TableMetricSummary = "METRIC_SUMMARY" + // TableMetricDetail is a detail table that contains all metrics detail info. + TableMetricDetail = "METRIC_DETAIL" ) var tableIDMap = map[string]int64{ @@ -163,6 +165,7 @@ var tableIDMap = map[string]int64{ TableClusterSystemInfo: autoid.InformationSchemaDBID + 50, TableInspectionResult: autoid.InformationSchemaDBID + 51, TableMetricSummary: autoid.InformationSchemaDBID + 52, + TableMetricDetail: autoid.InformationSchemaDBID + 53, } type columnInfo struct { @@ -1134,6 +1137,15 @@ var tableMetricSummaryCols = []columnInfo{ {"MIN_VALUE", mysql.TypeDouble, 22, 0, nil, nil}, {"MAX_VALUE", mysql.TypeDouble, 22, 0, nil, nil}, } +var tableMetricDetailCols = []columnInfo{ + {"METRIC_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, + {"LABEL", mysql.TypeVarchar, 64, 0, nil, nil}, + {"TIME", mysql.TypeDatetime, -1, 0, nil, nil}, + {"SUM_VALUE", mysql.TypeDouble, 22, 0, nil, nil}, + {"AVG_VALUE", mysql.TypeDouble, 22, 0, nil, nil}, + {"MIN_VALUE", mysql.TypeDouble, 22, 0, nil, nil}, + {"MAX_VALUE", mysql.TypeDouble, 22, 0, nil, nil}, +} func dataForSchemata(ctx sessionctx.Context, schemas []*model.DBInfo) [][]types.Datum { checker := privilege.GetPrivilegeManager(ctx) @@ -2345,6 +2357,7 @@ var tableNameToColumns = map[string][]columnInfo{ TableClusterSystemInfo: tableClusterSystemInfoCols, TableInspectionResult: tableInspectionResultCols, TableMetricSummary: tableMetricSummaryCols, + TableMetricDetail: tableMetricDetailCols, } func createInfoSchemaTable(_ autoid.Allocators, meta *model.TableInfo) (table.Table, error) { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index bda7da337dd4a..2e8685ee2c034 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2735,7 +2735,7 @@ func (b *PlanBuilder) buildMemTable(ctx context.Context, dbName model.CIStr, tab p.Extractor = &ClusterLogTableExtractor{} case infoschema.TableInspectionResult: p.Extractor = &InspectionResultTableExtractor{} - case infoschema.TableMetricSummary: + case infoschema.TableMetricSummary, infoschema.TableMetricDetail: p.Extractor = newMetricTableExtractor() } } diff --git a/session/session.go b/session/session.go index 57d42a2010e3a..b5b0da8ea71c6 100644 --- a/session/session.go +++ b/session/session.go @@ -853,6 +853,10 @@ func createSessionFunc(store kv.Storage) pools.Factory { if err != nil { return nil, errors.Trace(err) } + err = variable.SetSessionSystemVar(se.sessionVars, variable.MaxAllowedPacket, types.NewStringDatum("67108864")) + if err != nil { + return nil, errors.Trace(err) + } se.sessionVars.CommonGlobalLoaded = true se.sessionVars.InRestrictedSQL = true return se, nil From 5656d8d6d990bf5e97d25e8d4900c8303b180d5b Mon Sep 17 00:00:00 2001 From: crazycs Date: Fri, 7 Feb 2020 15:17:47 +0800 Subject: [PATCH 2/5] change table name --- infoschema/tables.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infoschema/tables.go b/infoschema/tables.go index dedc6baaea6f5..4c565d4fe8730 100755 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -107,9 +107,9 @@ const ( // TableInspectionResult is the string constant of inspection result table TableInspectionResult = "INSPECTION_RESULT" // TableMetricSummary is a summary table that contains all metrics. - TableMetricSummary = "METRIC_SUMMARY" + TableMetricSummary = "METRICS_SUMMARY" // TableMetricDetail is a detail table that contains all metrics detail info. - TableMetricDetail = "METRIC_DETAIL" + TableMetricDetail = "METRICS_DETAIL" ) var tableIDMap = map[string]int64{ From 3b5b795d5da27dec7ed58617ea2cd0556ad2fccb Mon Sep 17 00:00:00 2001 From: crazycs Date: Fri, 7 Feb 2020 15:34:27 +0800 Subject: [PATCH 3/5] rename table Signed-off-by: crazycs --- infoschema/tables.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infoschema/tables.go b/infoschema/tables.go index 4c565d4fe8730..d8fde16207c36 100755 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -108,8 +108,8 @@ const ( TableInspectionResult = "INSPECTION_RESULT" // TableMetricSummary is a summary table that contains all metrics. TableMetricSummary = "METRICS_SUMMARY" - // TableMetricDetail is a detail table that contains all metrics detail info. - TableMetricDetail = "METRICS_DETAIL" + // TableMetricDetail is a detail table that contains all metrics detail info. It is the table metric_summary by label. + TableMetricDetail = "METRICS_SUMMARY_BY_LABEL" ) var tableIDMap = map[string]int64{ From 58044e92573639454fcf27a4a1f8a00e4a38515c Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 10 Feb 2020 16:30:36 +0800 Subject: [PATCH 4/5] address comment Signed-off-by: crazycs --- executor/builder.go | 4 ++-- executor/metric_reader.go | 8 ++++---- infoschema/tables.go | 10 +++++----- planner/core/logical_plan_builder.go | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index cffe8378edc62..e478b376dc55a 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1327,10 +1327,10 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo extractor: v.Extractor.(*plannercore.MetricTableExtractor), }, } - case strings.ToLower(infoschema.TableMetricDetail): + case strings.ToLower(infoschema.TableMetricSummaryByLabel): return &ClusterReaderExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), - retriever: &MetricDetailRetriever{ + retriever: &MetricSummaryByLabelRetriever{ table: v.Table, extractor: v.Extractor.(*plannercore.MetricTableExtractor), }, diff --git a/executor/metric_reader.go b/executor/metric_reader.go index 49d03479754a4..a9875b12ddd9b 100644 --- a/executor/metric_reader.go +++ b/executor/metric_reader.go @@ -244,15 +244,15 @@ func (e *MetricSummaryRetriever) genMetricQuerySQLS(name, startTime, endTime str return sqls } -// MetricDetailRetriever uses to read metric detail data. -type MetricDetailRetriever struct { +// MetricSummaryByLabelRetriever uses to read metric detail data. +type MetricSummaryByLabelRetriever struct { dummyCloser table *model.TableInfo extractor *plannercore.MetricTableExtractor retrieved bool } -func (e *MetricDetailRetriever) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) { +func (e *MetricSummaryByLabelRetriever) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) { if e.retrieved || e.extractor.SkipRequest { return nil, nil } @@ -280,7 +280,7 @@ func (e *MetricDetailRetriever) retrieve(ctx context.Context, sctx sessionctx.Co return totalRows, nil } -func (e *MetricDetailRetriever) genMetricQuerySQLS(name, startTime, endTime string, quantile float64, quantiles []float64, def infoschema.MetricTableDef) []string { +func (e *MetricSummaryByLabelRetriever) genMetricQuerySQLS(name, startTime, endTime string, quantile float64, quantiles []float64, def infoschema.MetricTableDef) []string { //labels := strings.Join(def.Labels, ",") labels := "" labelsColumn := `""` diff --git a/infoschema/tables.go b/infoschema/tables.go index d8fde16207c36..62c9255f6165f 100755 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -108,8 +108,8 @@ const ( TableInspectionResult = "INSPECTION_RESULT" // TableMetricSummary is a summary table that contains all metrics. TableMetricSummary = "METRICS_SUMMARY" - // TableMetricDetail is a detail table that contains all metrics detail info. It is the table metric_summary by label. - TableMetricDetail = "METRICS_SUMMARY_BY_LABEL" + // TableMetricSummaryByLabel is a metric table that contains all metrics that group by label info. + TableMetricSummaryByLabel = "METRICS_SUMMARY_BY_LABEL" ) var tableIDMap = map[string]int64{ @@ -165,7 +165,7 @@ var tableIDMap = map[string]int64{ TableClusterSystemInfo: autoid.InformationSchemaDBID + 50, TableInspectionResult: autoid.InformationSchemaDBID + 51, TableMetricSummary: autoid.InformationSchemaDBID + 52, - TableMetricDetail: autoid.InformationSchemaDBID + 53, + TableMetricSummaryByLabel: autoid.InformationSchemaDBID + 53, } type columnInfo struct { @@ -1137,7 +1137,7 @@ var tableMetricSummaryCols = []columnInfo{ {"MIN_VALUE", mysql.TypeDouble, 22, 0, nil, nil}, {"MAX_VALUE", mysql.TypeDouble, 22, 0, nil, nil}, } -var tableMetricDetailCols = []columnInfo{ +var tableMetricSummaryByLabelCols = []columnInfo{ {"METRIC_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, {"LABEL", mysql.TypeVarchar, 64, 0, nil, nil}, {"TIME", mysql.TypeDatetime, -1, 0, nil, nil}, @@ -2357,7 +2357,7 @@ var tableNameToColumns = map[string][]columnInfo{ TableClusterSystemInfo: tableClusterSystemInfoCols, TableInspectionResult: tableInspectionResultCols, TableMetricSummary: tableMetricSummaryCols, - TableMetricDetail: tableMetricDetailCols, + TableMetricSummaryByLabel: tableMetricSummaryByLabelCols, } func createInfoSchemaTable(_ autoid.Allocators, meta *model.TableInfo) (table.Table, error) { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 2e8685ee2c034..2315ff0af1a6a 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2735,7 +2735,7 @@ func (b *PlanBuilder) buildMemTable(ctx context.Context, dbName model.CIStr, tab p.Extractor = &ClusterLogTableExtractor{} case infoschema.TableInspectionResult: p.Extractor = &InspectionResultTableExtractor{} - case infoschema.TableMetricSummary, infoschema.TableMetricDetail: + case infoschema.TableMetricSummary, infoschema.TableMetricSummaryByLabel: p.Extractor = newMetricTableExtractor() } } From b7b2a0a42eb33c6a38541e784d6509d0903198f6 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 10 Feb 2020 22:24:11 +0800 Subject: [PATCH 5/5] address comment Signed-off-by: crazycs --- executor/metric_reader.go | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/executor/metric_reader.go b/executor/metric_reader.go index 5791e6c93be56..1ae10a2002911 100644 --- a/executor/metric_reader.go +++ b/executor/metric_reader.go @@ -266,7 +266,7 @@ func (e *MetricSummaryByLabelRetriever) retrieve(ctx context.Context, sctx sessi startTime := e.extractor.StartTime.Format(plannercore.MetricTableTimeFormat) endTime := e.extractor.EndTime.Format(plannercore.MetricTableTimeFormat) for name, def := range infoschema.MetricTableMap { - sqls := e.genMetricQuerySQLS(name, startTime, endTime, def.Quantile, quantiles, def) + sqls := e.genMetricQuerySQLS(name, startTime, endTime, quantiles, def) for _, sql := range sqls { rows, _, err := sctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(sql) if err != nil { @@ -280,31 +280,26 @@ func (e *MetricSummaryByLabelRetriever) retrieve(ctx context.Context, sctx sessi return totalRows, nil } -func (e *MetricSummaryByLabelRetriever) genMetricQuerySQLS(name, startTime, endTime string, quantile float64, quantiles []float64, def infoschema.MetricTableDef) []string { - //labels := strings.Join(def.Labels, ",") +func (e *MetricSummaryByLabelRetriever) genMetricQuerySQLS(name, startTime, endTime string, quantiles []float64, def infoschema.MetricTableDef) []string { labels := "" labelsColumn := `""` - for i, label := range def.Labels { - if i > 0 { - labels = labels + "," - } - labels = labels + "`" + label + "`" - } - if len(labels) > 0 { - labelsColumn = fmt.Sprintf("concat_ws(' = ','%s',concat_ws(', ',%s))", strings.Join(def.Labels, ", "), labels) + if len(def.Labels) > 0 { + labels = "`" + strings.Join(def.Labels, "`, `") + "`" + labelsColumn = fmt.Sprintf("concat_ws(' = ', '%s', concat_ws(', ', %s))", strings.Join(def.Labels, ", "), labels) } - if quantile == 0 { - sql := fmt.Sprintf(`select "%[1]s", %[2]s as label,min(time),sum(value),avg(value),min(value),max(value) from metric_schema.%[1]s where time > '%[3]s' and time < '%[4]s'`, - name, labelsColumn, startTime, endTime) - if len(def.Labels) > 0 { - sql += " group by " + labels - } - return []string{sql} + if def.Quantile == 0 { + quantiles = []float64{0} } sqls := []string{} for _, quantile := range quantiles { - sql := fmt.Sprintf(`select "%[1]s_%[5]v", %[2]s as label,min(time),sum(value),avg(value),min(value),max(value) from metric_schema.%[1]s where time > '%[3]s' and time < '%[4]s' and quantile=%[5]v`, - name, labelsColumn, startTime, endTime, quantile) + var sql string + if quantile == 0 { + sql = fmt.Sprintf(`select "%[1]s", %[2]s as label,min(time),sum(value),avg(value),min(value),max(value) from metric_schema.%[1]s where time > '%[3]s' and time < '%[4]s'`, + name, labelsColumn, startTime, endTime) + } else { + sql = fmt.Sprintf(`select "%[1]s_%[5]v", %[2]s as label,min(time),sum(value),avg(value),min(value),max(value) from metric_schema.%[1]s where time > '%[3]s' and time < '%[4]s' and quantile=%[5]v`, + name, labelsColumn, startTime, endTime, quantile) + } if len(def.Labels) > 0 { sql += " group by " + labels }