Merge pull request #7 from optimyze/jbcrail/render-elastic-flamegraph

Render Elastic flamegraph correctly

src/plugins/profiling/server/routes/flamegraph.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export class FlameGraph {
+  constructor(events, totalEvents, stackTraces, stackFrames, executables) {
+    this.events = events;
+    this.totalEvents = totalEvents;
+    this.stacktraces = stackTraces;
+    this.stackframes = stackFrames;
+    this.executables = executables;
+  }
+
+  toElastic() {
+    let facts = [];
+    for (const trace of this.stacktraces) {
+      if (trace.found) {
+        const pairs = ['root'].concat(trace._source.FrameID).map((item, i) => [i, item]);
+        const fact = {
+          id: trace._source.FrameID[trace._source.FrameID.length - 1],
+          value: 1,
+          depth: trace._source.FrameID.length,
+          layers: Object.fromEntries(pairs),
+        };
+        facts.push(fact);
+      }
+    }
+    return { facts };
+  }
+
+  toPixi() {
+    return {};
+  }
+}

src/plugins/profiling/server/routes/load_flameChartElastic.ts

@@ -25,7 +25,7 @@ export function registerFlameChartElasticRoute(router: IRouter<DataRequestHandle
         }),
       },
     },
-   async (ctx, request, response) => {
+    async (ctx, request, response) => {
       const [timeFrom, timeTo] = timeRangeFromRequest(request);
       const src = await import(`../fixtures/flamechart_${timeTo - timeFrom}`);
       delete src.default;

src/plugins/profiling/server/routes/load_topNHosts.ts

@@ -8,7 +8,7 @@
 import { schema } from '@kbn/config-schema';
 import type { DataRequestHandlerContext } from '../../../data/server';
 import type { IRouter } from '../../../../core/server';
-import {getLocalRoutePaths, timeRangeFromRequest} from '../../common';
+import { getLocalRoutePaths, timeRangeFromRequest } from '../../common';
 
 export function registerTraceEventsTopNHostsRoute(router: IRouter<DataRequestHandlerContext>) {
   const paths = getLocalRoutePaths();

src/plugins/profiling/server/routes/mappings.ts

@@ -123,39 +123,3 @@ export function mapFlamechart(src) {
     facts: newRoot,
   };
 }
-
-export function mapKibanaFlameChart(src) {
-  src.ExeFileName = 'root';
-
-  return {
-    facts: [
-      {
-        id: 'pf-collection-agent: runtime.releaseSudog() in runtime2.go#282',
-        value: 1,
-        depth: 19,
-        layers: {
-          '0': 'root',
-          '1': 'pf-collection-agent: runtime.goexit() in asm_amd64.s#1581',
-          '2': 'pf-collection-agent: github.com/optimyze/prodfiler/pf-storage-backend/storagebackend/storagebackendv1.(*ScyllaExecutor).Start.func1 in scyllaexecutor.go#102',
-          '3': 'pf-collection-agent: github.com/optimyze/prodfiler/pf-storage-backend/storagebackend/storagebackendv1.(*ScyllaExecutor).executeQueryAndReadResults in scyllaexecutor.go#158',
-          '4': 'pf-collection-agent: github.com/gocql/gocql.(*Query).Iter in session.go#1246',
-          '5': 'pf-collection-agent: github.com/gocql/gocql.(*Session).executeQuery in session.go#463',
-          '6': 'pf-collection-agent: github.com/gocql/gocql.(*queryExecutor).executeQuery in query_executor.go#66',
-          '7': 'pf-collection-agent: github.com/gocql/gocql.(*queryExecutor).do in query_executor.go#127',
-          '8': 'pf-collection-agent: github.com/gocql/gocql.(*queryExecutor).attemptQuery in query_executor.go#32',
-          '9': 'pf-collection-agent: github.com/gocql/gocql.(*Query).execute in session.go#1044',
-          '10': 'pf-collection-agent: github.com/gocql/gocql.(*Conn).executeQuery in conn.go#1129',
-          '11': 'pf-collection-agent: github.com/gocql/gocql.(*Conn).exec in conn.go#916',
-          '12': 'pf-collection-agent: github.com/gocql/gocql.(*writeExecuteFrame).writeFrame in frame.go#1618',
-          '13': 'pf-collection-agent: github.com/gocql/gocql.(*framer).writeExecuteFrame in frame.go#1643',
-          '14': 'pf-collection-agent: github.com/gocql/gocql.(*framer).finishWrite in frame.go#788',
-          '15': 'pf-collection-agent: github.com/gocql/gocql.(*Conn).Write in conn.go#319',
-          '16': 'pf-collection-agent: github.com/gocql/gocql.(*writeCoalescer).Write in conn.go#829',
-          '17': 'pf-collection-agent: sync.(*Cond).Wait in cond.go#83',
-          '18': 'pf-collection-agent: sync.runtime_notifyListWait() in sema.go#498',
-          '19': 'pf-collection-agent: runtime.releaseSudog() in runtime2.go#282',
-        },
-      },
-    ],
-  };
-}

src/plugins/profiling/server/routes/search_flameChart.ts

@@ -6,11 +6,10 @@
  * Side Public License, v 1.
  */
 import { schema } from '@kbn/config-schema';
-import { IEsSearchRequest } from '../../../data/server';
-import { IEsSearchResponse } from '../../../data/common';
 import type { DataRequestHandlerContext } from '../../../data/server';
 import type { IRouter } from '../../../../core/server';
 import { getRemoteRoutePaths } from '../../common';
+import { FlameGraph } from './flamegraph';
 
 export function registerFlameChartSearchRoute(router: IRouter<DataRequestHandlerContext>) {
   const paths = getRemoteRoutePaths();
@@ -30,152 +29,152 @@ export function registerFlameChartSearchRoute(router: IRouter<DataRequestHandler
       const { index, projectID, timeFrom, timeTo } = request.query;
 
       try {
-        const resFlamegraphTraces = await context
-          .search!.search(
-            {
-              params: {
-                index,
-                body: {
-                  query: {
-                    function_score: {
-                      query: {
-                        bool: {
-                          must: [
-                            {
-                              term: {
-                                ProjectID: {
-                                  value: projectID,
-                                  boost: 1.0,
-                                },
-                              },
-                            },
-                            {
-                              range: {
-                                '@timestamp': {
-                                  gte: timeFrom,
-                                  lt: timeTo,
-                                  format: 'epoch_second',
-                                  boost: 1.0,
-                                },
-                              },
-                            },
-                          ],
+        const esClient = context.core.elasticsearch.client.asCurrentUser;
+
+        const resEvents = await esClient.search({
+          index,
+          body: {
+            query: {
+              function_score: {
+                query: {
+                  bool: {
+                    must: [
+                      {
+                        term: {
+                          ProjectID: {
+                            value: projectID,
+                            boost: 1.0,
+                          },
                         },
                       },
-                      random_score: {},
-                    },
-                  },
-                  aggs: {
-                    sample: {
-                      sampler: {
-                        shard_size: 20000,
-                      },
-                      aggs: {
-                        group_by: {
-                          terms: {
-                            field: 'StackTraceID',
-                            size: 20000,
+                      {
+                        range: {
+                          '@timestamp': {
+                            gte: timeFrom,
+                            lt: timeTo,
+                            format: 'epoch_second',
+                            boost: 1.0,
                           },
                         },
                       },
+                    ],
+                  },
+                },
+              },
+            },
+            aggs: {
+              sample: {
+                sampler: {
+                  shard_size: 20000,
+                },
+                aggs: {
+                  group_by: {
+                    terms: {
+                      field: 'StackTraceID',
+                      size: 20000,
                     },
                   },
                 },
               },
-            } as IEsSearchRequest,
-            {}
-          )
-          .toPromise();
+            },
+          },
+        });
 
-        const resTotalTraces = await context
-          .search!.search(
-            {
-              params: {
-                index,
-                body: {
-                  query: {
-                    bool: {
-                      must: [
-                        {
-                          term: {
-                            ProjectID: {
-                              value: projectID,
-                              boost: 1.0,
-                            },
-                          },
-                        },
-                        {
-                          range: {
-                            '@timestamp': {
-                              gte: timeFrom,
-                              lt: timeTo,
-                              format: 'epoch_second',
-                              boost: 1.0,
-                            },
-                          },
-                        },
-                      ],
+        const resTotalEvents = await esClient.search({
+          index,
+          body: {
+            query: {
+              bool: {
+                must: [
+                  {
+                    term: {
+                      ProjectID: {
+                        value: projectID,
+                        boost: 1.0,
+                      },
                     },
                   },
-                  aggs: {
-                    histogram: {
-                      auto_date_histogram: {
-                        field: '@timestamp',
-                        buckets: 100,
-                      },
-                      aggs: {
-                        Count: {
-                          sum: {
-                            field: 'Count',
-                          },
-                        },
+                  {
+                    range: {
+                      '@timestamp': {
+                        gte: timeFrom,
+                        lt: timeTo,
+                        format: 'epoch_second',
+                        boost: 1.0,
                       },
                     },
                   },
+                ],
+              },
+            },
+            aggs: {
+              histogram: {
+                auto_date_histogram: {
+                  field: '@timestamp',
+                  buckets: 100,
+                },
+                aggs: {
+                  Count: {
+                    sum: {
+                      field: 'Count',
+                    },
+                  },
                 },
               },
-            } as IEsSearchRequest,
-            {}
-          )
-          .toPromise();
+            },
+          },
+        });
 
         const tracesDocIDs: string[] = [];
-        resFlamegraphTraces.rawResponse.aggregations.sample.group_by.buckets.forEach(
-          (stackTraceItem: any) => {
-            tracesDocIDs.push(stackTraceItem.key);
-          }
-        );
-
-        const esClient = context.core.elasticsearch.client.asCurrentUser;
+        resEvents.body.aggregations.sample.group_by.buckets.forEach((stackTraceItem: any) => {
+          tracesDocIDs.push(stackTraceItem.key);
+        });
 
         const resStackTraces = await esClient.mget<any>({
           index: 'profiling-stacktraces',
           body: { ids: tracesDocIDs },
         });
 
-        const stackFrameDocIDs: string[] = [];
-        resStackTraces.body.docs.forEach((trace: any) => {
-          // sometimes we don't find the trace - needs investigation as we should always find
-          // profiling-events to profiling-stack-traces
-          if (trace._source) {
-            for (let i = 0; i < trace._source.Offset.length; i++) {
-              stackFrameDocIDs.push(trace._source.FrameID[i]);
+        // sometimes we don't find the trace - needs investigation as we should always find
+        // profiling-events to profiling-stack-traces
+        const stackFrameDocIDs = new Set<string>();
+        for (const trace of resStackTraces.body.docs) {
+          if (trace.found) {
+            for (const frameID of trace._source.FrameID) {
+              stackFrameDocIDs.add(frameID);
             }
           }
-        });
+        }
 
         const resStackFrames = await esClient.mget<any>({
           index: 'profiling-stackframes',
-          body: { ids: stackFrameDocIDs },
+          body: { ids: [...stackFrameDocIDs] },
+        });
+
+        const executableDocIDs = new Set<string>();
+        for (const trace of resStackTraces.body.docs) {
+          if (trace.found) {
+            for (const fileID of trace._source.FileID) {
+              executableDocIDs.add(fileID);
+            }
+          }
+        }
+
+        const resExecutables = await esClient.mget<any>({
+          index: 'profiling-executables',
+          body: { ids: [...executableDocIDs] },
         });
 
+        const flamegraph = new FlameGraph(
+          resEvents.body,
+          resTotalEvents.body,
+          resStackTraces.body.docs,
+          resStackFrames.body.docs,
+          resExecutables.body.docs
+        );
+
         return response.ok({
-          body: {
-            flameChart: (resFlamegraphTraces as IEsSearchResponse).rawResponse,
-            totalTraces: (resTotalTraces as IEsSearchResponse).rawResponse.aggregations,
-            stackTraces: resStackTraces.body.docs,
-            stackFrames: resStackFrames.body.docs,
-          },
+          body: flamegraph.toElastic(),
         });
       } catch (e) {
         return response.customError({

src/plugins/profiling/server/routes/search_topNDeployments.ts

@@ -8,7 +8,7 @@
 import type { DataRequestHandlerContext } from '../../../data/server';
 import type { IRouter } from '../../../../core/server';
 import { getRemoteRoutePaths } from '../../common';
-import { queryTopNCommon } from "./search_topN";
+import { queryTopNCommon } from './search_topN';
 
 export function registerTraceEventsTopNDeploymentsSearchRoute(
   router: IRouter<DataRequestHandlerContext>

src/plugins/profiling/server/routes/search_topNHosts.ts

@@ -8,7 +8,7 @@
 import type { DataRequestHandlerContext } from '../../../data/server';
 import type { IRouter } from '../../../../core/server';
 import { getRemoteRoutePaths } from '../../common';
-import { queryTopNCommon } from "./search_topN";
+import { queryTopNCommon } from './search_topN';
 
 export function registerTraceEventsTopNHostsSearchRoute(
   router: IRouter<DataRequestHandlerContext>