[Reporting] Add additional PNG and PDF metrics (#125450)

* Update browser driver to return metrics along with the results * Add metrics to the reporting job * Add metrics to the event logging * Add screenshot metrics to the report info panel
elastic · Feb 15, 2022 · 556b629 · 556b629
1 parent 0202164
commit 556b629
Show file tree

Hide file tree

Showing 33 changed files with 281 additions and 81 deletions.
diff --git a/x-pack/plugins/reporting/common/types/index.ts b/x-pack/plugins/reporting/common/types/index.ts
@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import type { ScreenshotResult } from '../../../screenshotting/server';
 import type { BaseParams, BaseParamsV2, BasePayload, BasePayloadV2, JobId } from './base';
 
 export type { JobParamsPNGDeprecated } from './export_types/png';
@@ -33,12 +35,33 @@ export interface ReportOutput extends TaskRunResult {
   size: number;
 }
 
+type ScreenshotMetrics = Required<ScreenshotResult>['metrics'];
+
+export interface CsvMetrics {
+  rows: number;
+}
+
+export type PngMetrics = ScreenshotMetrics;
+
+export interface PdfMetrics extends Partial<ScreenshotMetrics> {
+  /**
+   * A number of emitted pages in the generated PDF report.
+   */
+  pages: number;
+}
+
+export interface TaskRunMetrics {
+  csv?: CsvMetrics;
+  png?: PngMetrics;
+  pdf?: PdfMetrics;
+}
+
 export interface TaskRunResult {
   content_type: string | null;
   csv_contains_formulas?: boolean;
-  csv_rows?: number;
   max_size_reached?: boolean;
   warnings?: string[];
+  metrics?: TaskRunMetrics;
 }
 
 export interface ReportSource {
@@ -76,6 +99,7 @@ export interface ReportSource {
   started_at?: string; // timestamp in UTC
   completed_at?: string; // timestamp in UTC
   process_expiration?: string | null; // timestamp in UTC - is overwritten with `null` when the job needs a retry
+  metrics?: TaskRunMetrics;
 }
 
 /*
@@ -131,7 +155,6 @@ export interface JobSummary {
   title: ReportSource['payload']['title'];
   maxSizeReached: TaskRunResult['max_size_reached'];
   csvContainsFormulas: TaskRunResult['csv_contains_formulas'];
-  csvRows: TaskRunResult['csv_rows'];
 }
 
 export interface JobSummarySet {

diff --git a/x-pack/plugins/reporting/public/lib/__snapshots__/stream_handler.test.ts.snap b/x-pack/plugins/reporting/public/lib/__snapshots__/stream_handler.test.ts.snap
diff --git a/x-pack/plugins/reporting/public/lib/job.tsx b/x-pack/plugins/reporting/public/lib/job.tsx
@@ -55,8 +55,8 @@ export class Job {
   public size?: ReportOutput['size'];
   public content_type?: TaskRunResult['content_type'];
   public csv_contains_formulas?: TaskRunResult['csv_contains_formulas'];
-  public csv_rows?: TaskRunResult['csv_rows'];
   public max_size_reached?: TaskRunResult['max_size_reached'];
+  public metrics?: ReportSource['metrics'];
   public warnings?: TaskRunResult['warnings'];
 
   public locatorParams?: BaseParamsV2['locatorParams'];
@@ -88,10 +88,10 @@ export class Job {
     this.isDeprecated = report.payload.isDeprecated || false;
     this.spaceId = report.payload.spaceId;
     this.csv_contains_formulas = report.output?.csv_contains_formulas;
-    this.csv_rows = report.output?.csv_rows;
     this.max_size_reached = report.output?.max_size_reached;
     this.warnings = report.output?.warnings;
     this.locatorParams = (report.payload as BaseParamsV2).locatorParams;
+    this.metrics = report.metrics;
   }
 
   getStatusMessage() {

diff --git a/x-pack/plugins/reporting/public/lib/stream_handler.test.ts b/x-pack/plugins/reporting/public/lib/stream_handler.test.ts
@@ -24,7 +24,7 @@ const mockJobsFound: Job[] = [
   { id: 'job-source-mock1', status: 'completed', output: { csv_contains_formulas: false, max_size_reached: false }, payload: { title: 'specimen' } },
   { id: 'job-source-mock2', status: 'failed', output: { csv_contains_formulas: false, max_size_reached: false }, payload: { title: 'specimen' } },
   { id: 'job-source-mock3', status: 'pending', output: { csv_contains_formulas: false, max_size_reached: false }, payload: { title: 'specimen' } },
-  { id: 'job-source-mock4', status: 'completed', output: { csv_contains_formulas: true, csv_rows: 42000000, max_size_reached: false }, payload: { title: 'specimen' } },
+  { id: 'job-source-mock4', status: 'completed', output: { csv_contains_formulas: true, max_size_reached: false }, payload: { title: 'specimen' } },
 ].map((j) => new Job(j as ReportApiJSON)); // prettier-ignore
 
 const coreSetup = coreMock.createSetup();

diff --git a/x-pack/plugins/reporting/public/lib/stream_handler.ts b/x-pack/plugins/reporting/public/lib/stream_handler.ts
@@ -33,7 +33,6 @@ function getReportStatus(src: Job): JobSummary {
     jobtype: src.prettyJobTypeName ?? src.jobtype,
     maxSizeReached: src.max_size_reached,
     csvContainsFormulas: src.csv_contains_formulas,
-    csvRows: src.csv_rows,
   };
 }
 

diff --git a/x-pack/plugins/reporting/public/management/components/report_info_flyout_content.tsx b/x-pack/plugins/reporting/public/management/components/report_info_flyout_content.tsx
@@ -50,8 +50,12 @@ export const ReportInfoFlyoutContent: FunctionComponent<Props> = ({ info }) => {
 
   const formatDate = createDateFormatter(uiSettings.get('dateFormat'), timezone);
 
-  const hasCsvRows = info.csv_rows != null;
+  const cpuInPercentage = info.metrics?.pdf?.cpuInPercentage ?? info.metrics?.png?.cpuInPercentage;
+  const memoryInMegabytes =
+    info.metrics?.pdf?.memoryInMegabytes ?? info.metrics?.png?.memoryInMegabytes;
+  const hasCsvRows = info.metrics?.csv?.rows != null;
   const hasScreenshot = USES_HEADLESS_JOB_TYPES.includes(info.jobtype);
+  const hasPdfPagesMetric = info.metrics?.pdf?.pages != null;
 
   const outputInfo = [
     {
@@ -99,7 +103,7 @@ export const ReportInfoFlyoutContent: FunctionComponent<Props> = ({ info }) => {
       title: i18n.translate('xpack.reporting.listing.infoPanel.csvRows', {
         defaultMessage: 'CSV rows',
       }),
-      description: info.csv_rows?.toString() || NA,
+      description: info.metrics?.csv?.rows?.toString() || NA,
     },
 
     hasScreenshot && {
@@ -118,6 +122,12 @@ export const ReportInfoFlyoutContent: FunctionComponent<Props> = ({ info }) => {
       description:
         info.layout?.dimensions?.width != null ? Math.ceil(info.layout.dimensions.width) : UNKNOWN,
     },
+    hasPdfPagesMetric && {
+      title: i18n.translate('xpack.reporting.listing.infoPanel.pdfPagesInfo', {
+        defaultMessage: 'Pages count',
+      }),
+      description: info.metrics?.pdf?.pages,
+    },
 
     {
       title: i18n.translate('xpack.reporting.listing.infoPanel.processedByInfo', {
@@ -132,6 +142,20 @@ export const ReportInfoFlyoutContent: FunctionComponent<Props> = ({ info }) => {
       }),
       description: info.prettyTimeout,
     },
+
+    cpuInPercentage != null && {
+      title: i18n.translate('xpack.reporting.listing.infoPanel.cpuInfo', {
+        defaultMessage: 'CPU usage',
+      }),
+      description: `${cpuInPercentage}%`,
+    },
+
+    memoryInMegabytes != null && {
+      title: i18n.translate('xpack.reporting.listing.infoPanel.memoryInfo', {
+        defaultMessage: 'RAM usage',
+      }),
+      description: `${memoryInMegabytes}MB`,
+    },
   ].filter(Boolean) as EuiDescriptionListProps['listItems'];
 
   const timestampsInfo = [

diff --git a/x-pack/plugins/reporting/server/export_types/common/generate_png.ts b/x-pack/plugins/reporting/server/export_types/common/generate_png.ts
@@ -10,15 +10,22 @@ import * as Rx from 'rxjs';
 import { finalize, map, tap } from 'rxjs/operators';
 import { LayoutTypes } from '../../../../screenshotting/common';
 import { REPORTING_TRANSACTION_TYPE } from '../../../common/constants';
+import type { PngMetrics } from '../../../common/types';
 import { ReportingCore } from '../../';
 import { ScreenshotOptions } from '../../types';
 import { LevelLogger } from '../../lib';
 
+interface PngResult {
+  buffer: Buffer;
+  metrics?: PngMetrics;
+  warnings: string[];
+}
+
 export function generatePngObservable(
   reporting: ReportingCore,
   logger: LevelLogger,
   options: ScreenshotOptions
-): Rx.Observable<{ buffer: Buffer; warnings: string[] }> {
+): Rx.Observable<PngResult> {
   const apmTrans = apm.startTransaction('generate-png', REPORTING_TRANSACTION_TYPE);
   const apmLayout = apmTrans?.startSpan('create-layout', 'setup');
   if (!options.layout.dimensions) {
@@ -35,15 +42,16 @@ export function generatePngObservable(
   let apmBuffer: typeof apm.currentSpan;
 
   return reporting.getScreenshots({ ...options, layout }).pipe(
-    tap(({ metrics$ }) => {
-      metrics$.subscribe(({ cpu, memory }) => {
-        apmTrans?.setLabel('cpu', cpu, false);
-        apmTrans?.setLabel('memory', memory, false);
-      });
+    tap(({ metrics }) => {
+      if (metrics) {
+        apmTrans?.setLabel('cpu', metrics.cpu, false);
+        apmTrans?.setLabel('memory', metrics.memory, false);
+      }
       apmScreenshots?.end();
       apmBuffer = apmTrans?.startSpan('get-buffer', 'output') ?? null;
     }),
-    map(({ results }) => ({
+    map(({ metrics, results }) => ({
+      metrics,
       buffer: results[0].screenshots[0].data,
       warnings: results.reduce((found, current) => {
         if (current.error) {

diff --git a/x-pack/plugins/reporting/server/export_types/common/pdf/integration_tests/pdfmaker.test.ts b/x-pack/plugins/reporting/server/export_types/common/pdf/integration_tests/pdfmaker.test.ts
@@ -32,4 +32,29 @@ describe('PdfMaker', () => {
       await expect(pdf.getBuffer()).resolves.toBeInstanceOf(Buffer);
     });
   });
+
+  describe('getPageCount', () => {
+    it('should return zero pages on no content', () => {
+      expect(pdf.getPageCount()).toBe(0);
+    });
+
+    it('should return a number of generated pages', () => {
+      for (let i = 0; i < 100; i++) {
+        pdf.addImage(imageBase64, { title: `${i} viz`, description: '☃️' });
+      }
+      pdf.generate();
+
+      expect(pdf.getPageCount()).toBe(100);
+    });
+
+    it('should return a number of already flushed pages', async () => {
+      for (let i = 0; i < 100; i++) {
+        pdf.addImage(imageBase64, { title: `${i} viz`, description: '☃️' });
+      }
+      pdf.generate();
+      await pdf.getBuffer();
+
+      expect(pdf.getPageCount()).toBe(100);
+    });
+  });
 });
diff --git a/x-pack/plugins/reporting/server/export_types/common/pdf/pdfmaker.ts b/x-pack/plugins/reporting/server/export_types/common/pdf/pdfmaker.ts
@@ -157,4 +157,14 @@ export class PdfMaker {
       this._pdfDoc.end();
     });
   }
+
+  getPageCount(): number {
+    const pageRange = this._pdfDoc?.bufferedPageRange();
+    if (!pageRange) {
+      return 0;
+    }
+    const { count, start } = pageRange;
+
+    return start + count;
+  }
 }
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts
@@ -395,8 +395,10 @@ export class CsvGenerator {
     return {
       content_type: CONTENT_TYPE_CSV,
       csv_contains_formulas: this.csvContainsFormulas && !escapeFormulaValues,
-      csv_rows: this.csvRowCount,
       max_size_reached: this.maxSizeReached,
+      metrics: {
+        csv: { rows: this.csvRowCount },
+      },
       warnings,
     };
   }

diff --git a/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts b/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts
@@ -49,8 +49,9 @@ export const runTaskFnFactory: RunTaskFnFactory<RunTaskFn<TaskPayloadPNG>> =
           });
         }),
         tap(({ buffer }) => stream.write(buffer)),
-        map(({ warnings }) => ({
+        map(({ metrics, warnings }) => ({
           content_type: 'image/png',
+          metrics: { png: metrics },
           warnings,
         })),
         tap({ error: (error) => jobLogger.error(error) }),

diff --git a/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.ts b/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.ts
@@ -50,8 +50,9 @@ export const runTaskFnFactory: RunTaskFnFactory<RunTaskFn<TaskPayloadPNGV2>> =
           });
         }),
         tap(({ buffer }) => stream.write(buffer)),
-        map(({ warnings }) => ({
+        map(({ metrics, warnings }) => ({
           content_type: 'image/png',
+          metrics: { png: metrics },
           warnings,
         })),
         tap({ error: (error) => jobLogger.error(error) }),

diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts
@@ -65,8 +65,9 @@ export const runTaskFnFactory: RunTaskFnFactory<RunTaskFn<TaskPayloadPDF>> =
             stream.write(buffer);
           }
         }),
-        map(({ warnings }) => ({
+        map(({ metrics, warnings }) => ({
           content_type: 'application/pdf',
+          metrics: { pdf: metrics },
           warnings,
         })),
         catchError((err) => {

diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf/lib/generate_pdf.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf/lib/generate_pdf.ts
@@ -7,8 +7,9 @@
 
 import { groupBy } from 'lodash';
 import * as Rx from 'rxjs';
-import { mergeMap } from 'rxjs/operators';
+import { mergeMap, tap } from 'rxjs/operators';
 import { ScreenshotResult } from '../../../../../screenshotting/server';
+import type { PdfMetrics } from '../../../../common/types';
 import { ReportingCore } from '../../../';
 import { LevelLogger } from '../../../lib';
 import { ScreenshotOptions } from '../../../types';
@@ -25,25 +26,32 @@ const getTimeRange = (urlScreenshots: ScreenshotResult['results']) => {
   return null;
 };
 
+interface PdfResult {
+  buffer: Buffer | null;
+  metrics?: PdfMetrics;
+  warnings: string[];
+}
+
 export function generatePdfObservable(
   reporting: ReportingCore,
   logger: LevelLogger,
   title: string,
   options: ScreenshotOptions,
   logo?: string
-): Rx.Observable<{ buffer: Buffer | null; warnings: string[] }> {
+): Rx.Observable<PdfResult> {
   const tracker = getTracker();
   tracker.startScreenshots();
 
   return reporting.getScreenshots(options).pipe(
-    mergeMap(async ({ layout, metrics$, results }) => {
-      metrics$.subscribe(({ cpu, memory }) => {
-        tracker.setCpuUsage(cpu);
-        tracker.setMemoryUsage(memory);
-      });
+    tap(({ metrics }) => {
+      if (metrics) {
+        tracker.setCpuUsage(metrics.cpu);
+        tracker.setMemoryUsage(metrics.memory);
+      }
       tracker.endScreenshots();
       tracker.startSetup();
-
+    }),
+    mergeMap(async ({ layout, metrics, results }) => {
       const pdfOutput = new PdfMaker(layout, logo);
       if (title) {
         const timeRange = getTimeRange(results);
@@ -89,6 +97,10 @@ export function generatePdfObservable(
 
       return {
         buffer,
+        metrics: {
+          ...metrics,
+          pages: pdfOutput.getPageCount(),
+        },
         warnings: results.reduce((found, current) => {
           if (current.error) {
             found.push(current.error.message);

diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf_v2/execute_job.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf_v2/execute_job.ts
@@ -57,15 +57,16 @@ export const runTaskFnFactory: RunTaskFnFactory<RunTaskFn<TaskPayloadPDFV2>> =
             logo
           );
         }),
-        tap(({ buffer, warnings }) => {
+        tap(({ buffer }) => {
           apmGeneratePdf?.end();
 
           if (buffer) {
             stream.write(buffer);
           }
         }),
-        map(({ warnings }) => ({
+        map(({ metrics, warnings }) => ({
           content_type: 'application/pdf',
+          metrics: { pdf: metrics },
           warnings,
         })),
         catchError((err) => {