ci(NODE-5899): remove unneeded stats from benchmark results (#646)

.evergreen/config.yml

@@ -235,7 +235,8 @@ tasks:
           ITERATIONS: 1000
       - command: perf.send
         params:
-          file: src/test/bench/etc/resultsCollected.json
+          file: src/test/bench/etc/resultsCollectedMeans.json
+
   - name: run-spec-benchmarks-node-18
     commands:
       - func: fetch source

test/bench/README.md

@@ -16,28 +16,43 @@ microbenchmarking specification.
 
 Granular tests can be run from the repository root by running:
 
-```
+```bash
 WARMUP=<warmup iterations> ITERATIONS=<measured iterations> npm run check:granular-bench
 ```
 
 This will build the granular tests and run them with `test/bench/etc/run_granular_benchmarks.js`. The `WARMUP` and `ITERATIONS` environment variables can be optionally provided to configure how these granular benchmarks
 are run. `WARMUP` changes the number of iterations run before results are collected to give v8's
 optimizing compiler time to reach steady state. `ITERATIONS` changes the number of iterations that
-are measured and use to calculate summary statistics.
+are measured and used to calculate summary statistics. Note also that the test can be configured to
+make use of the local copy of bson when testing performance changes locally by setting the `LIBRARY`
+variable to the root directory of the bson library to be tested.
+
+```bash
+WARMUP=100 ITERATIONS=1000 LIBRARY=$(pwd) npm run check:granular-bench
+```
+When the `LIBRARY` environment variable is unset, the benchmark clones and runs against the main
+branch of this repository.
 
-When this script is complete, results will be output to `test/bench/etc/resultsCollected.json`. These results will
+When the script is complete, results will be output to `test/bench/etc/resultsCollectedMeans.json`. These results will
 be in a format compatible with evergreen's perf.send command. To convert these results to CSV, run
 the following command from the repository root:
 
-```
-./test/bench/etc/convertToCSV.js < test/bench/etc/resultsCollected.json > resultsCollected.csv
+```bash
+./test/bench/etc/convertToCSV.js < test/bench/etc/resultsCollectedMeans.json > resultsCollected.csv
 ```
 
 Spec tests can be run from the repository root by running:
 
-```
+``` bash
 npm run check:spec-bench
 ```
 
 This will run the spec benchmarks in `test/bench/spec/bsonBench.ts` which also makes use of the
-`bson-bench` library. Results will be written to `bsonBench`.
+`bson-bench` library. Results will be written to `bsonBench`. The warmup and iterations are not
+configurable as these are determined by the common driver benchmarking specification, but similar
+to the granular benchmarks, the spec benchmarks can be run against the local copy of bson by setting
+the `LIBRARY` environment variable appropriately.
+
+```bash
+LIBRARY=$(pwd) npm run check:spec-bench
+```

test/bench/etc/run_granular_benchmarks.js

@@ -64,7 +64,7 @@ const DOCUMENT_ROOT = path.resolve(`${__dirname}/../documents`);
 
   console.log('No duplcate testName:Option pairs found. Now merging files...');
 
-  const resultFile = `${__dirname}/resultsCollected.json`;
+  const meansFile = `${__dirname}/resultsCollectedMeans.json`;
   // Iterate over all result files and merge into one file
   const collectedResults = [];
   for (const resultPath of resultPaths) {
@@ -74,7 +74,13 @@ const DOCUMENT_ROOT = path.resolve(`${__dirname}/../documents`);
     }
   }
 
-  await fs.writeFile(resultFile, JSON.stringify(collectedResults));
+  const means = collectedResults.map(result => {
+    const rv = { ...result };
+    rv.metrics = rv.metrics.filter(metric => metric.type === 'MEAN');
+    return rv;
+  });
 
-  console.log(`Collected results in ${resultFile}`);
+  await fs.writeFile(meansFile, JSON.stringify(means));
+
+  console.log(`Means in ${meansFile}`);
 })();

test/bench/granular/common.ts

@@ -6,8 +6,6 @@ export const DOCUMENT_ROOT = `${__dirname}/../../documents`;
 export const OPERATIONS: ('serialize' | 'deserialize')[] = ['serialize', 'deserialize'];
 export const BOOL = [true, false];
 
-export const LIBRARY_SPEC = 'bson#main';
-
 export const isDeserialize = (s: string) => s === 'deserialize';
 export async function getTestDocs(type: string) {
   const docs = ['singleFieldDocument', 'singleElementArray'].map(testType =>
@@ -33,19 +31,24 @@ export async function runSuiteAndWriteResults(suite: Suite) {
   await suite.writeResults(`${targetDirectory}/${suite.name.toLowerCase()}Results.json`);
 }
 
-export function readEnvVars(): { warmup: number; iterations: number } {
+export function readEnvVars(): { warmup: number; iterations: number; library: string } {
   const envWarmup = Number(process.env.WARMUP);
   const envIterations = Number(process.env.ITERATIONS);
+  const libraryPath = process.env.LIBRARY;
   const rv = {
     warmup: Number.isSafeInteger(envWarmup) && envWarmup > 0 ? envWarmup : 100_000,
-    iterations: Number.isSafeInteger(envIterations) && envIterations > 0 ? envIterations : 10_000
+    iterations: Number.isSafeInteger(envIterations) && envIterations > 0 ? envIterations : 10_000,
+    library: libraryPath ? `bson:${libraryPath}` : 'bson#main'
   };
 
-  console.log(`warmup iterations: ${rv.warmup}\nmeasured iterations: ${rv.iterations}`);
+  console.log(
+    `warmup iterations: ${rv.warmup}\nmeasured iterations: ${rv.iterations}\nlibrary: ${rv.library}`
+  );
 
   return rv;
 }
 
 const envVars = readEnvVars();
 export const ITERATIONS = envVars.iterations;
 export const WARMUP = envVars.warmup;
+export const LIBRARY_SPEC = envVars.library;

test/bench/spec/bsonBench.ts

@@ -1,14 +1,14 @@
 import { Suite } from 'dbx-js-tools/packages/bson-bench';
 import { join, resolve } from 'path';
 import { writeFile } from 'fs/promises';
+import { LIBRARY_SPEC } from '../granular/common';
 
 const suite = new Suite('bson micro benchmarks');
-const LIBRARY = `bson#main`;
 const DOCUMENT_ROOT = resolve(`${__dirname}/../../documents`);
 // Add flat bson encoding
 suite.task({
   documentPath: join(DOCUMENT_ROOT, 'flat_bson.json'),
-  library: LIBRARY,
+  library: LIBRARY_SPEC,
   warmup: 1000,
   iterations: 10_000,
   operation: 'serialize',
@@ -18,7 +18,7 @@ suite.task({
 // Add flat bson decoding
 suite.task({
   documentPath: join(DOCUMENT_ROOT, 'flat_bson.json'),
-  library: LIBRARY,
+  library: LIBRARY_SPEC,
   warmup: 1000,
   iterations: 10_000,
   operation: 'deserialize',
@@ -28,7 +28,7 @@ suite.task({
 // Add deep bson encoding
 suite.task({
   documentPath: join(DOCUMENT_ROOT, 'deep_bson.json'),
-  library: LIBRARY,
+  library: LIBRARY_SPEC,
   warmup: 1000,
   iterations: 10_000,
   operation: 'serialize',
@@ -38,7 +38,7 @@ suite.task({
 // Add deep bson decoding
 suite.task({
   documentPath: join(DOCUMENT_ROOT, 'deep_bson.json'),
-  library: LIBRARY,
+  library: LIBRARY_SPEC,
   warmup: 1000,
   iterations: 10_000,
   operation: 'deserialize',
@@ -48,7 +48,7 @@ suite.task({
 // Add full bson encoding
 suite.task({
   documentPath: join(DOCUMENT_ROOT, 'full_bson.json'),
-  library: LIBRARY,
+  library: LIBRARY_SPEC,
   warmup: 1000,
   iterations: 10_000,
   operation: 'serialize',
@@ -58,7 +58,7 @@ suite.task({
 // Add full bson decoding
 suite.task({
   documentPath: join(DOCUMENT_ROOT, 'full_bson.json'),
-  library: LIBRARY,
+  library: LIBRARY_SPEC,
   warmup: 1000,
   iterations: 10_000,
   operation: 'deserialize',
@@ -67,7 +67,11 @@ suite.task({
 
 suite.run().then(
   () => {
-    const results = suite.results;
+    const results = suite.results.map(result => {
+      const rv = { ...result };
+      rv.metrics = rv.metrics.filter(metric => metric.type === 'MEAN');
+      return rv;
+    });
     // calculte BSONBench composite score
     const bsonBenchComposite =
       results.reduce((prev, result) => {