Cleanup

packages/cli/package.json

@@ -24,7 +24,7 @@
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
-    "build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs && node dist/benchmark/scripts/document-suites.js",
+    "build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs && node dist/benchmark/scripts/list-suites.js",
     "buildAndDev": "pnpm run build && pnpm run dev",
     "dev": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon\"",
     "dev:worker": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon worker\"",

packages/cli/src/benchmark/benchmark.md

@@ -16,12 +16,10 @@ To create a benchmark, @TODO
 
 <!-- BENCHMARK_SUITES_LIST -->
 
-### 1. Production workflow with authless webhook node
+### 001 - Production workflow with authless webhook node
 
-Suite file: `webhook.tasks.js`
-
-(1.1) using "Respond immediately" mode
-(1.2) using "When last node finishes" mode
-(1.3) using "Respond to Webhook" node mode
+- [using "Respond immediately" mode](./suites/workflows/001-1.json)
+- [using "When last node finishes" mode](./suites/workflows/001-2.json)
+- [using "Respond to Webhook" node mode](./suites/workflows/001-3.json)
 
 <!-- /BENCHMARK_SUITES_LIST -->

packages/cli/src/benchmark/lib/hooks.ts

@@ -15,10 +15,9 @@ import type { WorkflowRequest } from '@/workflows/workflow.request';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 import { Logger } from '@/Logger';
 
-/**
- * Create a temp `.n8n` dir for encryption key, sqlite DB, etc.
- */
-function n8nDir() {
+const logger = Container.get(Logger);
+
+function tempN8nDir() {
 	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
 
 	mkdirSync(baseDirPath, { recursive: true });
@@ -41,14 +40,11 @@ function n8nDir() {
 	instanceSettings.n8nFolder = _n8nDir;
 	Container.set(InstanceSettings, instanceSettings);
 
-	Container.get(Logger).info(`Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
+	logger.info(`[Benchmarking] Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
 }
 
-/**
- * Load into DB and activate in memory all workflows to use in benchmarks.
- */
-async function prepareWorkflows(owner: User) {
-	const files = await glob('workflows/*.json', {
+async function loadWorkflows(owner: User) {
+	const files = await glob('suites/workflows/*.json', {
 		cwd: path.join('dist', 'benchmark'),
 		absolute: true,
 	});
@@ -64,41 +60,25 @@ async function prepareWorkflows(owner: User) {
 		// @ts-ignore @TODO Fix typing
 		await Container.get(WorkflowsController).create({ body: workflow, user: owner });
 	}
-
-	await Container.get(ActiveWorkflowRunner).init();
 }
 
 let main: Start;
 
-/**
- * Start the main n8n process to use in benchmarks.
- */
-async function mainProcess() {
-	const args: string[] = [];
-	const _config = new Config({ root: __dirname });
+export async function globalSetup() {
+	tempN8nDir();
 
-	main = new Start(args, _config);
+	main = new Start([], new Config({ root: __dirname }));
 
 	await main.init();
 	await main.run();
-}
-
-/**
- * Setup to run before once all benchmarks.
- */
-export async function globalSetup() {
-	n8nDir();
-
-	await mainProcess();
 
 	const owner = await Container.get(UserRepository).createTestOwner();
 
-	await prepareWorkflows(owner); // @TODO: Load all here or as part of each benchmark's `beforeEach`?
+	await loadWorkflows(owner);
+
+	await Container.get(ActiveWorkflowRunner).init();
 }
 
-/**
- * Teardown to run after all benchmarks.
- */
 export async function globalTeardown() {
 	await main.stopProcess();
 }

packages/cli/src/benchmark/lib/index.ts

@@ -1 +1,108 @@
-export { suite, collectSuites, task, beforeEach, afterEach } from './suites';
+import 'reflect-metadata';
+import path from 'node:path';
+import type Bench from 'tinybench';
+import { assert } from 'n8n-workflow';
+import glob from 'fast-glob';
+import callsites from 'callsites';
+import type { Suites, Task, Callback } from './types';
+import { DuplicateHookError } from './errors/duplicate-hook.error';
+import { DuplicateSuiteError } from './errors/duplicate-suite.error';
+
+const suites: Suites = {};
+
+export async function collectSuites() {
+	const files = await glob('**/*.suite.js', {
+		cwd: path.join('dist', 'benchmark'),
+		absolute: true,
+	});
+
+	for (const f of files) {
+		await import(f);
+	}
+
+	return suites;
+}
+
+export function registerSuites(bench: Bench) {
+	for (const { hooks, tasks } of Object.values(suites)) {
+		/**
+		 * In tinybench, `beforeAll` and `afterAll` refer to all iterations of
+		 * a single task, while `beforeEach` and `afterEach` refer to each iteration.
+		 *
+		 * In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
+		 * while `beforeEach` and `afterEach` refer to each individual test.
+		 *
+		 * We rename tinybench's hooks to prevent confusion from this difference.
+		 */
+		const options: Record<string, Callback> = {};
+
+		if (hooks.beforeEach) options.beforeAll = hooks.beforeEach;
+		if (hooks.afterEach) options.afterAll = hooks.afterEach;
+
+		for (const t of tasks) {
+			bench.add(t.name, t.operation, options);
+		}
+	}
+}
+
+function suiteFilePath() {
+	const filePath = callsites()
+		.map((site) => site.getFileName())
+		.filter((site): site is string => site !== null)
+		.find((site) => site.endsWith('.suite.js'));
+
+	assert(filePath !== undefined);
+
+	return filePath;
+}
+
+export function suite(suiteName: string, suiteFn: () => void) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]) throw new DuplicateSuiteError(filePath);
+
+	suites[filePath] = { name: suiteName, hooks: {}, tasks: [] };
+
+	suiteFn();
+}
+
+export function task(taskName: string, operation: Task['operation']) {
+	const filePath = suiteFilePath();
+
+	suites[filePath].tasks.push({
+		name: suites[filePath].name + ' ' + taskName,
+		operation,
+	});
+}
+
+// @TODO: Rename next two utils to dismbiguate?
+
+/**
+ * Setup step to run once before all iterations of each benchmarking task in a suite.
+ */
+export function beforeEach(fn: Callback) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]?.hooks.beforeEach) {
+		throw new DuplicateHookError('beforeEach', filePath);
+	}
+
+	suites[filePath].hooks.beforeEach = fn;
+}
+
+/**
+ * Teardown step to run once after all iterations of each benchmarking task in a suite.
+ */
+export function afterEach(fn: Callback) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]?.hooks.afterEach) {
+		throw new DuplicateHookError('afterEach', filePath);
+	}
+
+	suites[filePath].hooks.afterEach = fn;
+}
+
+export const BACKEND_BASE_URL = 'http://localhost:5678';
+
+export type { Suites } from './types';

packages/cli/src/benchmark/lib/types.ts

@@ -1,8 +1,8 @@
 /**
- * Benchmarking suites, i.e. `*.tasks.ts` files containing benchmarking tasks.
+ * Benchmarking suites, i.e. `*.suite.ts` files containing benchmarking tasks.
  */
 export type Suites = {
-	[suiteFilepath: string]: {
+	[suiteFilePath: string]: {
 		name: string;
 		hooks: {
 			beforeEach?: Callback;
@@ -16,7 +16,7 @@ export type Suites = {
  * A benchmarking task, i.e. a single operation whose performance to measure.
  */
 export type Task = {
-	description: string;
+	name: string;
 	operation: Callback;
 };
 

packages/cli/src/benchmark/main.ts

@@ -1,10 +1,10 @@
 import 'reflect-metadata';
-import * as hooks from './lib/hooks';
-import { collectSuites, registerSuites } from './lib/suites';
+import Container from 'typedi';
 import config from '@/config';
-import { UnsupportedDatabaseError } from './lib/errors/unsupported-database.error';
 import { Logger } from '@/Logger';
-import Container from 'typedi';
+import * as hooks from './lib/hooks';
+import { collectSuites, registerSuites } from './lib';
+import { UnsupportedDatabaseError } from './lib/errors/unsupported-db.error';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';
@@ -23,11 +23,11 @@ async function main() {
 	const logger = Container.get(Logger);
 
 	if (count === 0) {
-		logger.info('No benchmarking suites found. Exiting...');
+		logger.info('[Benchmarking] Found no suites. Exiting...');
 		return;
 	}
 
-	logger.info(`Running ${count} benchmarking ${count === 1 ? 'suite' : 'suites'}...`);
+	logger.info(`[Benchmarking] Running ${count} ${count === 1 ? 'suite' : 'suites'}...`);
 
 	await hooks.globalSetup();