Skip to content

Commit

Permalink
fix(benchmark): make it fair (#2929)
Browse files Browse the repository at this point in the history
* fix(benchmark): make it fair

* chore: update benchmark results
  • Loading branch information
tsctx authored Mar 7, 2024
1 parent 7298524 commit 6a4c8fc
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 152 deletions.
53 changes: 53 additions & 0 deletions benchmarks/_util/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
'use strict'

const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100

function makeParallelRequests (cb) {
const promises = new Array(parallelRequests)
for (let i = 0; i < parallelRequests; ++i) {
promises[i] = new Promise(cb)
}
return Promise.all(promises)
}

function printResults (results) {
// Sort results by least performant first, then compare relative performances and also printing padding
let last

const rows = Object.entries(results)
// If any failed, put on the top of the list, otherwise order by mean, ascending
.sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
.map(([name, result]) => {
if (!result.success) {
return {
Tests: name,
Samples: result.size,
Result: 'Errored',
Tolerance: 'N/A',
'Difference with Slowest': 'N/A'
}
}

// Calculate throughput and relative performance
const { size, mean, standardError } = result
const relative = last !== 0 ? (last / mean - 1) * 100 : 0

// Save the slowest for relative comparison
if (typeof last === 'undefined') {
last = mean
}

return {
Tests: name,
Samples: size,
Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
Tolerance: ${((standardError / mean) * 100).toFixed(2)} %`,
'Difference with slowest':
relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
}
})

return console.table(rows)
}

module.exports = { makeParallelRequests, printResults }
104 changes: 37 additions & 67 deletions benchmarks/benchmark.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,20 @@ const { isMainThread } = require('node:worker_threads')

const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..')

const { makeParallelRequests, printResults } = require('./_util')

let nodeFetch
const axios = require('axios')
let superagent
let got

const util = require('node:util')
const _request = require('request')
const request = util.promisify(_request)
const { promisify } = require('node:util')
const request = promisify(require('request'))

const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1
const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3
const connections = parseInt(process.env.CONNECTIONS, 10) || 50
const pipelining = parseInt(process.env.PIPELINING, 10) || 10
const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100
const headersTimeout = parseInt(process.env.HEADERS_TIMEOUT, 10) || 0
const bodyTimeout = parseInt(process.env.BODY_TIMEOUT, 10) || 0
const dest = {}
Expand Down Expand Up @@ -136,53 +136,6 @@ class SimpleRequest {
}
}

function makeParallelRequests (cb) {
const promises = new Array(parallelRequests)
for (let i = 0; i < parallelRequests; ++i) {
promises[i] = new Promise(cb)
}
return Promise.all(promises)
}

function printResults (results) {
// Sort results by least performant first, then compare relative performances and also printing padding
let last

const rows = Object.entries(results)
// If any failed, put on the top of the list, otherwise order by mean, ascending
.sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
.map(([name, result]) => {
if (!result.success) {
return {
Tests: name,
Samples: result.size,
Result: 'Errored',
Tolerance: 'N/A',
'Difference with Slowest': 'N/A'
}
}

// Calculate throughput and relative performance
const { size, mean, standardError } = result
const relative = last !== 0 ? (last / mean - 1) * 100 : 0

// Save the slowest for relative comparison
if (typeof last === 'undefined') {
last = mean
}

return {
Tests: name,
Samples: size,
Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
Tolerance: ${((standardError / mean) * 100).toFixed(2)} %`,
'Difference with slowest': relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
}
})

return console.table(rows)
}

const experiments = {
'http - no keepalive' () {
return makeParallelRequests(resolve => {
Expand Down Expand Up @@ -217,8 +170,8 @@ const experiments = {
'undici - pipeline' () {
return makeParallelRequests(resolve => {
dispatcher
.pipeline(undiciOptions, data => {
return data.body
.pipeline(undiciOptions, ({ body }) => {
return body
})
.end()
.pipe(
Expand Down Expand Up @@ -288,9 +241,15 @@ if (process.env.PORT) {
})
}

const axiosOptions = {
url: dest.url,
method: 'GET',
responseType: 'stream',
httpAgent: axiosAgent
}
experiments.axios = () => {
return makeParallelRequests(resolve => {
axios.get(dest.url, { responseType: 'stream', httpAgent: axiosAgent }).then(res => {
axios.request(axiosOptions).then(res => {
res.data.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
Expand All @@ -300,26 +259,37 @@ if (process.env.PORT) {
})
}

const gotOptions = {
url: dest.url,
method: 'GET',
agent: {
http: gotAgent
},
// avoid body processing
isStream: true
}
experiments.got = () => {
return makeParallelRequests(resolve => {
got.get(dest.url, { agent: { http: gotAgent } }).then(res => {
res.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
}).catch(console.log)
got(gotOptions).pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
})
}

const requestOptions = {
url: dest.url,
method: 'GET',
agent: requestAgent,
// avoid body toString
encoding: null
}
experiments.request = () => {
return makeParallelRequests(resolve => {
request(dest.url, { agent: requestAgent }).then(res => {
res.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
request(requestOptions).then(() => {
// already body consumed
resolve()
}).catch(console.log)
})
}
Expand Down
88 changes: 21 additions & 67 deletions benchmarks/post-benchmark.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,23 @@ const os = require('node:os')
const path = require('node:path')
const { Writable, Readable, pipeline } = require('node:stream')
const { isMainThread } = require('node:worker_threads')

const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..')

const { makeParallelRequests, printResults } = require('./_util')

let nodeFetch
const axios = require('axios')
let superagent
let got

const util = require('node:util')
const _request = require('request')
const request = util.promisify(_request)
const { promisify } = require('node:util')
const request = promisify(require('request'))

const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1
const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3
const connections = parseInt(process.env.CONNECTIONS, 10) || 50
const pipelining = parseInt(process.env.PIPELINING, 10) || 10
const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100
const headersTimeout = parseInt(process.env.HEADERS_TIMEOUT, 10) || 0
const bodyTimeout = parseInt(process.env.BODY_TIMEOUT, 10) || 0
const dest = {}
Expand Down Expand Up @@ -147,53 +148,6 @@ class SimpleRequest {
}
}

function makeParallelRequests (cb) {
const promises = new Array(parallelRequests)
for (let i = 0; i < parallelRequests; ++i) {
promises[i] = new Promise(cb)
}
return Promise.all(promises)
}

function printResults (results) {
// Sort results by least performant first, then compare relative performances and also printing padding
let last

const rows = Object.entries(results)
// If any failed, put on the top of the list, otherwise order by mean, ascending
.sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
.map(([name, result]) => {
if (!result.success) {
return {
Tests: name,
Samples: result.size,
Result: 'Errored',
Tolerance: 'N/A',
'Difference with Slowest': 'N/A'
}
}

// Calculate throughput and relative performance
const { size, mean, standardError } = result
const relative = last !== 0 ? (last / mean - 1) * 100 : 0

// Save the slowest for relative comparison
if (typeof last === 'undefined') {
last = mean
}

return {
Tests: name,
Samples: size,
Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
Tolerance: ${((standardError / mean) * 100).toFixed(2)} %`,
'Difference with slowest': relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
}
})

return console.table(rows)
}

const experiments = {
'http - no keepalive' () {
return makeParallelRequests(resolve => {
Expand Down Expand Up @@ -236,8 +190,8 @@ const experiments = {
this.push(null)
}
}),
dispatcher.pipeline(undiciOptions, data => {
return data.body
dispatcher.pipeline(undiciOptions, ({ body }) => {
return body
}),
new Writable({
write (chunk, encoding, callback) {
Expand Down Expand Up @@ -341,22 +295,23 @@ if (process.env.PORT) {
}

const gotOptions = {
url: dest.url,
method: 'POST',
headers,
agent: {
http: gotAgent
},
// avoid body processing
isStream: true,
body: data
}
experiments.got = () => {
return makeParallelRequests(resolve => {
got(dest.url, gotOptions).then(res => {
res.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
}).catch(console.log)
got(gotOptions).pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
})
}

Expand All @@ -365,16 +320,15 @@ if (process.env.PORT) {
method: 'POST',
headers,
agent: requestAgent,
body: data
body: data,
// avoid body toString
encoding: null
}
experiments.request = () => {
return makeParallelRequests(resolve => {
request(requestOptions).then(res => {
res.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
request(requestOptions).then(() => {
// already body consumed
resolve()
}).catch(console.log)
})
}
Expand Down
10 changes: 8 additions & 2 deletions benchmarks/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,16 @@ if (cluster.isPrimary) {
}
} else {
const buf = Buffer.alloc(64 * 1024, '_')

const headers = {
'Content-Length': `${buf.byteLength}`,
'Content-Type': 'text/plain; charset=UTF-8'
}
let i = 0
const server = createServer((req, res) => {
const server = createServer((_req, res) => {
i++
setTimeout(function () {
setTimeout(() => {
res.writeHead(200, headers)
res.end(buf)
}, timeout)
}).listen(port)
Expand Down
Loading

0 comments on commit 6a4c8fc

Please sign in to comment.