From f013a5d0bf9148d884df9eeaaa41afed37f6fb40 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Mon, 26 Jul 2021 15:16:31 +0800 Subject: [PATCH 01/29] Remove deprecated functions. --- CHANGELOG.md | 8 + benches/benchmarks/compare_functions.rs | 49 +- benches/benchmarks/external_process.rs | 1 - benches/benchmarks/iter_with_large_drop.rs | 28 +- benches/benchmarks/iter_with_large_setup.rs | 36 +- plot/src/data.rs | 2 - src/benchmark.rs | 554 +------------------- src/lib.rs | 182 +------ tests/criterion_tests.rs | 244 ++++----- 9 files changed, 137 insertions(+), 967 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 985e3cf5e..f51d3f264 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [0.4.0] - IN DEVELOPMENT + +### Removed +- The `Criterion::can_plot` function has been removed. +- The `Criterion::bench_function_over_inputs` function has been removed. +- The `Criterion::bench_functions` function has been removed. +- The `Criterion::bench` function has been removed. + ## [Unreleased] ### Fixed - Corrected `Criterion.toml` in the book. diff --git a/benches/benchmarks/compare_functions.rs b/benches/benchmarks/compare_functions.rs index ce44180f9..d9af837a8 100644 --- a/benches/benchmarks/compare_functions.rs +++ b/benches/benchmarks/compare_functions.rs @@ -1,6 +1,4 @@ -#![allow(deprecated)] - -use criterion::{criterion_group, BenchmarkId, Criterion, Fun, ParameterizedBenchmark}; +use criterion::{criterion_group, BenchmarkId, Criterion}; fn fibonacci_slow(n: u64) -> u64 { match n { @@ -27,23 +25,10 @@ fn fibonacci_fast(n: u64) -> u64 { } fn compare_fibonaccis(c: &mut Criterion) { - let fib_slow = Fun::new("Recursive", |b, i| b.iter(|| fibonacci_slow(*i))); - let fib_fast = Fun::new("Iterative", |b, i| b.iter(|| fibonacci_fast(*i))); - - let functions = vec![fib_slow, fib_fast]; + let mut group = c.benchmark_group("Fibonacci"); - c.bench_functions("Fibonacci", functions, 20); -} -fn compare_fibonaccis_builder(c: &mut Criterion) { - c.bench( - "Fibonacci2", - ParameterizedBenchmark::new( - "Recursive", - |b, i| b.iter(|| fibonacci_slow(*i)), - vec![20u64, 21u64], - ) - .with_function("Iterative", |b, i| b.iter(|| fibonacci_fast(*i))), - ); + group.bench_with_input("Recursive", &20, |b, i| b.iter(|| fibonacci_slow(*i))); + group.bench_with_input("Iterative", &20, |b, i| b.iter(|| fibonacci_fast(*i))); } fn compare_fibonaccis_group(c: &mut Criterion) { let mut group = c.benchmark_group("Fibonacci3"); @@ -58,28 +43,4 @@ fn compare_fibonaccis_group(c: &mut Criterion) { group.finish() } -fn compare_looped(c: &mut Criterion) { - use criterion::black_box; - - c.bench( - "small", - ParameterizedBenchmark::new("unlooped", |b, i| b.iter(|| i + 10), vec![10]).with_function( - "looped", - |b, i| { - b.iter(|| { - for _ in 0..10_000 { - black_box(i + 10); - } - }) - }, - ), - ); -} - -criterion_group!( - fibonaccis, - compare_fibonaccis, - compare_fibonaccis_builder, - compare_fibonaccis_group, - compare_looped -); +criterion_group!(fibonaccis, compare_fibonaccis, compare_fibonaccis_group,); diff --git a/benches/benchmarks/external_process.rs b/benches/benchmarks/external_process.rs index c823df5cd..7667a53b8 100644 --- a/benches/benchmarks/external_process.rs +++ b/benches/benchmarks/external_process.rs @@ -14,7 +14,6 @@ fn create_command() -> Command { command } -#[allow(deprecated)] fn python_fibonacci(c: &mut Criterion) { let has_python3 = Command::new("python3") .arg("--version") diff --git a/benches/benchmarks/iter_with_large_drop.rs b/benches/benchmarks/iter_with_large_drop.rs index ee9a8e932..ee01de057 100644 --- a/benches/benchmarks/iter_with_large_drop.rs +++ b/benches/benchmarks/iter_with_large_drop.rs @@ -1,28 +1,22 @@ -#![allow(deprecated)] - -use criterion::{criterion_group, Benchmark, Criterion, Throughput}; +use criterion::{criterion_group, Criterion, Throughput}; use std::time::Duration; const SIZE: usize = 1024 * 1024; fn large_drop(c: &mut Criterion) { - c.bench( - "iter_with_large_drop", - Benchmark::new("large_drop", |b| { - let v: Vec<_> = (0..SIZE).map(|i| i as u8).collect(); - b.iter_with_large_drop(|| v.clone()); - }) - .throughput(Throughput::Bytes(SIZE as u64)), - ); + let mut group = c.benchmark_group("iter_with_large_drop"); + group.throughput(Throughput::Bytes(SIZE as u64)); + group.bench_function("large_drop", |b| { + let v: Vec<_> = (0..SIZE).map(|i| i as u8).collect(); + b.iter_with_large_drop(|| v.clone()); + }); } fn small_drop(c: &mut Criterion) { - c.bench( - "iter_with_large_drop", - Benchmark::new("small_drop", |b| { - b.iter_with_large_drop(|| SIZE); - }), - ); + let mut group = c.benchmark_group("iter_with_large_drop"); + group.bench_function("small_drop", |b| { + b.iter_with_large_drop(|| SIZE); + }); } fn short_warmup() -> Criterion { diff --git a/benches/benchmarks/iter_with_large_setup.rs b/benches/benchmarks/iter_with_large_setup.rs index 217d27151..01b6e9cd6 100644 --- a/benches/benchmarks/iter_with_large_setup.rs +++ b/benches/benchmarks/iter_with_large_setup.rs @@ -1,32 +1,26 @@ -#![allow(deprecated)] - -use criterion::{criterion_group, Benchmark, Criterion, Throughput}; +use criterion::{criterion_group, Criterion, Throughput}; use std::time::Duration; const SIZE: usize = 1024 * 1024; fn large_setup(c: &mut Criterion) { - c.bench( - "iter_with_large_setup", - Benchmark::new("large_setup", |b| { - // NOTE: iter_with_large_setup is deprecated. Use iter_batched instead. - b.iter_with_large_setup( - || (0..SIZE).map(|i| i as u8).collect::>(), - |v| v.clone(), - ) - }) - .throughput(Throughput::Bytes(SIZE as u64)), - ); + let mut group = c.benchmark_group("iter_with_large_setup"); + group.throughput(Throughput::Bytes(SIZE as u64)); + group.bench_function("large_setup", |b| { + // NOTE: iter_with_large_setup is deprecated. Use iter_batched instead. + b.iter_with_large_setup( + || (0..SIZE).map(|i| i as u8).collect::>(), + |v| v.clone(), + ) + }); } fn small_setup(c: &mut Criterion) { - c.bench( - "iter_with_large_setup", - Benchmark::new("small_setup", |b| { - // NOTE: iter_with_large_setup is deprecated. Use iter_batched instead. - b.iter_with_large_setup(|| SIZE, |size| size) - }), - ); + let mut group = c.benchmark_group("iter_with_large_setup"); + group.bench_function("small_setup", |b| { + // NOTE: iter_with_large_setup is deprecated. Use iter_batched instead. + b.iter_with_large_setup(|| SIZE, |size| size) + }); } fn short_warmup() -> Criterion { diff --git a/plot/src/data.rs b/plot/src/data.rs index eb1f73864..20ed3d41d 100644 --- a/plot/src/data.rs +++ b/plot/src/data.rs @@ -1,5 +1,3 @@ -#![allow(deprecated)] - use std::mem; use cast::From as _0; diff --git a/src/benchmark.rs b/src/benchmark.rs index bc7f6109e..38acad36a 100644 --- a/src/benchmark.rs +++ b/src/benchmark.rs @@ -1,14 +1,4 @@ -#![allow(deprecated)] - -use crate::analysis; -use crate::connection::OutgoingMessage; -use crate::measurement::{Measurement, WallTime}; -use crate::report::{BenchmarkId, Report, ReportContext}; -use crate::routine::{Function, Routine}; -use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput}; -use std::cell::RefCell; -use std::fmt::Debug; -use std::marker::Sized; +use crate::{PlotConfiguration, SamplingMode}; use std::time::Duration; // TODO: Move the benchmark config stuff to a separate module for easier use. @@ -71,545 +61,3 @@ impl PartialBenchmarkConfig { } } } - -pub(crate) struct NamedRoutine { - pub id: String, - pub(crate) f: Box>>, -} - -/// Structure representing a benchmark (or group of benchmarks) -/// which take one parameter. -#[doc(hidden)] -#[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")] -pub struct ParameterizedBenchmark { - config: PartialBenchmarkConfig, - values: Vec, - routines: Vec>, - throughput: Option Throughput>>, -} - -/// Structure representing a benchmark (or group of benchmarks) -/// which takes no parameters. -#[doc(hidden)] -#[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")] -pub struct Benchmark { - config: PartialBenchmarkConfig, - routines: Vec>, - throughput: Option, -} - -/// Common trait for `Benchmark` and `ParameterizedBenchmark`. Not intended to be -/// used outside of Criterion.rs. -#[doc(hidden)] -pub trait BenchmarkDefinition: Sized { - #[doc(hidden)] - fn run(self, group_id: &str, c: &mut Criterion); -} - -macro_rules! benchmark_config { - ($type:tt) => { - /// Changes the size of the sample for this benchmark - /// - /// A bigger sample should yield more accurate results if paired with a sufficiently large - /// measurement time. - /// - /// Sample size must be at least 10. - /// - /// # Panics - /// - /// Panics if n < 10. - pub fn sample_size(mut self, n: usize) -> Self { - assert!(n >= 10); - - self.config.sample_size = Some(n); - self - } - - /// Changes the warm up time for this benchmark - /// - /// # Panics - /// - /// Panics if the input duration is zero - pub fn warm_up_time(mut self, dur: Duration) -> Self { - assert!(dur.to_nanos() > 0); - - self.config.warm_up_time = Some(dur); - self - } - - /// Changes the target measurement time for this benchmark. Criterion will attempt - /// to spend approximately this amount of time measuring the benchmark. - /// With a longer time, the measurement will become more resilient to transitory peak loads - /// caused by external programs. - /// - /// # Panics - /// - /// Panics if the input duration in zero - pub fn measurement_time(mut self, dur: Duration) -> Self { - assert!(dur.to_nanos() > 0); - - self.config.measurement_time = Some(dur); - self - } - - /// Changes the number of resamples for this benchmark - /// - /// Number of resamples to use for the - /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling) - /// - /// A larger number of resamples reduces the random sampling errors, which are inherent to the - /// bootstrap method, but also increases the analysis time. - /// - /// # Panics - /// - /// Panics if the number of resamples is set to zero - pub fn nresamples(mut self, n: usize) -> Self { - assert!(n > 0); - if n <= 1000 { - println!("\nWarning: It is not recommended to reduce nresamples below 1000."); - } - - self.config.nresamples = Some(n); - self - } - - /// Changes the default noise threshold for this benchmark. The noise threshold - /// is used to filter out small changes in performance, even if they are statistically - /// significant. Sometimes benchmarking the same code twice will result in small but - /// statistically significant differences solely because of noise. This provides a way to filter - /// out some of these false positives at the cost of making it harder to detect small changes - /// to the true performance of the benchmark. - /// - /// The default is 0.01, meaning that changes smaller than 1% will be ignored. - /// - /// # Panics - /// - /// Panics if the threshold is set to a negative value - pub fn noise_threshold(mut self, threshold: f64) -> Self { - assert!(threshold >= 0.0); - - self.config.noise_threshold = Some(threshold); - self - } - - /// Changes the default confidence level for this benchmark. The confidence - /// level is the desired probability that the true runtime lies within the estimated - /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is - /// 0.95, meaning that the confidence interval should capture the true value 95% of the time. - /// - /// # Panics - /// - /// Panics if the confidence level is set to a value outside the `(0, 1)` range - pub fn confidence_level(mut self, cl: f64) -> Self { - assert!(cl > 0.0 && cl < 1.0); - if cl < 0.5 { - println!("\nWarning: It is not recommended to reduce confidence level below 0.5."); - } - - self.config.confidence_level = Some(cl); - self - } - - /// Changes the default [significance level](https://en.wikipedia.org/wiki/Statistical_significance) - /// for this benchmark. This is used to perform a - /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if - /// the measurements from this run are different from the measured performance of the last run. - /// The significance level is the desired probability that two measurements of identical code - /// will be considered 'different' due to noise in the measurements. The default value is 0.05, - /// meaning that approximately 5% of identical benchmarks will register as different due to - /// noise. - /// - /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase - /// the statistical robustness against noise, but it also weakens Criterion.rs' ability to - /// detect small but real changes in the performance. By setting the significance level - /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also - /// report more spurious differences. - /// - /// See also the noise threshold setting. - /// - /// # Panics - /// - /// Panics if the significance level is set to a value outside the `(0, 1)` range - pub fn significance_level(mut self, sl: f64) -> Self { - assert!(sl > 0.0 && sl < 1.0); - - self.config.significance_level = Some(sl); - self - } - - /// Changes the plot configuration for this benchmark. - pub fn plot_config(mut self, new_config: PlotConfiguration) -> Self { - self.config.plot_config = new_config; - self - } - - /// Changes the sampling mode for this benchmark. - pub fn sampling_mode(mut self, new_mode: SamplingMode) -> Self { - self.config.sampling_mode = Some(new_mode); - self - } - }; -} - -impl Benchmark -where - M: Measurement + 'static, -{ - benchmark_config!(Benchmark); - - /// Create a new benchmark group and adds the given function to it. - /// - /// # Example - /// - /// ```rust - /// # #[macro_use] extern crate criterion; - /// # use criterion::*; - /// - /// fn bench(c: &mut Criterion) { - /// // One-time setup goes here - /// c.bench( - /// "my_group", - /// Benchmark::new("my_function", |b| b.iter(|| { - /// // Code to benchmark goes here - /// })), - /// ); - /// } - /// - /// criterion_group!(benches, bench); - /// criterion_main!(benches); - /// ``` - pub fn new(id: S, f: F) -> Benchmark - where - S: Into, - F: FnMut(&mut Bencher<'_, M>) + 'static, - { - Benchmark { - config: PartialBenchmarkConfig::default(), - routines: vec![], - throughput: None, - } - .with_function(id, f) - } - - /// Add a function to the benchmark group. - pub fn with_function(mut self, id: S, mut f: F) -> Benchmark - where - S: Into, - F: FnMut(&mut Bencher<'_, M>) + 'static, - { - let routine = NamedRoutine { - id: id.into(), - f: Box::new(RefCell::new(Function::new(move |b, _| f(b)))), - }; - self.routines.push(routine); - self - } - - /// Set the input size for this benchmark group. Used for reporting the - /// throughput. - pub fn throughput(mut self, throughput: Throughput) -> Benchmark { - self.throughput = Some(throughput); - self - } -} - -impl BenchmarkDefinition for Benchmark { - fn run(self, group_id: &str, c: &mut Criterion) { - let report_context = ReportContext { - output_directory: c.output_directory.clone(), - plot_config: self.config.plot_config.clone(), - }; - - let config = self.config.to_complete(&c.config); - let num_routines = self.routines.len(); - - let mut all_ids = vec![]; - let mut any_matched = false; - - if let Some(conn) = &c.connection { - conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id }) - .unwrap(); - } - - for routine in self.routines { - let function_id = if num_routines == 1 && group_id == routine.id { - None - } else { - Some(routine.id) - }; - - let mut id = BenchmarkId::new( - group_id.to_owned(), - function_id, - None, - self.throughput.clone(), - ); - - id.ensure_directory_name_unique(&c.all_directories); - c.all_directories.insert(id.as_directory_name().to_owned()); - id.ensure_title_unique(&c.all_titles); - c.all_titles.insert(id.as_title().to_owned()); - - let do_run = c.filter_matches(id.id()); - any_matched |= do_run; - - execute_benchmark( - do_run, - &id, - c, - &config, - &mut *routine.f.borrow_mut(), - &report_context, - &(), - self.throughput.clone(), - ); - - all_ids.push(id); - } - - if let Some(conn) = &c.connection { - conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id }) - .unwrap(); - conn.serve_value_formatter(c.measurement.formatter()) - .unwrap(); - } - - if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() { - c.report - .summarize(&report_context, &all_ids, c.measurement.formatter()); - } - if any_matched { - c.report.group_separator(); - } - } -} - -impl ParameterizedBenchmark -where - T: Debug + 'static, - M: Measurement + 'static, -{ - benchmark_config!(ParameterizedBenchmark); - - pub(crate) fn with_functions( - functions: Vec>, - parameters: Vec, - ) -> ParameterizedBenchmark { - ParameterizedBenchmark { - config: PartialBenchmarkConfig::default(), - values: parameters, - routines: functions, - throughput: None, - } - } - - /// Create a new parameterized benchmark group and adds the given function - /// to it. - /// The function under test must follow the setup - bench - teardown pattern: - /// - /// # Example - /// - /// ```rust - /// # #[macro_use] extern crate criterion; - /// # use criterion::*; - /// - /// fn bench(c: &mut Criterion) { - /// let parameters = vec![1u64, 2u64, 3u64]; - /// - /// // One-time setup goes here - /// c.bench( - /// "my_group", - /// ParameterizedBenchmark::new( - /// "my_function", - /// |b, param| b.iter(|| { - /// // Code to benchmark using param goes here - /// }), - /// parameters - /// ) - /// ); - /// } - /// - /// criterion_group!(benches, bench); - /// criterion_main!(benches); - /// ``` - pub fn new(id: S, f: F, parameters: I) -> ParameterizedBenchmark - where - S: Into, - F: FnMut(&mut Bencher<'_, M>, &T) + 'static, - I: IntoIterator, - { - ParameterizedBenchmark { - config: PartialBenchmarkConfig::default(), - values: parameters.into_iter().collect(), - routines: vec![], - throughput: None, - } - .with_function(id, f) - } - - /// Add a function to the benchmark group. - pub fn with_function(mut self, id: S, f: F) -> ParameterizedBenchmark - where - S: Into, - F: FnMut(&mut Bencher<'_, M>, &T) + 'static, - { - let routine = NamedRoutine { - id: id.into(), - f: Box::new(RefCell::new(Function::new(f))), - }; - self.routines.push(routine); - self - } - - /// Use the given function to calculate the input size for a given input. - pub fn throughput(mut self, throughput: F) -> ParameterizedBenchmark - where - F: Fn(&T) -> Throughput + 'static, - { - self.throughput = Some(Box::new(throughput)); - self - } -} -impl BenchmarkDefinition for ParameterizedBenchmark -where - T: Debug + 'static, - M: Measurement + 'static, -{ - fn run(self, group_id: &str, c: &mut Criterion) { - let report_context = ReportContext { - output_directory: c.output_directory.clone(), - plot_config: self.config.plot_config.clone(), - }; - - let config = self.config.to_complete(&c.config); - let num_parameters = self.values.len(); - let num_routines = self.routines.len(); - - let mut all_ids = vec![]; - let mut any_matched = false; - - if let Some(conn) = &c.connection { - conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id }) - .unwrap(); - } - - for routine in self.routines { - for value in &self.values { - let function_id = if num_routines == 1 && group_id == routine.id { - None - } else { - Some(routine.id.clone()) - }; - - let value_str = if num_parameters == 1 { - None - } else { - Some(format!("{:?}", value)) - }; - - let throughput = self.throughput.as_ref().map(|func| func(value)); - let mut id = BenchmarkId::new( - group_id.to_owned(), - function_id, - value_str, - throughput.clone(), - ); - - id.ensure_directory_name_unique(&c.all_directories); - c.all_directories.insert(id.as_directory_name().to_owned()); - id.ensure_title_unique(&c.all_titles); - c.all_titles.insert(id.as_title().to_owned()); - - let do_run = c.filter_matches(id.id()); - any_matched |= do_run; - - execute_benchmark( - do_run, - &id, - c, - &config, - &mut *routine.f.borrow_mut(), - &report_context, - value, - throughput, - ); - - all_ids.push(id); - } - } - - if let Some(conn) = &c.connection { - conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id }) - .unwrap(); - conn.serve_value_formatter(c.measurement.formatter()) - .unwrap(); - } - - if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() { - c.report - .summarize(&report_context, &all_ids, c.measurement.formatter()); - } - if any_matched { - c.report.group_separator(); - } - } -} - -#[cfg_attr(feature = "cargo-clippy", allow(clippy::too_many_arguments))] -fn execute_benchmark( - do_run: bool, - id: &BenchmarkId, - c: &Criterion, - config: &BenchmarkConfig, - routine: &mut dyn Routine, - report_context: &ReportContext, - parameter: &T, - throughput: Option, -) where - T: Debug, - M: Measurement, -{ - match c.mode { - Mode::Benchmark => { - if let Some(conn) = &c.connection { - if do_run { - conn.send(&OutgoingMessage::BeginningBenchmark { id: id.into() }) - .unwrap(); - } else { - conn.send(&OutgoingMessage::SkippingBenchmark { id: id.into() }) - .unwrap(); - } - } - - if do_run { - analysis::common( - id, - routine, - config, - c, - report_context, - parameter, - throughput, - ); - } - } - Mode::List => { - if do_run { - println!("{}: bench", id); - } - } - Mode::Test => { - if do_run { - // In test mode, run the benchmark exactly once, then exit. - c.report.test_start(id, report_context); - routine.test(&c.measurement, parameter); - c.report.test_pass(id, report_context); - } - } - Mode::Profile(duration) => { - if do_run { - routine.profile(&c.measurement, id, c, report_context, duration, parameter); - } - } - } -} diff --git a/src/lib.rs b/src/lib.rs index 98dcf1e64..cdfe1f873 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -76,9 +76,6 @@ use std::cell::RefCell; use std::collections::HashSet; use std::default::Default; use std::env; -use std::fmt; -use std::iter::IntoIterator; -use std::marker::PhantomData; use std::net::TcpStream; use std::path::{Path, PathBuf}; use std::process::Command; @@ -88,7 +85,6 @@ use std::time::Duration; use criterion_plot::{Version, VersionError}; use crate::benchmark::BenchmarkConfig; -use crate::benchmark::NamedRoutine; use crate::connection::Connection; use crate::connection::OutgoingMessage; use crate::csv_report::FileCsvReport; @@ -97,13 +93,10 @@ use crate::measurement::{Measurement, WallTime}; use crate::plot::{Gnuplot, Plotter, PlottersBackend}; use crate::profiler::{ExternalProfiler, Profiler}; use crate::report::{BencherReport, CliReport, Report, ReportContext, Reports}; -use crate::routine::Function; #[cfg(feature = "async")] pub use crate::bencher::AsyncBencher; pub use crate::bencher::Bencher; -#[allow(deprecated)] -pub use crate::benchmark::{Benchmark, BenchmarkDefinition, ParameterizedBenchmark}; pub use crate::benchmark_group::{BenchmarkGroup, BenchmarkId}; lazy_static! { @@ -177,36 +170,6 @@ pub fn black_box(dummy: T) -> T { } } -/// Representing a function to benchmark together with a name of that function. -/// Used together with `bench_functions` to represent one out of multiple functions -/// under benchmark. -#[doc(hidden)] -pub struct Fun { - f: NamedRoutine, - _phantom: PhantomData, -} - -impl Fun -where - I: fmt::Debug + 'static, -{ - /// Create a new `Fun` given a name and a closure - pub fn new(name: &str, f: F) -> Fun - where - F: FnMut(&mut Bencher<'_, M>, &I) + 'static, - { - let routine = NamedRoutine { - id: name.to_owned(), - f: Box::new(RefCell::new(Function::new(f))), - }; - - Fun { - f: routine, - _phantom: PhantomData, - } - } -} - /// Argument to [`Bencher::iter_batched`](struct.Bencher.html#method.iter_batched) and /// [`Bencher::iter_batched_ref`](struct.Bencher.html#method.iter_batched_ref) which controls the /// batch size. @@ -645,17 +608,6 @@ impl Criterion { self } - /// Return true if generation of the plots is possible. - #[deprecated( - since = "0.3.4", - note = "No longer useful; since the plotters backend is available Criterion.rs can always generate plots" - )] - pub fn can_plot(&self) -> bool { - // Trivially true now that we have plotters. - // TODO: Deprecate and remove this. - true - } - /// Names an explicit baseline and enables overwriting the previous results. pub fn save_baseline(mut self, baseline: String) -> Criterion { self.baseline_directory = baseline; @@ -1180,138 +1132,6 @@ where ); self } - - /// Benchmarks a function under various inputs - /// - /// This is a convenience method to execute several related benchmarks. Each benchmark will - /// receive the id: `${id}/${input}`. - /// - /// # Example - /// - /// ```rust - /// # #[macro_use] extern crate criterion; - /// # use self::criterion::*; - /// - /// fn bench(c: &mut Criterion) { - /// c.bench_function_over_inputs("from_elem", - /// |b: &mut Bencher, size: &usize| { - /// b.iter(|| vec![0u8; *size]); - /// }, - /// vec![1024, 2048, 4096] - /// ); - /// } - /// - /// criterion_group!(benches, bench); - /// criterion_main!(benches); - /// ``` - #[doc(hidden)] - #[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")] - #[allow(deprecated)] - pub fn bench_function_over_inputs( - &mut self, - id: &str, - f: F, - inputs: I, - ) -> &mut Criterion - where - I: IntoIterator, - I::Item: fmt::Debug + 'static, - F: FnMut(&mut Bencher<'_, M>, &I::Item) + 'static, - { - self.bench(id, ParameterizedBenchmark::new(id, f, inputs)) - } - - /// Benchmarks multiple functions - /// - /// All functions get the same input and are compared with the other implementations. - /// Works similar to `bench_function`, but with multiple functions. - /// - /// # Example - /// - /// ``` rust - /// # #[macro_use] extern crate criterion; - /// # use self::criterion::*; - /// # fn seq_fib(i: &u32) {} - /// # fn par_fib(i: &u32) {} - /// - /// fn bench_seq_fib(b: &mut Bencher, i: &u32) { - /// b.iter(|| { - /// seq_fib(i); - /// }); - /// } - /// - /// fn bench_par_fib(b: &mut Bencher, i: &u32) { - /// b.iter(|| { - /// par_fib(i); - /// }); - /// } - /// - /// fn bench(c: &mut Criterion) { - /// let sequential_fib = Fun::new("Sequential", bench_seq_fib); - /// let parallel_fib = Fun::new("Parallel", bench_par_fib); - /// let funs = vec![sequential_fib, parallel_fib]; - /// - /// c.bench_functions("Fibonacci", funs, 14); - /// } - /// - /// criterion_group!(benches, bench); - /// criterion_main!(benches); - /// ``` - #[doc(hidden)] - #[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")] - #[allow(deprecated)] - pub fn bench_functions( - &mut self, - id: &str, - funs: Vec>, - input: I, - ) -> &mut Criterion - where - I: fmt::Debug + 'static, - { - let benchmark = ParameterizedBenchmark::with_functions( - funs.into_iter().map(|fun| fun.f).collect(), - vec![input], - ); - - self.bench(id, benchmark) - } - - /// Executes the given benchmark. Use this variant to execute benchmarks - /// with complex configuration. This can be used to compare multiple - /// functions, execute benchmarks with custom configuration settings and - /// more. See the Benchmark and ParameterizedBenchmark structs for more - /// information. - /// - /// ```rust - /// # #[macro_use] extern crate criterion; - /// # use criterion::*; - /// # fn routine_1() {} - /// # fn routine_2() {} - /// - /// fn bench(c: &mut Criterion) { - /// // Setup (construct data, allocate memory, etc) - /// c.bench( - /// "routines", - /// Benchmark::new("routine_1", |b| b.iter(|| routine_1())) - /// .with_function("routine_2", |b| b.iter(|| routine_2())) - /// .sample_size(50) - /// ); - /// } - /// - /// criterion_group!(benches, bench); - /// criterion_main!(benches); - /// ``` - #[doc(hidden)] - #[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")] - pub fn bench>( - &mut self, - group_id: &str, - benchmark: B, - ) -> &mut Criterion { - benchmark.run(group_id, self); - self - } } trait DurationExt { @@ -1358,7 +1178,7 @@ pub enum AxisScale { /// or benchmark group. /// /// ```rust -/// use self::criterion::{Bencher, Criterion, Benchmark, PlotConfiguration, AxisScale}; +/// use self::criterion::{Bencher, Criterion, PlotConfiguration, AxisScale}; /// /// let plot_config = PlotConfiguration::default() /// .summary_scale(AxisScale::Logarithmic); diff --git a/tests/criterion_tests.rs b/tests/criterion_tests.rs index cca448e02..7a2c7b44f 100644 --- a/tests/criterion_tests.rs +++ b/tests/criterion_tests.rs @@ -1,11 +1,9 @@ -#![allow(deprecated)] - use criterion; use serde_json; use criterion::{ - criterion_group, criterion_main, profiler::Profiler, BatchSize, Benchmark, BenchmarkId, - Criterion, Fun, ParameterizedBenchmark, SamplingMode, Throughput, + criterion_group, criterion_main, profiler::Profiler, BatchSize, BenchmarkId, Criterion, + SamplingMode, }; use serde_json::value::Value; use std::cell::{Cell, RefCell}; @@ -250,27 +248,6 @@ fn test_bench_function() { short_benchmark(&dir).bench_function("test_bench_function", move |b| b.iter(|| 10)); } -#[test] -fn test_bench_functions() { - let dir = temp_dir(); - let function_1 = Fun::new("times 10", |b, i| b.iter(|| *i * 10)); - let function_2 = Fun::new("times 20", |b, i| b.iter(|| *i * 20)); - - let functions = vec![function_1, function_2]; - - short_benchmark(&dir).bench_functions("test_bench_functions", functions, 20); -} - -#[test] -fn test_bench_function_over_inputs() { - let dir = temp_dir(); - short_benchmark(&dir).bench_function_over_inputs( - "test_bench_function_over_inputs", - |b, i| b.iter(|| *i * 10), - vec![100, 1000], - ); -} - #[test] fn test_filtering() { let dir = temp_dir(); @@ -288,67 +265,47 @@ fn test_filtering() { #[test] fn test_timing_loops() { let dir = temp_dir(); - short_benchmark(&dir).bench( - "test_timing_loops", - Benchmark::new("iter", |b| b.iter(|| 10)) - .with_function("iter_with_setup", |b| { - b.iter_with_setup(|| vec![10], |v| v[0]) - }) - .with_function("iter_with_large_setup", |b| { - b.iter_with_large_setup(|| vec![10], |v| v[0]) - }) - .with_function("iter_with_large_drop", |b| { - b.iter_with_large_drop(|| vec![10; 100]) - }) - .with_function("iter_batched_small", |b| { - b.iter_batched(|| vec![10], |v| v[0], BatchSize::SmallInput) - }) - .with_function("iter_batched_large", |b| { - b.iter_batched(|| vec![10], |v| v[0], BatchSize::LargeInput) - }) - .with_function("iter_batched_per_iteration", |b| { - b.iter_batched(|| vec![10], |v| v[0], BatchSize::PerIteration) - }) - .with_function("iter_batched_one_batch", |b| { - b.iter_batched(|| vec![10], |v| v[0], BatchSize::NumBatches(1)) - }) - .with_function("iter_batched_10_iterations", |b| { - b.iter_batched(|| vec![10], |v| v[0], BatchSize::NumIterations(10)) - }) - .with_function("iter_batched_ref_small", |b| { - b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::SmallInput) - }) - .with_function("iter_batched_ref_large", |b| { - b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::LargeInput) - }) - .with_function("iter_batched_ref_per_iteration", |b| { - b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::PerIteration) - }) - .with_function("iter_batched_ref_one_batch", |b| { - b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::NumBatches(1)) - }) - .with_function("iter_batched_ref_10_iterations", |b| { - b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::NumIterations(10)) - }), - ); -} - -#[test] -fn test_throughput() { - let dir = temp_dir(); - short_benchmark(&dir).bench( - "test_throughput_bytes", - Benchmark::new("strlen", |b| b.iter(|| "foo".len())).throughput(Throughput::Bytes(3)), - ); - short_benchmark(&dir).bench( - "test_throughput_elems", - ParameterizedBenchmark::new( - "veclen", - |b, v| b.iter(|| v.len()), - vec![vec![1], vec![1, 2, 3]], - ) - .throughput(|v| Throughput::Elements(v.len() as u64)), - ); + let mut c = short_benchmark(&dir); + let mut group = c.benchmark_group("test_timing_loops"); + group.bench_function("iter_with_setup", |b| { + b.iter_with_setup(|| vec![10], |v| v[0]) + }); + group.bench_function("iter_with_large_setup", |b| { + b.iter_with_large_setup(|| vec![10], |v| v[0]) + }); + group.bench_function("iter_with_large_drop", |b| { + b.iter_with_large_drop(|| vec![10; 100]) + }); + group.bench_function("iter_batched_small", |b| { + b.iter_batched(|| vec![10], |v| v[0], BatchSize::SmallInput) + }); + group.bench_function("iter_batched_large", |b| { + b.iter_batched(|| vec![10], |v| v[0], BatchSize::LargeInput) + }); + group.bench_function("iter_batched_per_iteration", |b| { + b.iter_batched(|| vec![10], |v| v[0], BatchSize::PerIteration) + }); + group.bench_function("iter_batched_one_batch", |b| { + b.iter_batched(|| vec![10], |v| v[0], BatchSize::NumBatches(1)) + }); + group.bench_function("iter_batched_10_iterations", |b| { + b.iter_batched(|| vec![10], |v| v[0], BatchSize::NumIterations(10)) + }); + group.bench_function("iter_batched_ref_small", |b| { + b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::SmallInput) + }); + group.bench_function("iter_batched_ref_large", |b| { + b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::LargeInput) + }); + group.bench_function("iter_batched_ref_per_iteration", |b| { + b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::PerIteration) + }); + group.bench_function("iter_batched_ref_one_batch", |b| { + b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::NumBatches(1)) + }); + group.bench_function("iter_batched_ref_10_iterations", |b| { + b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::NumIterations(10)) + }); } // Verify that all expected output files are present @@ -357,13 +314,12 @@ fn test_output_files() { let tempdir = temp_dir(); // Run benchmarks twice to produce comparisons for _ in 0..2 { - short_benchmark(&tempdir).bench( - "test_output", - Benchmark::new("output_1", |b| b.iter(|| 10)) - .with_function("output_2", |b| b.iter(|| 20)) - .with_function("output_\\/*\"?", |b| b.iter(|| 30)) - .sampling_mode(SamplingMode::Linear), - ); + let mut c = short_benchmark(&tempdir); + let mut group = c.benchmark_group("test_output"); + group.sampling_mode(SamplingMode::Linear); + group.bench_function("output_1", |b| b.iter(|| 10)); + group.bench_function("output_2", |b| b.iter(|| 20)); + group.bench_function("output_\\/*\"?", |b| b.iter(|| 30)); } // For each benchmark, assert that the expected files are present. @@ -379,44 +335,38 @@ fn test_output_files() { verify_stats(&dir, "base"); verify_json(&dir, "change/estimates.json"); - if short_benchmark(&tempdir).can_plot() { - verify_svg(&dir, "report/MAD.svg"); - verify_svg(&dir, "report/mean.svg"); - verify_svg(&dir, "report/median.svg"); - verify_svg(&dir, "report/pdf.svg"); - verify_svg(&dir, "report/regression.svg"); - verify_svg(&dir, "report/SD.svg"); - verify_svg(&dir, "report/slope.svg"); - verify_svg(&dir, "report/typical.svg"); - verify_svg(&dir, "report/both/pdf.svg"); - verify_svg(&dir, "report/both/regression.svg"); - verify_svg(&dir, "report/change/mean.svg"); - verify_svg(&dir, "report/change/median.svg"); - verify_svg(&dir, "report/change/t-test.svg"); - - verify_svg(&dir, "report/pdf_small.svg"); - verify_svg(&dir, "report/regression_small.svg"); - verify_svg(&dir, "report/relative_pdf_small.svg"); - verify_svg(&dir, "report/relative_regression_small.svg"); - verify_html(&dir, "report/index.html"); - } + verify_svg(&dir, "report/MAD.svg"); + verify_svg(&dir, "report/mean.svg"); + verify_svg(&dir, "report/median.svg"); + verify_svg(&dir, "report/pdf.svg"); + verify_svg(&dir, "report/regression.svg"); + verify_svg(&dir, "report/SD.svg"); + verify_svg(&dir, "report/slope.svg"); + verify_svg(&dir, "report/typical.svg"); + verify_svg(&dir, "report/both/pdf.svg"); + verify_svg(&dir, "report/both/regression.svg"); + verify_svg(&dir, "report/change/mean.svg"); + verify_svg(&dir, "report/change/median.svg"); + verify_svg(&dir, "report/change/t-test.svg"); + + verify_svg(&dir, "report/pdf_small.svg"); + verify_svg(&dir, "report/regression_small.svg"); + verify_svg(&dir, "report/relative_pdf_small.svg"); + verify_svg(&dir, "report/relative_regression_small.svg"); + verify_html(&dir, "report/index.html"); } // Check for overall report files - if short_benchmark(&tempdir).can_plot() { - let dir = tempdir.path().join("test_output"); + let dir = tempdir.path().join("test_output"); - verify_svg(&dir, "report/violin.svg"); - verify_html(&dir, "report/index.html"); - } + verify_svg(&dir, "report/violin.svg"); + verify_html(&dir, "report/index.html"); // Run the final summary process and check for the report that produces short_benchmark(&tempdir).final_summary(); - if short_benchmark(&tempdir).can_plot() { - let dir = tempdir.path().to_owned(); + let dir = tempdir.path().to_owned(); - verify_html(&dir, "report/index.html"); - } + verify_html(&dir, "report/index.html"); } #[test] @@ -424,10 +374,10 @@ fn test_output_files_flat_sampling() { let tempdir = temp_dir(); // Run benchmark twice to produce comparisons for _ in 0..2 { - short_benchmark(&tempdir).bench( - "test_output", - Benchmark::new("output_flat", |b| b.iter(|| 10)).sampling_mode(SamplingMode::Flat), - ); + let mut c = short_benchmark(&tempdir); + let mut group = c.benchmark_group("test_output"); + group.sampling_mode(SamplingMode::Flat); + group.bench_function("output_flat", |b| b.iter(|| 10)); } let dir = tempdir.path().join("test_output/output_flat"); @@ -436,33 +386,31 @@ fn test_output_files_flat_sampling() { verify_stats(&dir, "base"); verify_json(&dir, "change/estimates.json"); - if short_benchmark(&tempdir).can_plot() { - verify_svg(&dir, "report/MAD.svg"); - verify_svg(&dir, "report/mean.svg"); - verify_svg(&dir, "report/median.svg"); - verify_svg(&dir, "report/pdf.svg"); - verify_svg(&dir, "report/iteration_times.svg"); - verify_svg(&dir, "report/SD.svg"); - verify_svg(&dir, "report/typical.svg"); - verify_svg(&dir, "report/both/pdf.svg"); - verify_svg(&dir, "report/both/iteration_times.svg"); - verify_svg(&dir, "report/change/mean.svg"); - verify_svg(&dir, "report/change/median.svg"); - verify_svg(&dir, "report/change/t-test.svg"); - - verify_svg(&dir, "report/pdf_small.svg"); - verify_svg(&dir, "report/iteration_times_small.svg"); - verify_svg(&dir, "report/relative_pdf_small.svg"); - verify_svg(&dir, "report/relative_iteration_times_small.svg"); - verify_html(&dir, "report/index.html"); - } + verify_svg(&dir, "report/MAD.svg"); + verify_svg(&dir, "report/mean.svg"); + verify_svg(&dir, "report/median.svg"); + verify_svg(&dir, "report/pdf.svg"); + verify_svg(&dir, "report/iteration_times.svg"); + verify_svg(&dir, "report/SD.svg"); + verify_svg(&dir, "report/typical.svg"); + verify_svg(&dir, "report/both/pdf.svg"); + verify_svg(&dir, "report/both/iteration_times.svg"); + verify_svg(&dir, "report/change/mean.svg"); + verify_svg(&dir, "report/change/median.svg"); + verify_svg(&dir, "report/change/t-test.svg"); + + verify_svg(&dir, "report/pdf_small.svg"); + verify_svg(&dir, "report/iteration_times_small.svg"); + verify_svg(&dir, "report/relative_pdf_small.svg"); + verify_svg(&dir, "report/relative_iteration_times_small.svg"); + verify_html(&dir, "report/index.html"); } #[test] #[should_panic(expected = "Benchmark function must call Bencher::iter or related method.")] fn test_bench_with_no_iteration_panics() { let dir = temp_dir(); - short_benchmark(&dir).bench("test_no_iter", Benchmark::new("no_iter", |_b| {})); + short_benchmark(&dir).bench_function("no_iter", |_b| {}); } #[test] From 0ef6c82bf5e6380140682114d812490a8800a302 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Mon, 26 Jul 2021 15:54:29 +0800 Subject: [PATCH 02/29] Run CI for pull requests to the version-0.4 branch. --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c3990474..37679cd29 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,6 +5,7 @@ on: pull_request: branches: - master + - version-0.4 name: tests From 63551485c88353112e82ed4095ae3c508000ef90 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Mon, 26 Jul 2021 16:17:54 +0800 Subject: [PATCH 03/29] Honor optional feature flags. (#497) * Disable warnings about HTML reports and standalone support. * Include 'html_reports' in the set of stable features. * Suppress html and csv if the feature flags are missing. * Make the CSV dependency optional. * Group optional dependencies in cargo manifest. * Test both with all features and no features. --- .github/workflows/ci.yaml | 10 +++++++ CHANGELOG.md | 4 +++ Cargo.toml | 23 +++++++++++----- src/error.rs | 8 ++++++ src/lib.rs | 57 +++------------------------------------ src/macros.rs | 3 --- src/report.rs | 9 ++++--- tests/criterion_tests.rs | 1 + 8 files changed, 48 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 37679cd29..92ed6e78f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,11 +37,21 @@ jobs: command: build args: --features stable + - uses: actions-rs/cargo@v1 + with: + command: build + args: --no-default-features + - uses: actions-rs/cargo@v1 with: command: test args: --features stable + - uses: actions-rs/cargo@v1 + with: + command: test + args: --no-default-features + - uses: actions-rs/cargo@v1 if: ${{ matrix.rust == 'stable' }} with: diff --git a/CHANGELOG.md b/CHANGELOG.md index f51d3f264..6ecd03554 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - The `Criterion::bench_functions` function has been removed. - The `Criterion::bench` function has been removed. +### Changed +- HTML report hidden behind non-default feature flag: 'html_reports' +- Standalone support (ie without cargo-criterion) feature flag: 'cargo_bench_support' + ## [Unreleased] ### Fixed - Corrected `Criterion.toml` in the book. diff --git a/Cargo.toml b/Cargo.toml index 4489966b5..757871f92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,6 @@ serde_derive = "1.0" serde_cbor = "0.11" atty = "0.2" clap = { version = "2.33", default-features = false } -csv = "1.1" walkdir = "2.3" tinytemplate = "1.1" cast = "0.2" @@ -34,10 +33,13 @@ num-traits = { version = "0.2", default-features = false } oorandom = "11.1" rayon = "1.3" regex = { version = "1.3", default-features = false, features = ["std"] } -futures = { version = "0.3", default_features = false, optional = true } -smol = { version = "1.2", default-features = false, optional = true } -tokio = { version = "1.0", default-features = false, features = ["rt"], optional = true } -async-std = { version = "1.9", optional = true } + +# Optional dependencies +csv = { version = "1.1", optional = true } +futures = { version = "0.3", default_features = false, optional = true } +smol = { version = "1.2", default-features = false, optional = true } +tokio = { version = "1.0", default-features = false, features = ["rt"], optional = true } +async-std = { version = "1.9", optional = true } [dependencies.plotters] version = "^0.3.1" @@ -55,7 +57,14 @@ futures = { version = "0.3", default_features = false, features = ["executor" maintenance = { status = "passively-maintained" } [features] -stable = ["async_futures", "async_smol", "async_tokio", "async_std"] +stable = [ + "csv_output", + "html_reports", + "async_futures", + "async_smol", + "async_tokio", + "async_std", +] default = ["cargo_bench_support"] # Enable use of the nightly-only test::black_box function to discourage compiler optimizations. @@ -82,7 +91,7 @@ cargo_bench_support = [] # This feature _currently_ does nothing, but in 0.4.0 it will be # required in order to have Criterion.rs generate CSV files. This feature is deprecated in favor of # cargo-criterion's --message-format=json option. -csv_output = [] +csv_output = ["csv"] [workspace] exclude = ["cargo-criterion"] diff --git a/src/error.rs b/src/error.rs index 9b7eb17b1..459a716f5 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,3 +1,4 @@ +#[cfg(feature = "csv_output")] use csv::Error as CsvError; use serde_json::Error as SerdeError; use std::error::Error as StdError; @@ -21,6 +22,8 @@ pub enum Error { path: PathBuf, inner: SerdeError, }, + #[cfg(feature = "csv_output")] + /// This API requires the following crate features to be activated: csv_output CsvError(CsvError), } impl fmt::Display for Error { @@ -37,6 +40,7 @@ impl fmt::Display for Error { "Failed to read or write file {:?} due to serialization error: {}", path, inner ), + #[cfg(feature = "csv_output")] Error::CsvError(inner) => write!(f, "CSV error: {}", inner), } } @@ -47,6 +51,7 @@ impl StdError for Error { Error::AccessError { .. } => "AccessError", Error::CopyError { .. } => "CopyError", Error::SerdeError { .. } => "SerdeError", + #[cfg(feature = "csv_output")] Error::CsvError(_) => "CsvError", } } @@ -56,10 +61,13 @@ impl StdError for Error { Error::AccessError { inner, .. } => Some(inner), Error::CopyError { inner, .. } => Some(inner), Error::SerdeError { inner, .. } => Some(inner), + #[cfg(feature = "csv_output")] Error::CsvError(inner) => Some(inner), } } } + +#[cfg(feature = "csv_output")] impl From for Error { fn from(other: CsvError) -> Error { Error::CsvError(other) diff --git a/src/lib.rs b/src/lib.rs index cdfe1f873..458ee6aaf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -57,6 +57,7 @@ mod benchmark_group; pub mod async_executor; mod bencher; mod connection; +#[cfg(feature = "csv_output")] mod csv_report; mod error; mod estimate; @@ -87,7 +88,6 @@ use criterion_plot::{Version, VersionError}; use crate::benchmark::BenchmarkConfig; use crate::connection::Connection; use crate::connection::OutgoingMessage; -use crate::csv_report::FileCsvReport; use crate::html::Html; use crate::measurement::{Measurement, WallTime}; use crate::plot::{Gnuplot, Plotter, PlottersBackend}; @@ -372,10 +372,9 @@ impl Default for Criterion { cli: CliReport::new(false, false, false), bencher_enabled: false, bencher: BencherReport, - html_enabled: true, + html_enabled: cfg!(feature = "html_reports"), html: Html::new(DEFAULT_PLOTTING_BACKEND.create_plotter()), - csv_enabled: true, - csv: FileCsvReport, + csv_enabled: cfg!(feature = "csv_output"), }; let mut criterion = Criterion { @@ -1386,53 +1385,3 @@ pub fn runner(benches: &[&dyn Fn()]) { } Criterion::default().configure_from_args().final_summary(); } - -/// Print a warning informing users about upcoming changes to features -#[cfg(not(feature = "html_reports"))] -#[doc(hidden)] -pub fn __warn_about_html_reports_feature() { - if CARGO_CRITERION_CONNECTION.is_none() { - println!( - "WARNING: HTML report generation will become a non-default optional feature in Criterion.rs 0.4.0." - ); - println!( - "This feature is being moved to cargo-criterion \ - (https://github.com/bheisler/cargo-criterion) and will be optional in a future \ - version of Criterion.rs. To silence this warning, either switch to cargo-criterion or \ - enable the 'html_reports' feature in your Cargo.toml." - ); - println!(); - } -} - -/// Print a warning informing users about upcoming changes to features -#[cfg(feature = "html_reports")] -#[doc(hidden)] -pub fn __warn_about_html_reports_feature() { - // They have the feature enabled, so they're ready for the update. -} - -/// Print a warning informing users about upcoming changes to features -#[cfg(not(feature = "cargo_bench_support"))] -#[doc(hidden)] -pub fn __warn_about_cargo_bench_support_feature() { - if CARGO_CRITERION_CONNECTION.is_none() { - println!( - "WARNING: In Criterion.rs 0.4.0, running criterion benchmarks outside of cargo-criterion will become a default optional feature." - ); - println!( - "The statistical analysis and reporting is being moved to cargo-criterion \ - (https://github.com/bheisler/cargo-criterion) and will be optional in a future \ - version of Criterion.rs. To silence this warning, either switch to cargo-criterion or \ - enable the 'cargo_bench_support' feature in your Cargo.toml." - ); - println!(); - } -} - -/// Print a warning informing users about upcoming changes to features -#[cfg(feature = "cargo_bench_support")] -#[doc(hidden)] -pub fn __warn_about_cargo_bench_support_feature() { - // They have the feature enabled, so they're ready for the update. -} diff --git a/src/macros.rs b/src/macros.rs index 85d8c5956..df7a44d9a 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -120,9 +120,6 @@ macro_rules! criterion_group { macro_rules! criterion_main { ( $( $group:path ),+ $(,)* ) => { fn main() { - $crate::__warn_about_html_reports_feature(); - $crate::__warn_about_cargo_bench_support_feature(); - $( $group(); )+ diff --git a/src/report.rs b/src/report.rs index 60a144a29..72a6ffe47 100644 --- a/src/report.rs +++ b/src/report.rs @@ -1,5 +1,7 @@ +#[cfg(feature = "csv_output")] +use crate::csv_report::FileCsvReport; +use crate::stats::bivariate::regression::Slope; use crate::stats::univariate::outliers::tukey::LabeledSample; -use crate::{csv_report::FileCsvReport, stats::bivariate::regression::Slope}; use crate::{html::Html, stats::bivariate::Data}; use crate::estimate::{ChangeDistributions, ChangeEstimates, Distributions, Estimate, Estimates}; @@ -46,6 +48,7 @@ impl<'a> MeasurementData<'a> { self.data.x() } + #[cfg(feature = "csv_output")] pub fn sample_times(&self) -> &Sample { self.data.y() } @@ -304,7 +307,6 @@ pub(crate) struct Reports { pub(crate) bencher_enabled: bool, pub(crate) bencher: BencherReport, pub(crate) csv_enabled: bool, - pub(crate) csv: FileCsvReport, pub(crate) html_enabled: bool, pub(crate) html: Html, } @@ -317,8 +319,9 @@ macro_rules! reports_impl { if self.bencher_enabled { self.bencher.$name($($argn),*); } + #[cfg(feature = "csv_output")] if self.csv_enabled { - self.csv.$name($($argn),*); + FileCsvReport.$name($($argn),*); } if self.html_enabled { self.html.$name($($argn),*); diff --git a/tests/criterion_tests.rs b/tests/criterion_tests.rs index 7a2c7b44f..11ead0661 100644 --- a/tests/criterion_tests.rs +++ b/tests/criterion_tests.rs @@ -86,6 +86,7 @@ fn verify_stats(dir: &PathBuf, baseline: &str) { verify_json(&dir, &format!("{}/sample.json", baseline)); verify_json(&dir, &format!("{}/tukey.json", baseline)); verify_json(&dir, &format!("{}/benchmark.json", baseline)); + #[cfg(feature = "csv_output")] verify_file(&dir, &format!("{}/raw.csv", baseline)); } From e7c08b393bdf6ae17b60f0d1729a02042e378819 Mon Sep 17 00:00:00 2001 From: "NAKASHIMA, Makoto" Date: Mon, 26 Jul 2021 22:16:12 +0900 Subject: [PATCH 04/29] Changes error messages output to stderr (#412) Some tools (e.g., rhysd/github-actions-benchmark) that parse the output of criterion do not correctly parse the standard output if it contains an unexpected string, such as an error message. The fix in this commit ensures that the tool will work correctly in environments where gnuplot is not installed (e.g., GitHub Actions runners). Also, from the conventions of the command line interface, it is preferable that such error messages be output to stderr. --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 458ee6aaf..a2e2885db 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -107,8 +107,8 @@ lazy_static! { Ok(_) => PlottingBackend::Gnuplot, Err(e) => { match e { - VersionError::Exec(_) => println!("Gnuplot not found, using plotters backend"), - e => println!( + VersionError::Exec(_) => eprintln!("Gnuplot not found, using plotters backend"), + e => eprintln!( "Gnuplot not found or not usable, using plotters backend\n{}", e ), From 09efa53ff07b55c581011debd48fbf061501bc91 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Mon, 26 Jul 2021 23:14:36 +0800 Subject: [PATCH 05/29] Basic support for targeting wasm32-wasi (#498) * Make the plotters dependency optional. * Turn rayon into an optional dependency. * Add cli flag for discarding baselines rather than saving them to disk. * Disable plotting if html_reports is disabled. * Support testing with and without the html_reports flag. --- Cargo.toml | 7 ++- src/analysis/mod.rs | 8 +-- src/lib.rs | 81 ++++++++++++++++-------- src/plot/mod.rs | 2 + src/report.rs | 7 +-- src/stats/bivariate/mod.rs | 53 ++++++++++------ src/stats/univariate/kde/mod.rs | 10 ++- src/stats/univariate/mixed.rs | 61 ++++++++++++------ src/stats/univariate/mod.rs | 55 +++++++++++++---- src/stats/univariate/sample.rs | 56 +++++++++++------ tests/criterion_tests.rs | 106 ++++++++++++++++++-------------- 11 files changed, 292 insertions(+), 154 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 757871f92..a85276a61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,12 +29,12 @@ clap = { version = "2.33", default-features = false } walkdir = "2.3" tinytemplate = "1.1" cast = "0.2" -num-traits = { version = "0.2", default-features = false } +num-traits = { version = "0.2", default-features = false, features = ["std"] } oorandom = "11.1" -rayon = "1.3" regex = { version = "1.3", default-features = false, features = ["std"] } # Optional dependencies +rayon = { version = "1.3", optional = true } csv = { version = "1.1", optional = true } futures = { version = "0.3", default_features = false, optional = true } smol = { version = "1.2", default-features = false, optional = true } @@ -43,6 +43,7 @@ async-std = { version = "1.9", optional = true } [dependencies.plotters] version = "^0.3.1" +optional = true default-features = false features = ["svg_backend", "area_series", "line_series"] @@ -65,7 +66,7 @@ stable = [ "async_tokio", "async_std", ] -default = ["cargo_bench_support"] +default = ["rayon", "plotters", "cargo_bench_support"] # Enable use of the nightly-only test::black_box function to discourage compiler optimizations. real_blackbox = [] diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs index 5d84bef16..1cb58285e 100644 --- a/src/analysis/mod.rs +++ b/src/analysis/mod.rs @@ -128,7 +128,7 @@ pub(crate) fn common( .collect::>(); let avg_times = Sample::new(&avg_times); - if criterion.connection.is_none() && criterion.load_baseline.is_none() { + if criterion.should_save_baseline() { log_if_err!({ let mut new_dir = criterion.output_directory.clone(); new_dir.push(id.as_directory_name()); @@ -139,7 +139,7 @@ pub(crate) fn common( let data = Data::new(&iters, ×); let labeled_sample = tukey::classify(avg_times); - if criterion.connection.is_none() { + if criterion.should_save_baseline() { log_if_err!({ let mut tukey_file = criterion.output_directory.to_owned(); tukey_file.push(id.as_directory_name()); @@ -156,7 +156,7 @@ pub(crate) fn common( distributions.slope = Some(distribution); } - if criterion.connection.is_none() && criterion.load_baseline.is_none() { + if criterion.should_save_baseline() { log_if_err!({ let mut sample_file = criterion.output_directory.clone(); sample_file.push(id.as_directory_name()); @@ -237,7 +237,7 @@ pub(crate) fn common( criterion.measurement.formatter(), ); - if criterion.connection.is_none() && criterion.load_baseline.is_none() { + if criterion.should_save_baseline() { log_if_err!({ let mut benchmark_file = criterion.output_directory.clone(); benchmark_file.push(id.as_directory_name()); diff --git a/src/lib.rs b/src/lib.rs index a2e2885db..eca68012a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -90,7 +90,9 @@ use crate::connection::Connection; use crate::connection::OutgoingMessage; use crate::html::Html; use crate::measurement::{Measurement, WallTime}; -use crate::plot::{Gnuplot, Plotter, PlottersBackend}; +#[cfg(feature = "plotters")] +use crate::plot::PlottersBackend; +use crate::plot::{Gnuplot, Plotter}; use crate::profiler::{ExternalProfiler, Profiler}; use crate::report::{BencherReport, CliReport, Report, ReportContext, Reports}; @@ -103,18 +105,22 @@ lazy_static! { static ref DEBUG_ENABLED: bool = std::env::var_os("CRITERION_DEBUG").is_some(); static ref GNUPLOT_VERSION: Result = criterion_plot::version(); static ref DEFAULT_PLOTTING_BACKEND: PlottingBackend = { - match &*GNUPLOT_VERSION { - Ok(_) => PlottingBackend::Gnuplot, - Err(e) => { - match e { - VersionError::Exec(_) => eprintln!("Gnuplot not found, using plotters backend"), - e => eprintln!( - "Gnuplot not found or not usable, using plotters backend\n{}", - e - ), - }; - PlottingBackend::Plotters + if cfg!(feature = "html_reports") { + match &*GNUPLOT_VERSION { + Ok(_) => PlottingBackend::Gnuplot, + Err(e) => { + match e { + VersionError::Exec(_) => eprintln!("Gnuplot not found, using plotters backend"), + e => eprintln!( + "Gnuplot not found or not usable, using plotters backend\n{}", + e + ), + }; + PlottingBackend::Plotters + } } + } else { + PlottingBackend::None } }; static ref CARGO_CRITERION_CONNECTION: Option> = { @@ -265,6 +271,8 @@ pub enum Baseline { /// Save writes the benchmark results to the baseline directory, /// overwriting any results that were previously there. Save, + /// Discard benchmark results. + Discard, } /// Enum used to select the plotting backend. @@ -276,12 +284,18 @@ pub enum PlottingBackend { /// Plotting backend which uses the rust 'Plotters' library. This is the default if `gnuplot` /// is not installed. Plotters, + /// Null plotting backend which outputs nothing, + None, } impl PlottingBackend { - fn create_plotter(&self) -> Box { + fn create_plotter(&self) -> Option> { match self { - PlottingBackend::Gnuplot => Box::new(Gnuplot::default()), - PlottingBackend::Plotters => Box::new(PlottersBackend::default()), + PlottingBackend::Gnuplot => Some(Box::new(Gnuplot::default())), + #[cfg(feature = "plotters")] + PlottingBackend::Plotters => Some(Box::new(PlottersBackend::default())), + #[cfg(not(feature = "plotters"))] + PlottingBackend::Plotters => panic!("Criterion was built without plotters support."), + PlottingBackend::None => None, } } } @@ -372,8 +386,7 @@ impl Default for Criterion { cli: CliReport::new(false, false, false), bencher_enabled: false, bencher: BencherReport, - html_enabled: cfg!(feature = "html_reports"), - html: Html::new(DEFAULT_PLOTTING_BACKEND.create_plotter()), + html: DEFAULT_PLOTTING_BACKEND.create_plotter().map(Html::new), csv_enabled: cfg!(feature = "csv_output"), }; @@ -409,7 +422,7 @@ impl Default for Criterion { criterion.report.cli_enabled = false; criterion.report.bencher_enabled = false; criterion.report.csv_enabled = false; - criterion.report.html_enabled = false; + criterion.report.html = None; } criterion } @@ -457,7 +470,7 @@ impl Criterion { } } - self.report.html = Html::new(backend.create_plotter()); + self.report.html = backend.create_plotter().map(Html::new); self } @@ -595,15 +608,20 @@ impl Criterion { /// Enables plotting pub fn with_plots(mut self) -> Criterion { // If running under cargo-criterion then don't re-enable the reports; let it do the reporting. - if self.connection.is_none() { - self.report.html_enabled = true; + if self.connection.is_none() && self.report.html.is_none() { + let default_backend = DEFAULT_PLOTTING_BACKEND.create_plotter(); + if let Some(backend) = default_backend { + self.report.html = Some(Html::new(backend)); + } else { + panic!("Cannot find a default plotting backend!"); + } } self } /// Disables plotting pub fn without_plots(mut self) -> Criterion { - self.report.html_enabled = false; + self.report.html = None; self } @@ -707,6 +725,10 @@ impl Criterion { .long("save-baseline") .default_value("base") .help("Save results under a named baseline.")) + .arg(Arg::with_name("discard-baseline") + .long("discard-baseline") + .conflicts_with_all(&["save-baseline", "baseline"]) + .help("Discard benchmark results.")) .arg(Arg::with_name("baseline") .short("b") .long("baseline") @@ -878,14 +900,15 @@ https://bheisler.github.io/criterion.rs/book/faq.html if matches.is_present("noplot") { self = self.without_plots(); - } else { - self = self.with_plots(); } if let Some(dir) = matches.value_of("save-baseline") { self.baseline = Baseline::Save; self.baseline_directory = dir.to_owned() } + if matches.is_present("discard-baseline") { + self.baseline = Baseline::Discard; + } if let Some(dir) = matches.value_of("baseline") { self.baseline = Baseline::Compare; self.baseline_directory = dir.to_owned(); @@ -896,7 +919,7 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.report.cli_enabled = false; self.report.bencher_enabled = false; self.report.csv_enabled = false; - self.report.html_enabled = false; + self.report.html = None; } else { match matches.value_of("output-format") { Some("bencher") => { @@ -1017,6 +1040,14 @@ https://bheisler.github.io/criterion.rs/book/faq.html } } + /// Returns true iff we should save the benchmark results in + /// json files on the local disk. + fn should_save_baseline(&self) -> bool { + self.connection.is_none() + && self.load_baseline.is_none() + && !matches!(self.baseline, Baseline::Discard) + } + /// Return a benchmark group. All benchmarks performed using a benchmark group will be /// grouped together in the final report. /// diff --git a/src/plot/mod.rs b/src/plot/mod.rs index cb836a394..4bce39468 100644 --- a/src/plot/mod.rs +++ b/src/plot/mod.rs @@ -1,7 +1,9 @@ mod gnuplot_backend; +#[cfg(feature = "plotters")] mod plotters_backend; pub(crate) use gnuplot_backend::Gnuplot; +#[cfg(feature = "plotters")] pub(crate) use plotters_backend::PlottersBackend; use crate::estimate::Statistic; diff --git a/src/report.rs b/src/report.rs index 72a6ffe47..d51930f0b 100644 --- a/src/report.rs +++ b/src/report.rs @@ -307,8 +307,7 @@ pub(crate) struct Reports { pub(crate) bencher_enabled: bool, pub(crate) bencher: BencherReport, pub(crate) csv_enabled: bool, - pub(crate) html_enabled: bool, - pub(crate) html: Html, + pub(crate) html: Option, } macro_rules! reports_impl { (fn $name:ident(&self, $($argn:ident: $argt:ty),*)) => { @@ -323,8 +322,8 @@ macro_rules! reports_impl { if self.csv_enabled { FileCsvReport.$name($($argn),*); } - if self.html_enabled { - self.html.$name($($argn),*); + if let Some(reporter) = &self.html { + reporter.$name($($argn),*); } } }; diff --git a/src/stats/bivariate/mod.rs b/src/stats/bivariate/mod.rs index d1e8df703..2351c9ef6 100644 --- a/src/stats/bivariate/mod.rs +++ b/src/stats/bivariate/mod.rs @@ -8,6 +8,7 @@ use crate::stats::bivariate::resamples::Resamples; use crate::stats::float::Float; use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; use crate::stats::univariate::Sample; +#[cfg(feature = "rayon")] use rayon::iter::{IntoParallelIterator, ParallelIterator}; /// Bivariate `(X, Y)` data @@ -72,27 +73,41 @@ where T::Distributions: Send, T::Builder: Send, { - (0..nresamples) - .into_par_iter() - .map_init( - || Resamples::new(*self), - |resamples, _| statistic(resamples.next()), - ) - .fold( - || T::Builder::new(0), - |mut sub_distributions, sample| { + #[cfg(feature = "rayon")] + { + (0..nresamples) + .into_par_iter() + .map_init( + || Resamples::new(*self), + |resamples, _| statistic(resamples.next()), + ) + .fold( + || T::Builder::new(0), + |mut sub_distributions, sample| { + sub_distributions.push(sample); + sub_distributions + }, + ) + .reduce( + || T::Builder::new(0), + |mut a, mut b| { + a.extend(&mut b); + a + }, + ) + .complete() + } + #[cfg(not(feature = "rayon"))] + { + let mut resamples = Resamples::new(*self); + (0..nresamples) + .map(|_| statistic(resamples.next())) + .fold(T::Builder::new(0), |mut sub_distributions, sample| { sub_distributions.push(sample); sub_distributions - }, - ) - .reduce( - || T::Builder::new(0), - |mut a, mut b| { - a.extend(&mut b); - a - }, - ) - .complete() + }) + .complete() + } } /// Returns a view into the `X` data diff --git a/src/stats/univariate/kde/mod.rs b/src/stats/univariate/kde/mod.rs index 9b0836d74..c54de55a2 100644 --- a/src/stats/univariate/kde/mod.rs +++ b/src/stats/univariate/kde/mod.rs @@ -5,6 +5,7 @@ pub mod kernel; use self::kernel::Kernel; use crate::stats::float::Float; use crate::stats::univariate::Sample; +#[cfg(feature = "rayon")] use rayon::prelude::*; /// Univariate kernel density estimator @@ -42,8 +43,13 @@ where /// /// - Multihreaded pub fn map(&self, xs: &[A]) -> Box<[A]> { - xs.par_iter() - .map(|&x| self.estimate(x)) + #[cfg(feature = "rayon")] + let iter = xs.par_iter(); + + #[cfg(not(feature = "rayon"))] + let iter = xs.iter(); + + iter.map(|&x| self.estimate(x)) .collect::>() .into_boxed_slice() } diff --git a/src/stats/univariate/mixed.rs b/src/stats/univariate/mixed.rs index 5c0a59fac..d6b845d1b 100644 --- a/src/stats/univariate/mixed.rs +++ b/src/stats/univariate/mixed.rs @@ -4,6 +4,7 @@ use crate::stats::float::Float; use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; use crate::stats::univariate::Resamples; use crate::stats::univariate::Sample; +#[cfg(feature = "rayon")] use rayon::prelude::*; /// Performs a *mixed* two-sample bootstrap @@ -27,31 +28,51 @@ where c.extend_from_slice(b); let c = Sample::new(&c); - (0..nresamples) - .into_par_iter() - .map_init( - || Resamples::new(c), - |resamples, _| { + #[cfg(feature = "rayon")] + { + (0..nresamples) + .into_par_iter() + .map_init( + || Resamples::new(c), + |resamples, _| { + let resample = resamples.next(); + let a: &Sample = Sample::new(&resample[..n_a]); + let b: &Sample = Sample::new(&resample[n_a..]); + + statistic(a, b) + }, + ) + .fold( + || T::Builder::new(0), + |mut sub_distributions, sample| { + sub_distributions.push(sample); + sub_distributions + }, + ) + .reduce( + || T::Builder::new(0), + |mut a, mut b| { + a.extend(&mut b); + a + }, + ) + .complete() + } + #[cfg(not(feature = "rayon"))] + { + let mut resamples = Resamples::new(c); + (0..nresamples) + .map(|_| { let resample = resamples.next(); let a: &Sample = Sample::new(&resample[..n_a]); let b: &Sample = Sample::new(&resample[n_a..]); statistic(a, b) - }, - ) - .fold( - || T::Builder::new(0), - |mut sub_distributions, sample| { + }) + .fold(T::Builder::new(0), |mut sub_distributions, sample| { sub_distributions.push(sample); sub_distributions - }, - ) - .reduce( - || T::Builder::new(0), - |mut a, mut b| { - a.extend(&mut b); - a - }, - ) - .complete() + }) + .complete() + } } diff --git a/src/stats/univariate/mod.rs b/src/stats/univariate/mod.rs index 8dfb5f8a9..5b221272d 100644 --- a/src/stats/univariate/mod.rs +++ b/src/stats/univariate/mod.rs @@ -11,6 +11,7 @@ pub mod outliers; use crate::stats::float::Float; use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; +#[cfg(feature = "rayon")] use rayon::prelude::*; use std::cmp; @@ -42,11 +43,42 @@ where let nresamples_sqrt = (nresamples as f64).sqrt().ceil() as usize; let per_chunk = (nresamples + nresamples_sqrt - 1) / nresamples_sqrt; - (0..nresamples_sqrt) - .into_par_iter() - .map_init( - || (Resamples::new(a), Resamples::new(b)), - |(a_resamples, b_resamples), i| { + #[cfg(feature = "rayon")] + { + (0..nresamples_sqrt) + .into_par_iter() + .map_init( + || (Resamples::new(a), Resamples::new(b)), + |(a_resamples, b_resamples), i| { + let start = i * per_chunk; + let end = cmp::min((i + 1) * per_chunk, nresamples); + let a_resample = a_resamples.next(); + + let mut sub_distributions: T::Builder = + TupledDistributionsBuilder::new(end - start); + + for _ in start..end { + let b_resample = b_resamples.next(); + sub_distributions.push(statistic(a_resample, b_resample)); + } + sub_distributions + }, + ) + .reduce( + || T::Builder::new(0), + |mut a, mut b| { + a.extend(&mut b); + a + }, + ) + .complete() + } + #[cfg(not(feature = "rayon"))] + { + let mut a_resamples = Resamples::new(a); + let mut b_resamples = Resamples::new(b); + (0..nresamples_sqrt) + .map(|i| { let start = i * per_chunk; let end = cmp::min((i + 1) * per_chunk, nresamples); let a_resample = a_resamples.next(); @@ -59,14 +91,11 @@ where sub_distributions.push(statistic(a_resample, b_resample)); } sub_distributions - }, - ) - .reduce( - || T::Builder::new(0), - |mut a, mut b| { + }) + .fold(T::Builder::new(0), |mut a, mut b| { a.extend(&mut b); a - }, - ) - .complete() + }) + .complete() + } } diff --git a/src/stats/univariate/sample.rs b/src/stats/univariate/sample.rs index 8f10db7b1..506cd0484 100644 --- a/src/stats/univariate/sample.rs +++ b/src/stats/univariate/sample.rs @@ -4,6 +4,7 @@ use crate::stats::float::Float; use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; use crate::stats::univariate::Percentiles; use crate::stats::univariate::Resamples; +#[cfg(feature = "rayon")] use rayon::prelude::*; /// A collection of data points drawn from a population @@ -127,7 +128,10 @@ where } let mut v = self.to_vec().into_boxed_slice(); + #[cfg(feature = "rayon")] v.par_sort_unstable_by(cmp); + #[cfg(not(feature = "rayon"))] + v.sort_unstable_by(cmp); // NB :-1: to intra-crate privacy rules unsafe { mem::transmute(v) } @@ -206,27 +210,41 @@ where T::Distributions: Send, T::Builder: Send, { - (0..nresamples) - .into_par_iter() - .map_init( - || Resamples::new(self), - |resamples, _| statistic(resamples.next()), - ) - .fold( - || T::Builder::new(0), - |mut sub_distributions, sample| { + #[cfg(feature = "rayon")] + { + (0..nresamples) + .into_par_iter() + .map_init( + || Resamples::new(self), + |resamples, _| statistic(resamples.next()), + ) + .fold( + || T::Builder::new(0), + |mut sub_distributions, sample| { + sub_distributions.push(sample); + sub_distributions + }, + ) + .reduce( + || T::Builder::new(0), + |mut a, mut b| { + a.extend(&mut b); + a + }, + ) + .complete() + } + #[cfg(not(feature = "rayon"))] + { + let mut resamples = Resamples::new(self); + (0..nresamples) + .map(|_| statistic(resamples.next())) + .fold(T::Builder::new(0), |mut sub_distributions, sample| { sub_distributions.push(sample); sub_distributions - }, - ) - .reduce( - || T::Builder::new(0), - |mut a, mut b| { - a.extend(&mut b); - a - }, - ) - .complete() + }) + .complete() + } } #[cfg(test)] diff --git a/tests/criterion_tests.rs b/tests/criterion_tests.rs index 11ead0661..a2fef0e1a 100644 --- a/tests/criterion_tests.rs +++ b/tests/criterion_tests.rs @@ -1,9 +1,10 @@ use criterion; use serde_json; +#[cfg(feature = "plotters")] +use criterion::SamplingMode; use criterion::{ criterion_group, criterion_main, profiler::Profiler, BatchSize, BenchmarkId, Criterion, - SamplingMode, }; use serde_json::value::Value; use std::cell::{Cell, RefCell}; @@ -31,7 +32,6 @@ fn short_benchmark(dir: &TempDir) -> Criterion { .warm_up_time(Duration::from_millis(250)) .measurement_time(Duration::from_millis(500)) .nresamples(2000) - .with_plots() } #[derive(Clone)] @@ -73,10 +73,12 @@ fn verify_json(dir: &PathBuf, path: &str) { serde_json::from_reader::(f).unwrap(); } +#[cfg(feature = "html_reports")] fn verify_svg(dir: &PathBuf, path: &str) { verify_file(dir, path); } +#[cfg(feature = "html_reports")] fn verify_html(dir: &PathBuf, path: &str) { verify_file(dir, path); } @@ -310,6 +312,7 @@ fn test_timing_loops() { } // Verify that all expected output files are present +#[cfg(feature = "plotters")] #[test] fn test_output_files() { let tempdir = temp_dir(); @@ -336,40 +339,50 @@ fn test_output_files() { verify_stats(&dir, "base"); verify_json(&dir, "change/estimates.json"); - verify_svg(&dir, "report/MAD.svg"); - verify_svg(&dir, "report/mean.svg"); - verify_svg(&dir, "report/median.svg"); - verify_svg(&dir, "report/pdf.svg"); - verify_svg(&dir, "report/regression.svg"); - verify_svg(&dir, "report/SD.svg"); - verify_svg(&dir, "report/slope.svg"); - verify_svg(&dir, "report/typical.svg"); - verify_svg(&dir, "report/both/pdf.svg"); - verify_svg(&dir, "report/both/regression.svg"); - verify_svg(&dir, "report/change/mean.svg"); - verify_svg(&dir, "report/change/median.svg"); - verify_svg(&dir, "report/change/t-test.svg"); - - verify_svg(&dir, "report/pdf_small.svg"); - verify_svg(&dir, "report/regression_small.svg"); - verify_svg(&dir, "report/relative_pdf_small.svg"); - verify_svg(&dir, "report/relative_regression_small.svg"); - verify_html(&dir, "report/index.html"); + #[cfg(feature = "html_reports")] + { + verify_svg(&dir, "report/MAD.svg"); + verify_svg(&dir, "report/mean.svg"); + verify_svg(&dir, "report/median.svg"); + verify_svg(&dir, "report/pdf.svg"); + verify_svg(&dir, "report/regression.svg"); + verify_svg(&dir, "report/SD.svg"); + verify_svg(&dir, "report/slope.svg"); + verify_svg(&dir, "report/typical.svg"); + verify_svg(&dir, "report/both/pdf.svg"); + verify_svg(&dir, "report/both/regression.svg"); + verify_svg(&dir, "report/change/mean.svg"); + verify_svg(&dir, "report/change/median.svg"); + verify_svg(&dir, "report/change/t-test.svg"); + + verify_svg(&dir, "report/pdf_small.svg"); + verify_svg(&dir, "report/regression_small.svg"); + verify_svg(&dir, "report/relative_pdf_small.svg"); + verify_svg(&dir, "report/relative_regression_small.svg"); + verify_html(&dir, "report/index.html"); + } } - // Check for overall report files - let dir = tempdir.path().join("test_output"); + #[cfg(feature = "html_reports")] + { + // Check for overall report files + let dir = tempdir.path().join("test_output"); - verify_svg(&dir, "report/violin.svg"); - verify_html(&dir, "report/index.html"); + verify_svg(&dir, "report/violin.svg"); + verify_html(&dir, "report/index.html"); + } // Run the final summary process and check for the report that produces short_benchmark(&tempdir).final_summary(); - let dir = tempdir.path().to_owned(); - verify_html(&dir, "report/index.html"); + #[cfg(feature = "html_reports")] + { + let dir = tempdir.path().to_owned(); + verify_html(&dir, "report/index.html"); + } } +#[cfg(feature = "plotters")] #[test] fn test_output_files_flat_sampling() { let tempdir = temp_dir(); @@ -387,24 +400,27 @@ fn test_output_files_flat_sampling() { verify_stats(&dir, "base"); verify_json(&dir, "change/estimates.json"); - verify_svg(&dir, "report/MAD.svg"); - verify_svg(&dir, "report/mean.svg"); - verify_svg(&dir, "report/median.svg"); - verify_svg(&dir, "report/pdf.svg"); - verify_svg(&dir, "report/iteration_times.svg"); - verify_svg(&dir, "report/SD.svg"); - verify_svg(&dir, "report/typical.svg"); - verify_svg(&dir, "report/both/pdf.svg"); - verify_svg(&dir, "report/both/iteration_times.svg"); - verify_svg(&dir, "report/change/mean.svg"); - verify_svg(&dir, "report/change/median.svg"); - verify_svg(&dir, "report/change/t-test.svg"); - - verify_svg(&dir, "report/pdf_small.svg"); - verify_svg(&dir, "report/iteration_times_small.svg"); - verify_svg(&dir, "report/relative_pdf_small.svg"); - verify_svg(&dir, "report/relative_iteration_times_small.svg"); - verify_html(&dir, "report/index.html"); + #[cfg(feature = "html_reports")] + { + verify_svg(&dir, "report/MAD.svg"); + verify_svg(&dir, "report/mean.svg"); + verify_svg(&dir, "report/median.svg"); + verify_svg(&dir, "report/pdf.svg"); + verify_svg(&dir, "report/iteration_times.svg"); + verify_svg(&dir, "report/SD.svg"); + verify_svg(&dir, "report/typical.svg"); + verify_svg(&dir, "report/both/pdf.svg"); + verify_svg(&dir, "report/both/iteration_times.svg"); + verify_svg(&dir, "report/change/mean.svg"); + verify_svg(&dir, "report/change/median.svg"); + verify_svg(&dir, "report/change/t-test.svg"); + + verify_svg(&dir, "report/pdf_small.svg"); + verify_svg(&dir, "report/iteration_times_small.svg"); + verify_svg(&dir, "report/relative_pdf_small.svg"); + verify_svg(&dir, "report/relative_iteration_times_small.svg"); + verify_html(&dir, "report/index.html"); + } } #[test] From e8bb66dcbf38e6c1fd9d0ddb439626157b6bc217 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Tue, 27 Jul 2021 09:29:26 +0800 Subject: [PATCH 06/29] Don't try to copy CSV files that weren't generated. (#499) --- src/analysis/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs index 1cb58285e..ab44fb827 100644 --- a/src/analysis/mod.rs +++ b/src/analysis/mod.rs @@ -365,5 +365,6 @@ fn copy_new_dir_to_base(id: &str, baseline: &str, output_directory: &Path) { &new_dir.join("benchmark.json"), &base_dir.join("benchmark.json") )); + #[cfg(feature = "csv_output")] try_else_return!(fs::cp(&new_dir.join("raw.csv"), &base_dir.join("raw.csv"))); } From 4b8018795c65ef388aa02be4fa736a76bdcc4c55 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Tue, 27 Jul 2021 16:49:39 +0800 Subject: [PATCH 07/29] Use terminal code, send results to stdout, and messages to stderr. (#500) * Use terminal code to clear the line. * Print status messages to stderr. Status messages such as 'Warming up', 'Analyzing', etc. * Use the 'anes' library for handling terminal escape codes. --- Cargo.toml | 1 + src/report.rs | 71 ++++++++++++++++++++------------------------------- 2 files changed, 29 insertions(+), 43 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a85276a61..7a8b76330 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ license = "Apache-2.0/MIT" exclude = ["book/*"] [dependencies] +anes = "0.1" lazy_static = "1.4" criterion-plot = { path = "plot", version = "0.4.3" } itertools = "0.10" diff --git a/src/report.rs b/src/report.rs index d51930f0b..d75c739c8 100644 --- a/src/report.rs +++ b/src/report.rs @@ -10,11 +10,11 @@ use crate::measurement::ValueFormatter; use crate::stats::univariate::Sample; use crate::stats::Distribution; use crate::{PlotConfiguration, Throughput}; -use std::cell::Cell; +use anes::{Attribute, ClearLine, Color, ResetAttributes, SetAttribute, SetForegroundColor}; use std::cmp; use std::collections::HashSet; use std::fmt; -use std::io::stdout; +use std::io::stderr; use std::io::Write; use std::path::{Path, PathBuf}; @@ -369,8 +369,6 @@ pub(crate) struct CliReport { pub enable_text_overwrite: bool, pub enable_text_coloring: bool, pub verbose: bool, - - last_line_len: Cell, } impl CliReport { pub fn new( @@ -382,18 +380,12 @@ impl CliReport { enable_text_overwrite, enable_text_coloring, verbose, - - last_line_len: Cell::new(0), } } fn text_overwrite(&self) { if self.enable_text_overwrite { - print!("\r"); - for _ in 0..self.last_line_len.get() { - print!(" "); - } - print!("\r"); + eprint!("\r{}", ClearLine::All) } } @@ -401,41 +393,36 @@ impl CliReport { #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))] fn print_overwritable(&self, s: String) { if self.enable_text_overwrite { - self.last_line_len.set(s.len()); - print!("{}", s); - stdout().flush().unwrap(); + eprint!("{}", s); + stderr().flush().unwrap(); } else { - println!("{}", s); + eprintln!("{}", s); } } - fn green(&self, s: String) -> String { + fn with_color(&self, color: Color, s: &str) -> String { if self.enable_text_coloring { - format!("\x1B[32m{}\x1B[39m", s) + format!("{}{}{}", SetForegroundColor(color), s, ResetAttributes) } else { - s + String::from(s) } } - fn yellow(&self, s: String) -> String { - if self.enable_text_coloring { - format!("\x1B[33m{}\x1B[39m", s) - } else { - s - } + fn green(&self, s: &str) -> String { + self.with_color(Color::DarkGreen, s) } - fn red(&self, s: String) -> String { - if self.enable_text_coloring { - format!("\x1B[31m{}\x1B[39m", s) - } else { - s - } + fn yellow(&self, s: &str) -> String { + self.with_color(Color::DarkYellow, s) + } + + fn red(&self, s: &str) -> String { + self.with_color(Color::DarkRed, s) } fn bold(&self, s: String) -> String { if self.enable_text_coloring { - format!("\x1B[1m{}\x1B[22m", s) + format!("{}{}{}", SetAttribute(Attribute::Bold), s, ResetAttributes) } else { s } @@ -443,7 +430,7 @@ impl CliReport { fn faint(&self, s: String) -> String { if self.enable_text_coloring { - format!("\x1B[2m{}\x1B[22m", s) + format!("{}{}{}", SetAttribute(Attribute::Faint), s, ResetAttributes) } else { s } @@ -462,7 +449,7 @@ impl CliReport { println!( "{}", - self.yellow(format!( + self.yellow(&format!( "Found {} outliers among {} measurements ({:.2}%)", noutliers, sample_size, @@ -561,14 +548,14 @@ impl Report for CliReport { let mut id = id.as_title().to_owned(); if id.len() > 23 { - println!("{}", self.green(id.clone())); + println!("{}", self.green(&id)); id.clear(); } let id_len = id.len(); println!( "{}{}time: [{} {} {}]", - self.green(id), + self.green(&id), " ".repeat(24 - id_len), self.faint( formatter.format_value(typical_estimate.confidence_interval.lower_bound) @@ -614,16 +601,14 @@ impl Report for CliReport { let comparison = compare_to_threshold(mean_est, comp.noise_threshold); match comparison { ComparisonResult::Improved => { - point_estimate_str = self.green(self.bold(point_estimate_str)); - thrpt_point_estimate_str = self.green(self.bold(thrpt_point_estimate_str)); - explanation_str = - format!("Performance has {}.", self.green("improved".to_owned())); + point_estimate_str = self.green(&self.bold(point_estimate_str)); + thrpt_point_estimate_str = self.green(&self.bold(thrpt_point_estimate_str)); + explanation_str = format!("Performance has {}.", self.green("improved")); } ComparisonResult::Regressed => { - point_estimate_str = self.red(self.bold(point_estimate_str)); - thrpt_point_estimate_str = self.red(self.bold(thrpt_point_estimate_str)); - explanation_str = - format!("Performance has {}.", self.red("regressed".to_owned())); + point_estimate_str = self.red(&self.bold(point_estimate_str)); + thrpt_point_estimate_str = self.red(&self.bold(thrpt_point_estimate_str)); + explanation_str = format!("Performance has {}.", self.red("regressed")); } ComparisonResult::NonSignificant => { explanation_str = "Change within noise threshold.".to_owned(); From 43f9751a57781bb0603ca523106c6110df11b2f5 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Tue, 27 Jul 2021 18:22:19 +0800 Subject: [PATCH 08/29] Accept subsecond durations from the command line. (#501) * Accept subsecond durations from the command line. * Replace 'to_nanos' with 'as_nanos'. 'as_nanos' stabilized in 1.33. --- src/analysis/mod.rs | 2 +- src/benchmark_group.rs | 6 ++--- src/lib.rs | 42 ++++++++++++--------------------- src/measurement.rs | 3 +-- src/plot/gnuplot_backend/mod.rs | 2 +- src/routine.rs | 14 +++++------ 6 files changed, 28 insertions(+), 41 deletions(-) diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs index ab44fb827..d091f6341 100644 --- a/src/analysis/mod.rs +++ b/src/analysis/mod.rs @@ -26,7 +26,7 @@ macro_rules! elapsed { info!( "{} took {}", $msg, - crate::format::time(crate::DurationExt::to_nanos(elapsed) as f64) + crate::format::time(elapsed.as_nanos() as f64) ); out diff --git a/src/benchmark_group.rs b/src/benchmark_group.rs index 723d01e59..445398cdf 100644 --- a/src/benchmark_group.rs +++ b/src/benchmark_group.rs @@ -6,7 +6,7 @@ use crate::report::BenchmarkId as InternalBenchmarkId; use crate::report::Report; use crate::report::ReportContext; use crate::routine::{Function, Routine}; -use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput}; +use crate::{Bencher, Criterion, Mode, PlotConfiguration, SamplingMode, Throughput}; use std::time::Duration; /// Structure used to group together a set of related benchmarks, along with custom configuration @@ -107,7 +107,7 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> { /// /// Panics if the input duration is zero pub fn warm_up_time(&mut self, dur: Duration) -> &mut Self { - assert!(dur.to_nanos() > 0); + assert!(dur.as_nanos() > 0); self.partial_config.warm_up_time = Some(dur); self @@ -125,7 +125,7 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> { /// /// Panics if the input duration is zero pub fn measurement_time(&mut self, dur: Duration) -> &mut Self { - assert!(dur.to_nanos() > 0); + assert!(dur.as_nanos() > 0); self.partial_config.measurement_time = Some(dur); self diff --git a/src/lib.rs b/src/lib.rs index eca68012a..7a8dfbdac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -393,12 +393,12 @@ impl Default for Criterion { let mut criterion = Criterion { config: BenchmarkConfig { confidence_level: 0.95, - measurement_time: Duration::new(5, 0), + measurement_time: Duration::from_secs(5), noise_threshold: 0.01, nresamples: 100_000, sample_size: 100, significance_level: 0.05, - warm_up_time: Duration::new(3, 0), + warm_up_time: Duration::from_secs(3), sampling_mode: SamplingMode::Auto, }, filter: None, @@ -497,7 +497,7 @@ impl Criterion { /// /// Panics if the input duration is zero pub fn warm_up_time(mut self, dur: Duration) -> Criterion { - assert!(dur.to_nanos() > 0); + assert!(dur.as_nanos() > 0); self.config.warm_up_time = dur; self @@ -514,7 +514,7 @@ impl Criterion { /// /// Panics if the input duration in zero pub fn measurement_time(mut self, dur: Duration) -> Criterion { - assert!(dur.to_nanos() > 0); + assert!(dur.as_nanos() > 0); self.config.measurement_time = dur; self @@ -866,17 +866,17 @@ https://bheisler.github.io/criterion.rs/book/faq.html } else if matches.is_present("list") { Mode::List } else if matches.is_present("profile-time") { - let num_seconds = value_t!(matches.value_of("profile-time"), u64).unwrap_or_else(|e| { + let num_seconds = value_t!(matches.value_of("profile-time"), f64).unwrap_or_else(|e| { println!("{}", e); std::process::exit(1) }); - if num_seconds < 1 { + if num_seconds < 1.0 { println!("Profile time must be at least one second."); std::process::exit(1); } - Mode::Profile(Duration::from_secs(num_seconds)) + Mode::Profile(Duration::from_secs_f64(num_seconds)) } else { Mode::Benchmark }; @@ -963,25 +963,25 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.config.sample_size = num_size; } if matches.is_present("warm-up-time") { - let num_seconds = value_t!(matches.value_of("warm-up-time"), u64).unwrap_or_else(|e| { + let num_seconds = value_t!(matches.value_of("warm-up-time"), f64).unwrap_or_else(|e| { println!("{}", e); std::process::exit(1) }); - let dur = std::time::Duration::new(num_seconds, 0); - assert!(dur.to_nanos() > 0); + let dur = std::time::Duration::from_secs_f64(num_seconds); + assert!(dur.as_nanos() > 0); self.config.warm_up_time = dur; } if matches.is_present("measurement-time") { let num_seconds = - value_t!(matches.value_of("measurement-time"), u64).unwrap_or_else(|e| { + value_t!(matches.value_of("measurement-time"), f64).unwrap_or_else(|e| { println!("{}", e); std::process::exit(1) }); - let dur = std::time::Duration::new(num_seconds, 0); - assert!(dur.to_nanos() > 0); + let dur = std::time::Duration::from_secs_f64(num_seconds); + assert!(dur.as_nanos() > 0); self.config.measurement_time = dur; } @@ -1164,18 +1164,6 @@ where } } -trait DurationExt { - fn to_nanos(&self) -> u64; -} - -const NANOS_PER_SEC: u64 = 1_000_000_000; - -impl DurationExt for Duration { - fn to_nanos(&self) -> u64 { - self.as_secs() * NANOS_PER_SEC + u64::from(self.subsec_nanos()) - } -} - /// Enum representing different ways of measuring the throughput of benchmarked code. /// If the throughput setting is configured for a benchmark then the estimated throughput will /// be reported as well as the time per iteration. @@ -1305,7 +1293,7 @@ impl ActualSamplingMode { ActualSamplingMode::Linear => { let n = sample_count; let met = warmup_mean_execution_time; - let m_ns = target_time.to_nanos(); + let m_ns = target_time.as_nanos(); // Solve: [d + 2*d + 3*d + ... + n*d] * met = m_ns let total_runs = n * (n + 1) / 2; let d = ((m_ns as f64 / met / total_runs as f64).ceil() as u64).max(1); @@ -1333,7 +1321,7 @@ impl ActualSamplingMode { ActualSamplingMode::Flat => { let n = sample_count; let met = warmup_mean_execution_time; - let m_ns = target_time.to_nanos() as f64; + let m_ns = target_time.as_nanos() as f64; let time_per_sample = m_ns / (n as f64); // This is pretty simplistic; we could do something smarter to fit into the allotted time. let iterations_per_sample = ((time_per_sample / met).ceil() as u64).max(1); diff --git a/src/measurement.rs b/src/measurement.rs index e253670fc..62e2c5e9e 100644 --- a/src/measurement.rs +++ b/src/measurement.rs @@ -4,7 +4,6 @@ //! measurement. use crate::format::short; -use crate::DurationExt; use crate::Throughput; use std::time::{Duration, Instant}; @@ -204,7 +203,7 @@ impl Measurement for WallTime { Duration::from_secs(0) } fn to_f64(&self, val: &Self::Value) -> f64 { - val.to_nanos() as f64 + val.as_nanos() as f64 } fn formatter(&self) -> &dyn ValueFormatter { &DurationFormatter diff --git a/src/plot/gnuplot_backend/mod.rs b/src/plot/gnuplot_backend/mod.rs index 95e07efce..e7f2ae7ed 100644 --- a/src/plot/gnuplot_backend/mod.rs +++ b/src/plot/gnuplot_backend/mod.rs @@ -248,7 +248,7 @@ impl Plotter for Gnuplot { info!( "Waiting for {} gnuplot processes took {}", child_count, - format::time(crate::DurationExt::to_nanos(elapsed) as f64) + format::time(elapsed.as_nanos() as f64) ); } } diff --git a/src/routine.rs b/src/routine.rs index 5831415ac..b2ad51595 100644 --- a/src/routine.rs +++ b/src/routine.rs @@ -2,7 +2,7 @@ use crate::benchmark::BenchmarkConfig; use crate::connection::OutgoingMessage; use crate::measurement::Measurement; use crate::report::{BenchmarkId, Report, ReportContext}; -use crate::{ActualSamplingMode, Bencher, Criterion, DurationExt}; +use crate::{ActualSamplingMode, Bencher, Criterion}; use std::marker::PhantomData; use std::time::Duration; @@ -34,7 +34,7 @@ pub(crate) trait Routine { ) { criterion .report - .profile(id, report_context, time.to_nanos() as f64); + .profile(id, report_context, time.as_nanos() as f64); let mut profile_path = report_context.output_directory.clone(); if (*crate::CARGO_CRITERION_CONNECTION).is_some() { @@ -51,7 +51,7 @@ pub(crate) trait Routine { .borrow_mut() .start_profiling(id.id(), &profile_path); - let time = time.to_nanos(); + let time = time.as_nanos() as u64; // TODO: Some profilers will show the two batches of iterations as // being different code-paths even though they aren't really. @@ -89,16 +89,16 @@ pub(crate) trait Routine { parameter: &T, ) -> (ActualSamplingMode, Box<[f64]>, Box<[f64]>) { let wu = config.warm_up_time; - let m_ns = config.measurement_time.to_nanos(); + let m_ns = config.measurement_time.as_nanos(); criterion .report - .warmup(id, report_context, wu.to_nanos() as f64); + .warmup(id, report_context, wu.as_nanos() as f64); if let Some(conn) = &criterion.connection { conn.send(&OutgoingMessage::Warmup { id: id.into(), - nanos: wu.to_nanos() as f64, + nanos: wu.as_nanos() as f64, }) .unwrap(); } @@ -233,7 +233,7 @@ where total_iters += b.iters; elapsed_time += b.elapsed_time; if elapsed_time > how_long { - return (elapsed_time.to_nanos(), total_iters); + return (elapsed_time.as_nanos() as u64, total_iters); } b.iters = b.iters.wrapping_mul(2); From c1b3466245c97eb06739e9454f8c3bcdc5649595 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Tue, 27 Jul 2021 18:48:23 +0800 Subject: [PATCH 09/29] Remove the deprecated 'iter_with_large_setup'. (#502) --- benches/benchmarks/iter_with_large_setup.rs | 9 ++++----- benches/benchmarks/measurement_overhead.rs | 2 +- src/bencher.rs | 9 --------- tests/criterion_tests.rs | 2 +- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/benches/benchmarks/iter_with_large_setup.rs b/benches/benchmarks/iter_with_large_setup.rs index 01b6e9cd6..6578951c4 100644 --- a/benches/benchmarks/iter_with_large_setup.rs +++ b/benches/benchmarks/iter_with_large_setup.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, Criterion, Throughput}; +use criterion::{criterion_group, BatchSize, Criterion, Throughput}; use std::time::Duration; const SIZE: usize = 1024 * 1024; @@ -7,10 +7,10 @@ fn large_setup(c: &mut Criterion) { let mut group = c.benchmark_group("iter_with_large_setup"); group.throughput(Throughput::Bytes(SIZE as u64)); group.bench_function("large_setup", |b| { - // NOTE: iter_with_large_setup is deprecated. Use iter_batched instead. - b.iter_with_large_setup( + b.iter_batched( || (0..SIZE).map(|i| i as u8).collect::>(), |v| v.clone(), + BatchSize::NumBatches(1), ) }); } @@ -18,8 +18,7 @@ fn large_setup(c: &mut Criterion) { fn small_setup(c: &mut Criterion) { let mut group = c.benchmark_group("iter_with_large_setup"); group.bench_function("small_setup", |b| { - // NOTE: iter_with_large_setup is deprecated. Use iter_batched instead. - b.iter_with_large_setup(|| SIZE, |size| size) + b.iter_batched(|| SIZE, |size| size, BatchSize::NumBatches(1)) }); } diff --git a/benches/benchmarks/measurement_overhead.rs b/benches/benchmarks/measurement_overhead.rs index 15b243dab..c424efb01 100644 --- a/benches/benchmarks/measurement_overhead.rs +++ b/benches/benchmarks/measurement_overhead.rs @@ -5,7 +5,7 @@ fn some_benchmark(c: &mut Criterion) { group.bench_function("iter", |b| b.iter(|| 1)); group.bench_function("iter_with_setup", |b| b.iter_with_setup(|| (), |_| 1)); group.bench_function("iter_with_large_setup", |b| { - b.iter_with_large_setup(|| (), |_| 1) + b.iter_batched(|| (), |_| 1, BatchSize::NumBatches(1)) }); group.bench_function("iter_with_large_drop", |b| b.iter_with_large_drop(|| 1)); group.bench_function("iter_batched_small_input", |b| { diff --git a/src/bencher.rs b/src/bencher.rs index fa9ad3ddd..9f99738ff 100644 --- a/src/bencher.rs +++ b/src/bencher.rs @@ -189,15 +189,6 @@ impl<'a, M: Measurement> Bencher<'a, M> { self.iter_batched(|| (), |_| routine(), BatchSize::SmallInput); } - #[doc(hidden)] - pub fn iter_with_large_setup(&mut self, setup: S, routine: R) - where - S: FnMut() -> I, - R: FnMut(I) -> O, - { - self.iter_batched(setup, routine, BatchSize::NumBatches(1)); - } - /// Times a `routine` that requires some input by generating a batch of input, then timing the /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for /// details on choosing the batch size. Use this when the routine must consume its input. diff --git a/tests/criterion_tests.rs b/tests/criterion_tests.rs index a2fef0e1a..2576a7998 100644 --- a/tests/criterion_tests.rs +++ b/tests/criterion_tests.rs @@ -274,7 +274,7 @@ fn test_timing_loops() { b.iter_with_setup(|| vec![10], |v| v[0]) }); group.bench_function("iter_with_large_setup", |b| { - b.iter_with_large_setup(|| vec![10], |v| v[0]) + b.iter_batched(|| vec![10], |v| v[0], BatchSize::NumBatches(1)) }); group.bench_function("iter_with_large_drop", |b| { b.iter_with_large_drop(|| vec![10; 100]) From 6d3161edbc9e28963b4d2d6bf87a262045770d2b Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Tue, 27 Jul 2021 19:38:10 +0800 Subject: [PATCH 10/29] Test WASI functionality in CI run. (#503) --- .github/workflows/ci.yaml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 92ed6e78f..a615440f5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 + name: Setup rust toolchain with: profile: minimal toolchain: ${{ matrix.rust }} @@ -31,35 +32,56 @@ jobs: components: rustfmt, clippy - uses: Swatinem/rust-cache@v1 + name: Load dependencies from cache - uses: actions-rs/cargo@v1 + name: Build with stable features with: command: build args: --features stable - uses: actions-rs/cargo@v1 + if: ${{ matrix.rust == 'nightly' }} + name: Build with unstable features + with: + command: build + args: --all-features + + - uses: actions-rs/cargo@v1 + name: Build with minimal features with: command: build args: --no-default-features - uses: actions-rs/cargo@v1 + name: Test with stable features with: command: test args: --features stable - uses: actions-rs/cargo@v1 + name: Test with minimal features with: command: test args: --no-default-features - uses: actions-rs/cargo@v1 + name: Check for non-standard formatting if: ${{ matrix.rust == 'stable' }} with: command: fmt args: --all -- --check - uses: actions-rs/cargo@v1 - if: ${{ matrix.rust != '1.40.0' }} # 1.40 has horrible lints. + name: Check for clippy hints with: command: clippy args: -- -D warnings + + - name: Test run targeting WASI + run: | + curl https://wasmtime.dev/install.sh -sSf | bash + source ~/.bashrc + export PATH=$HOME/.wasmtime/bin/:$PATH + cargo install cargo-wasi + cargo wasi bench --no-default-features -- --test From f862a1bbb60a19e57fccd11d0dfe1e7a00692db3 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Wed, 28 Jul 2021 12:38:43 +0800 Subject: [PATCH 11/29] Remove a few unsafe ops since the compiler can prove they are safe. (#505) --- src/stats/univariate/percentiles.rs | 22 +++++++++------------- src/stats/univariate/sample.rs | 1 + 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/stats/univariate/percentiles.rs b/src/stats/univariate/percentiles.rs index be6bcf391..39def18e7 100644 --- a/src/stats/univariate/percentiles.rs +++ b/src/stats/univariate/percentiles.rs @@ -54,27 +54,23 @@ where /// Returns the interquartile range pub fn iqr(&self) -> A { - unsafe { - let q1 = self.at_unchecked(A::cast(25)); - let q3 = self.at_unchecked(A::cast(75)); + let q1 = self.at(A::cast(25)); + let q3 = self.at(A::cast(75)); - q3 - q1 - } + q3 - q1 } /// Returns the 50th percentile pub fn median(&self) -> A { - unsafe { self.at_unchecked(A::cast(50)) } + self.at(A::cast(50)) } /// Returns the 25th, 50th and 75th percentiles pub fn quartiles(&self) -> (A, A, A) { - unsafe { - ( - self.at_unchecked(A::cast(25)), - self.at_unchecked(A::cast(50)), - self.at_unchecked(A::cast(75)), - ) - } + ( + self.at(A::cast(25)), + self.at(A::cast(50)), + self.at(A::cast(75)), + ) } } diff --git a/src/stats/univariate/sample.rs b/src/stats/univariate/sample.rs index 506cd0484..6fbb4fb2d 100644 --- a/src/stats/univariate/sample.rs +++ b/src/stats/univariate/sample.rs @@ -13,6 +13,7 @@ use rayon::prelude::*; /// /// - The sample contains at least 2 data points /// - The sample contains no `NaN`s +#[repr(transparent)] pub struct Sample([A]); // TODO(rust-lang/rfcs#735) move this `impl` into a private percentiles module From 791a97ffc3cebac6fd8d583efe19e494f02486e2 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Tue, 10 Aug 2021 22:07:14 +0800 Subject: [PATCH 12/29] Fix trailing semicolon in macros. --- src/macros_private.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/macros_private.rs b/src/macros_private.rs index 982602fa3..26203d1da 100644 --- a/src/macros_private.rs +++ b/src/macros_private.rs @@ -16,7 +16,7 @@ macro_rules! log_if_err { /// be passed as second parameter. macro_rules! try_else_return { ($x:expr) => { - try_else_return!($x, || {}); + try_else_return!($x, || {}) }; ($x:expr, $el:expr) => { match $x { @@ -33,7 +33,7 @@ macro_rules! try_else_return { /// Print an error message to stdout. Format is the same as println! or format! macro_rules! error { ($($arg:tt)*) => ( - println!("Criterion.rs ERROR: {}", &format!($($arg)*)); + println!("Criterion.rs ERROR: {}", &format!($($arg)*)) ) } @@ -41,7 +41,7 @@ macro_rules! error { macro_rules! info { ($($arg:tt)*) => ( if $crate::debug_enabled() { - println!("Criterion.rs DEBUG: {}", &format!($($arg)*)); + println!("Criterion.rs DEBUG: {}", &format!($($arg)*)) } ) } From 7b8030b1223adacebf81b8e1cb3286a83e7a2470 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Tue, 10 Aug 2021 22:39:12 +0800 Subject: [PATCH 13/29] Add --quiet flag for printing a single line per benchmark. (#510) * Add --quiet flag. When enabled, only a single line is printed per benchmark. * Print warnings and errors to stderr. --- src/benchmark_group.rs | 4 +- src/lib.rs | 47 +++++++---- src/report.rs | 186 ++++++++++++++++++++++------------------- 3 files changed, 131 insertions(+), 106 deletions(-) diff --git a/src/benchmark_group.rs b/src/benchmark_group.rs index 445398cdf..dae1652a4 100644 --- a/src/benchmark_group.rs +++ b/src/benchmark_group.rs @@ -145,7 +145,7 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> { pub fn nresamples(&mut self, n: usize) -> &mut Self { assert!(n > 0); if n <= 1000 { - println!("\nWarning: It is not recommended to reduce nresamples below 1000."); + eprintln!("\nWarning: It is not recommended to reduce nresamples below 1000."); } self.partial_config.nresamples = Some(n); @@ -182,7 +182,7 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> { pub fn confidence_level(&mut self, cl: f64) -> &mut Self { assert!(cl > 0.0 && cl < 1.0); if cl < 0.5 { - println!("\nWarning: It is not recommended to reduce confidence level below 0.5."); + eprintln!("\nWarning: It is not recommended to reduce confidence level below 0.5."); } self.partial_config.confidence_level = Some(cl); diff --git a/src/lib.rs b/src/lib.rs index 7a8dfbdac..8e42a773b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -94,7 +94,7 @@ use crate::measurement::{Measurement, WallTime}; use crate::plot::PlottersBackend; use crate::plot::{Gnuplot, Plotter}; use crate::profiler::{ExternalProfiler, Profiler}; -use crate::report::{BencherReport, CliReport, Report, ReportContext, Reports}; +use crate::report::{BencherReport, CliReport, CliVerbosity, Report, ReportContext, Reports}; #[cfg(feature = "async")] pub use crate::bencher::AsyncBencher; @@ -383,7 +383,7 @@ impl Default for Criterion { fn default() -> Criterion { let reports = Reports { cli_enabled: true, - cli: CliReport::new(false, false, false), + cli: CliReport::new(false, false, CliVerbosity::Normal), bencher_enabled: false, bencher: BencherReport, html: DEFAULT_PLOTTING_BACKEND.create_plotter().map(Html::new), @@ -534,7 +534,7 @@ impl Criterion { pub fn nresamples(mut self, n: usize) -> Criterion { assert!(n > 0); if n <= 1000 { - println!("\nWarning: It is not recommended to reduce nresamples below 1000."); + eprintln!("\nWarning: It is not recommended to reduce nresamples below 1000."); } self.config.nresamples = n; @@ -571,7 +571,7 @@ impl Criterion { pub fn confidence_level(mut self, cl: f64) -> Criterion { assert!(cl > 0.0 && cl < 1.0); if cl < 0.5 { - println!("\nWarning: It is not recommended to reduce confidence level below 0.5."); + eprintln!("\nWarning: It is not recommended to reduce confidence level below 0.5."); } self.config.confidence_level = cl; @@ -716,6 +716,10 @@ impl Criterion { .short("v") .long("verbose") .help("Print additional statistical information.")) + .arg(Arg::with_name("quiet") + .long("quiet") + .conflicts_with("verbose") + .help("Print only the benchmark results.")) .arg(Arg::with_name("noplot") .short("n") .long("noplot") @@ -823,21 +827,21 @@ https://bheisler.github.io/criterion.rs/book/faq.html if self.connection.is_some() { if let Some(color) = matches.value_of("color") { if color != "auto" { - println!("Warning: --color will be ignored when running with cargo-criterion. Use `cargo criterion --color {} -- ` instead.", color); + eprintln!("Warning: --color will be ignored when running with cargo-criterion. Use `cargo criterion --color {} -- ` instead.", color); } } if matches.is_present("verbose") { - println!("Warning: --verbose will be ignored when running with cargo-criterion. Use `cargo criterion --output-format verbose -- ` instead."); + eprintln!("Warning: --verbose will be ignored when running with cargo-criterion. Use `cargo criterion --output-format verbose -- ` instead."); } if matches.is_present("noplot") { - println!("Warning: --noplot will be ignored when running with cargo-criterion. Use `cargo criterion --plotting-backend disabled -- ` instead."); + eprintln!("Warning: --noplot will be ignored when running with cargo-criterion. Use `cargo criterion --plotting-backend disabled -- ` instead."); } if let Some(backend) = matches.value_of("plotting-backend") { - println!("Warning: --plotting-backend will be ignored when running with cargo-criterion. Use `cargo criterion --plotting-backend {} -- ` instead.", backend); + eprintln!("Warning: --plotting-backend will be ignored when running with cargo-criterion. Use `cargo criterion --plotting-backend {} -- ` instead.", backend); } if let Some(format) = matches.value_of("output-format") { if format != "criterion" { - println!("Warning: --output-format will be ignored when running with cargo-criterion. Use `cargo criterion --output-format {} -- ` instead.", format); + eprintln!("Warning: --output-format will be ignored when running with cargo-criterion. Use `cargo criterion --output-format {} -- ` instead.", format); } } @@ -848,7 +852,7 @@ https://bheisler.github.io/criterion.rs/book/faq.html .unwrap_or(false) || matches.is_present("load-baseline") { - println!("Error: baselines are not supported when running with cargo-criterion."); + eprintln!("Error: baselines are not supported when running with cargo-criterion."); std::process::exit(1); } } @@ -872,7 +876,7 @@ https://bheisler.github.io/criterion.rs/book/faq.html }); if num_seconds < 1.0 { - println!("Profile time must be at least one second."); + eprintln!("Profile time must be at least one second."); std::process::exit(1); } @@ -928,6 +932,13 @@ https://bheisler.github.io/criterion.rs/book/faq.html } _ => { let verbose = matches.is_present("verbose"); + let verbosity = if verbose { + CliVerbosity::Verbose + } else if matches.is_present("quiet") { + CliVerbosity::Quiet + } else { + CliVerbosity::Normal + }; let stdout_isatty = atty::is(atty::Stream::Stdout); let mut enable_text_overwrite = stdout_isatty && !verbose && !debug_enabled(); let enable_text_coloring; @@ -944,7 +955,7 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.report.bencher_enabled = false; self.report.cli_enabled = true; self.report.cli = - CliReport::new(enable_text_overwrite, enable_text_coloring, verbose); + CliReport::new(enable_text_overwrite, enable_text_coloring, verbosity); } }; } @@ -1303,16 +1314,16 @@ impl ActualSamplingMode { let recommended_sample_size = ActualSamplingMode::recommend_linear_sample_size(m_ns as f64, met); let actual_time = Duration::from_nanos(expected_ns as u64); - print!("\nWarning: Unable to complete {} samples in {:.1?}. You may wish to increase target time to {:.1?}", + eprint!("\nWarning: Unable to complete {} samples in {:.1?}. You may wish to increase target time to {:.1?}", n, target_time, actual_time); if recommended_sample_size != n { - println!( + eprintln!( ", enable flat sampling, or reduce sample count to {}.", recommended_sample_size ); } else { - println!(" or enable flat sampling."); + eprintln!(" or enable flat sampling."); } } @@ -1332,13 +1343,13 @@ impl ActualSamplingMode { let recommended_sample_size = ActualSamplingMode::recommend_flat_sample_size(m_ns, met); let actual_time = Duration::from_nanos(expected_ns as u64); - print!("\nWarning: Unable to complete {} samples in {:.1?}. You may wish to increase target time to {:.1?}", + eprint!("\nWarning: Unable to complete {} samples in {:.1?}. You may wish to increase target time to {:.1?}", n, target_time, actual_time); if recommended_sample_size != n { - println!(", or reduce sample count to {}.", recommended_sample_size); + eprintln!(", or reduce sample count to {}.", recommended_sample_size); } else { - println!("."); + eprintln!("."); } } diff --git a/src/report.rs b/src/report.rs index d75c739c8..a379cca16 100644 --- a/src/report.rs +++ b/src/report.rs @@ -365,21 +365,28 @@ impl Report for Reports { reports_impl!(fn group_separator(&self, )); } +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum CliVerbosity { + Quiet, + Normal, + Verbose, +} + pub(crate) struct CliReport { pub enable_text_overwrite: bool, pub enable_text_coloring: bool, - pub verbose: bool, + pub verbosity: CliVerbosity, } impl CliReport { pub fn new( enable_text_overwrite: bool, enable_text_coloring: bool, - verbose: bool, + verbosity: CliVerbosity, ) -> CliReport { CliReport { enable_text_overwrite, enable_text_coloring, - verbose, + verbosity, } } @@ -518,7 +525,7 @@ impl Report for CliReport { iter_count: u64, ) { self.text_overwrite(); - let iter_string = if self.verbose { + let iter_string = if matches!(self.verbosity, CliVerbosity::Verbose) { format!("{} iterations", iter_count) } else { format::iter_count(iter_count) @@ -583,96 +590,103 @@ impl Report for CliReport { ) } - if let Some(ref comp) = meas.comparison { - let different_mean = comp.p_value < comp.significance_threshold; - let mean_est = &comp.relative_estimates.mean; - let point_estimate = mean_est.point_estimate; - let mut point_estimate_str = format::change(point_estimate, true); - // The change in throughput is related to the change in timing. Reducing the timing by - // 50% increases the throughput by 100%. - let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0; - let mut thrpt_point_estimate_str = - format::change(to_thrpt_estimate(point_estimate), true); - let explanation_str: String; - - if !different_mean { - explanation_str = "No change in performance detected.".to_owned(); - } else { - let comparison = compare_to_threshold(mean_est, comp.noise_threshold); - match comparison { - ComparisonResult::Improved => { - point_estimate_str = self.green(&self.bold(point_estimate_str)); - thrpt_point_estimate_str = self.green(&self.bold(thrpt_point_estimate_str)); - explanation_str = format!("Performance has {}.", self.green("improved")); - } - ComparisonResult::Regressed => { - point_estimate_str = self.red(&self.bold(point_estimate_str)); - thrpt_point_estimate_str = self.red(&self.bold(thrpt_point_estimate_str)); - explanation_str = format!("Performance has {}.", self.red("regressed")); - } - ComparisonResult::NonSignificant => { - explanation_str = "Change within noise threshold.".to_owned(); + if !matches!(self.verbosity, CliVerbosity::Quiet) { + if let Some(ref comp) = meas.comparison { + let different_mean = comp.p_value < comp.significance_threshold; + let mean_est = &comp.relative_estimates.mean; + let point_estimate = mean_est.point_estimate; + let mut point_estimate_str = format::change(point_estimate, true); + // The change in throughput is related to the change in timing. Reducing the timing by + // 50% increases the throughput by 100%. + let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0; + let mut thrpt_point_estimate_str = + format::change(to_thrpt_estimate(point_estimate), true); + let explanation_str: String; + + if !different_mean { + explanation_str = "No change in performance detected.".to_owned(); + } else { + let comparison = compare_to_threshold(mean_est, comp.noise_threshold); + match comparison { + ComparisonResult::Improved => { + point_estimate_str = self.green(&self.bold(point_estimate_str)); + thrpt_point_estimate_str = + self.green(&self.bold(thrpt_point_estimate_str)); + explanation_str = + format!("Performance has {}.", self.green("improved")); + } + ComparisonResult::Regressed => { + point_estimate_str = self.red(&self.bold(point_estimate_str)); + thrpt_point_estimate_str = + self.red(&self.bold(thrpt_point_estimate_str)); + explanation_str = format!("Performance has {}.", self.red("regressed")); + } + ComparisonResult::NonSignificant => { + explanation_str = "Change within noise threshold.".to_owned(); + } } } - } - if meas.throughput.is_some() { - println!("{}change:", " ".repeat(17)); + if meas.throughput.is_some() { + println!("{}change:", " ".repeat(17)); + + println!( + "{}time: [{} {} {}] (p = {:.2} {} {:.2})", + " ".repeat(24), + self.faint(format::change( + mean_est.confidence_interval.lower_bound, + true + )), + point_estimate_str, + self.faint(format::change( + mean_est.confidence_interval.upper_bound, + true + )), + comp.p_value, + if different_mean { "<" } else { ">" }, + comp.significance_threshold + ); + println!( + "{}thrpt: [{} {} {}]", + " ".repeat(24), + self.faint(format::change( + to_thrpt_estimate(mean_est.confidence_interval.upper_bound), + true + )), + thrpt_point_estimate_str, + self.faint(format::change( + to_thrpt_estimate(mean_est.confidence_interval.lower_bound), + true + )), + ); + } else { + println!( + "{}change: [{} {} {}] (p = {:.2} {} {:.2})", + " ".repeat(24), + self.faint(format::change( + mean_est.confidence_interval.lower_bound, + true + )), + point_estimate_str, + self.faint(format::change( + mean_est.confidence_interval.upper_bound, + true + )), + comp.p_value, + if different_mean { "<" } else { ">" }, + comp.significance_threshold + ); + } - println!( - "{}time: [{} {} {}] (p = {:.2} {} {:.2})", - " ".repeat(24), - self.faint(format::change( - mean_est.confidence_interval.lower_bound, - true - )), - point_estimate_str, - self.faint(format::change( - mean_est.confidence_interval.upper_bound, - true - )), - comp.p_value, - if different_mean { "<" } else { ">" }, - comp.significance_threshold - ); - println!( - "{}thrpt: [{} {} {}]", - " ".repeat(24), - self.faint(format::change( - to_thrpt_estimate(mean_est.confidence_interval.upper_bound), - true - )), - thrpt_point_estimate_str, - self.faint(format::change( - to_thrpt_estimate(mean_est.confidence_interval.lower_bound), - true - )), - ); - } else { - println!( - "{}change: [{} {} {}] (p = {:.2} {} {:.2})", - " ".repeat(24), - self.faint(format::change( - mean_est.confidence_interval.lower_bound, - true - )), - point_estimate_str, - self.faint(format::change( - mean_est.confidence_interval.upper_bound, - true - )), - comp.p_value, - if different_mean { "<" } else { ">" }, - comp.significance_threshold - ); + println!("{}{}", " ".repeat(24), explanation_str); } - - println!("{}{}", " ".repeat(24), explanation_str); } - self.outliers(&meas.avg_times); + if !matches!(self.verbosity, CliVerbosity::Quiet) { + self.outliers(&meas.avg_times); + } - if self.verbose { + if matches!(self.verbosity, CliVerbosity::Verbose) { let format_short_estimate = |estimate: &Estimate| -> String { format!( "[{} {}]", From 77f1b38906ffdc3b6edc506ea5aeb50beafa63fe Mon Sep 17 00:00:00 2001 From: Lucas Kent Date: Mon, 15 Nov 2021 13:55:54 +1100 Subject: [PATCH 14/29] Add flag to allow baseline comparisons to succeed when a baseline can not be found --- book/src/user_guide/command_line_options.md | 9 +++--- src/analysis/mod.rs | 2 +- src/lib.rs | 34 +++++++++++++++------ tests/criterion_tests.rs | 15 ++++++--- 4 files changed, 42 insertions(+), 18 deletions(-) diff --git a/book/src/user_guide/command_line_options.md b/book/src/user_guide/command_line_options.md index 99724871c..662bbc737 100644 --- a/book/src/user_guide/command_line_options.md +++ b/book/src/user_guide/command_line_options.md @@ -18,15 +18,16 @@ regular expression matching the benchmark ID. For example, running * To test that the benchmarks run successfully without performing the measurement or analysis (eg. in a CI setting), use `cargo test --benches`. * To override the default plotting backend, use `cargo bench -- --plotting-backend gnuplot` or `cargo bench --plotting-backend plotters`. `gnuplot` is used by default if it is installed. * To change the CLI output format, use `cargo bench -- --output-format `. Supported output formats are: - * `criterion` - Use Criterion's normal output format - * `bencher` - An output format similar to the output produced by the `bencher` crate or nightly `libtest` benchmarks. Though this provides less information than the `criterion` format, it may be useful to support external tools that can parse this output. + * `criterion` - Use Criterion's normal output format + * `bencher` - An output format similar to the output produced by the `bencher` crate or nightly `libtest` benchmarks. Though this provides less information than the `criterion` format, it may be useful to support external tools that can parse this output. ## Baselines By default, Criterion.rs will compare the measurements against the previous run (if any). Sometimes it's useful to keep a set of measurements around for several runs. For example, you might want to make multiple changes to the code while comparing against the master branch. For this situation, Criterion.rs supports custom baselines. -* `--save-baseline ` will compare against the named baseline, then overwrite it. -* `--baseline ` will compare against the named baseline without overwriting it. +* `--save-baseline ` will compare against the named baseline, then overwrite it. +* `--baseline ` will compare against the named baseline without overwriting it. Will fail if the specified baseline is missing any benchmark results. +* `--baseline-lenient ` will compare against the named baseline without overwriting it. Will not fail if the specified baseline is missing any benchmark results. This is useful for automatically comparing benchmark results between branches in CI. * `--load-baseline ` will load the named baseline as the new data set rather than the previous baseline. Using these options, you can manage multiple baseline measurements. For instance, if you want to compare against a static reference point such as the master branch, you might run: diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs index d091f6341..23647c137 100644 --- a/src/analysis/mod.rs +++ b/src/analysis/mod.rs @@ -47,7 +47,7 @@ pub(crate) fn common( ) { criterion.report.benchmark_start(id, report_context); - if let Baseline::Compare = criterion.baseline { + if let Baseline::CompareStrict = criterion.baseline { if !base_dir_exists( id, &criterion.baseline_directory, diff --git a/src/lib.rs b/src/lib.rs index 0f3d7a671..c0a8c1604 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -265,9 +265,12 @@ impl BatchSize { /// Baseline describes how the baseline_directory is handled. #[derive(Debug, Clone, Copy)] pub enum Baseline { - /// Compare ensures a previous saved version of the baseline - /// exists and runs comparison against that. - Compare, + /// CompareLenient compares against a previous saved version of the baseline. + /// If a previous baseline does not exist, the benchmark is run as normal but no comparison occurs. + CompareLenient, + /// CompareStrict compares against a previous saved version of the baseline. + /// If a previous baseline does not exist, a panic occurs. + CompareStrict, /// Save writes the benchmark results to the baseline directory, /// overwriting any results that were previously there. Save, @@ -636,9 +639,13 @@ impl Criterion { } /// Names an explicit baseline and disables overwriting the previous results. - pub fn retain_baseline(mut self, baseline: String) -> Criterion { + pub fn retain_baseline(mut self, baseline: String, strict: bool) -> Criterion { self.baseline_directory = baseline; - self.baseline = Baseline::Compare; + self.baseline = if strict { + Baseline::CompareStrict + } else { + Baseline::CompareLenient + }; self } @@ -734,14 +741,19 @@ impl Criterion { .help("Save results under a named baseline.")) .arg(Arg::with_name("discard-baseline") .long("discard-baseline") - .conflicts_with_all(&["save-baseline", "baseline"]) + .conflicts_with_all(&["save-baseline", "baseline", "baseline-lenient"]) .help("Discard benchmark results.")) .arg(Arg::with_name("baseline") .short("b") .long("baseline") .takes_value(true) - .conflicts_with("save-baseline") - .help("Compare to a named baseline.")) + .conflicts_with_all(&["save-baseline", "baseline-lenient"]) + .help("Compare to a named baseline. If any benchmarks do not have the specified baseline this command fails.")) + .arg(Arg::with_name("baseline-lenient") + .long("baseline-lenient") + .takes_value(true) + .conflicts_with_all(&["save-baseline", "baseline"]) + .help("Compare to a named baseline. If any benchmarks do not have the specified baseline then just those benchmarks are not compared against the baseline while every other benchmark is compared against the baseline.")) .arg(Arg::with_name("list") .long("list") .help("List all benchmarks") @@ -917,7 +929,11 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.baseline = Baseline::Discard; } if let Some(dir) = matches.value_of("baseline") { - self.baseline = Baseline::Compare; + self.baseline = Baseline::CompareStrict; + self.baseline_directory = dir.to_owned(); + } + if let Some(dir) = matches.value_of("baseline-lenient") { + self.baseline = Baseline::CompareLenient; self.baseline_directory = dir.to_owned(); } diff --git a/tests/criterion_tests.rs b/tests/criterion_tests.rs index 2576a7998..8c3c81a50 100644 --- a/tests/criterion_tests.rs +++ b/tests/criterion_tests.rs @@ -165,7 +165,7 @@ fn test_retain_baseline() { let pre_modified = latest_modified(&dir.path().join("test_retain_baseline/some-baseline")); short_benchmark(&dir) - .retain_baseline("some-baseline".to_owned()) + .retain_baseline("some-baseline".to_owned(), true) .bench_function("test_retain_baseline", |b| b.iter(|| 10)); let post_modified = latest_modified(&dir.path().join("test_retain_baseline/some-baseline")); @@ -175,11 +175,18 @@ fn test_retain_baseline() { #[test] #[should_panic(expected = "Baseline 'some-baseline' must exist before comparison is allowed")] -fn test_compare_baseline() { - // Initial benchmark to populate +fn test_compare_baseline_strict_panics_when_missing_baseline() { + let dir = temp_dir(); + short_benchmark(&dir) + .retain_baseline("some-baseline".to_owned(), true) + .bench_function("test_compare_baseline", |b| b.iter(|| 10)); +} + +#[test] +fn test_compare_baseline_lenient_when_missing_baseline() { let dir = temp_dir(); short_benchmark(&dir) - .retain_baseline("some-baseline".to_owned()) + .retain_baseline("some-baseline".to_owned(), false) .bench_function("test_compare_baseline", |b| b.iter(|| 10)); } From 412591edec4a0e8e05093c7f9ca0ba9611e5c17f Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Sun, 2 Jan 2022 19:20:47 +0000 Subject: [PATCH 15/29] Enable --quick mode. (#535) * Enable --quick mode. * Add basic user guide for the new 'quick mode'. --- book/src/SUMMARY.md | 1 + book/src/user_guide/command_line_options.md | 1 + book/src/user_guide/quick_mode.md | 17 +++++++++ src/benchmark.rs | 3 ++ src/lib.rs | 9 +++++ src/routine.rs | 41 +++++++++++++++++++++ 6 files changed, 72 insertions(+) create mode 100644 book/src/user_guide/quick_mode.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index e221a37d2..0ee2005da 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -19,6 +19,7 @@ - [Profiling](./user_guide/profiling.md) - [Custom Test Framework](./user_guide/custom_test_framework.md) - [Benchmarking async functions](./user_guide/benchmarking_async.md) + - [Quick Mode](./user_guide/quick_mode.md) - [cargo-criterion](./cargo_criterion/cargo_criterion.md) - [Configuring cargo-criterion](./cargo_criterion/configuring_cargo_criterion.md) - [External Tools](./cargo_criterion/external_tools.md) diff --git a/book/src/user_guide/command_line_options.md b/book/src/user_guide/command_line_options.md index 662bbc737..a38422fbb 100644 --- a/book/src/user_guide/command_line_options.md +++ b/book/src/user_guide/command_line_options.md @@ -20,6 +20,7 @@ regular expression matching the benchmark ID. For example, running * To change the CLI output format, use `cargo bench -- --output-format `. Supported output formats are: * `criterion` - Use Criterion's normal output format * `bencher` - An output format similar to the output produced by the `bencher` crate or nightly `libtest` benchmarks. Though this provides less information than the `criterion` format, it may be useful to support external tools that can parse this output. +* To run benchmarks quicker but with lower statistical guarantees, use `cargo bench -- --quick` ## Baselines diff --git a/book/src/user_guide/quick_mode.md b/book/src/user_guide/quick_mode.md new file mode 100644 index 000000000..a8322303e --- /dev/null +++ b/book/src/user_guide/quick_mode.md @@ -0,0 +1,17 @@ +## Quick mode + +Quick mode is enabled with the `--quick` flag and tells criterion to stop benchmarks early once the significance level is below a certail value (default 5%, see the `--significance-level` flag). + +Quick mode in criterion works exactly like `tasty-bench` which has a wealth of details: https://github.com/Bodigrim/tasty-bench + +### Statistical model + +1. Set n ← 1. +1. Measure execution time tₙ of n iterations and execution time t₂ₙ of 2n iterations. +1. Find t which minimizes deviation of (nt, 2nt) from (tₙ, t₂ₙ), namely t ← (tₙ + 2t₂ₙ) / 5n. +1. If deviation is small enough (see `--significance-level`) or time has run out (see `--measurement-time`), return t as a mean execution time. +1. Otherwise set n ← 2n and jump back to Step 2. + +### Disclaimer + +Statistics is a tricky matter, there is no one-size-fits-all approach. In the absence of a good theory simplistic approaches are as (un)sound as obscure ones. Those who seek statistical soundness should rather collect raw data and process it themselves using a proper statistical toolbox. Data reported by criterion in quick mode is only of indicative and comparative significance. diff --git a/src/benchmark.rs b/src/benchmark.rs index 3238cdb5b..3a1cb0012 100644 --- a/src/benchmark.rs +++ b/src/benchmark.rs @@ -13,6 +13,7 @@ pub struct BenchmarkConfig { pub significance_level: f64, pub warm_up_time: Duration, pub sampling_mode: SamplingMode, + pub quick_mode: bool, } /// Struct representing a partially-complete per-benchmark configuration. @@ -26,6 +27,7 @@ pub(crate) struct PartialBenchmarkConfig { pub(crate) significance_level: Option, pub(crate) warm_up_time: Option, pub(crate) sampling_mode: Option, + pub(crate) quick_mode: Option, pub(crate) plot_config: PlotConfiguration, } @@ -42,6 +44,7 @@ impl PartialBenchmarkConfig { .unwrap_or(defaults.significance_level), warm_up_time: self.warm_up_time.unwrap_or(defaults.warm_up_time), sampling_mode: self.sampling_mode.unwrap_or(defaults.sampling_mode), + quick_mode: self.quick_mode.unwrap_or(defaults.quick_mode), } } } diff --git a/src/lib.rs b/src/lib.rs index c0a8c1604..ac0a4e5a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -403,6 +403,7 @@ impl Default for Criterion { significance_level: 0.05, warm_up_time: Duration::from_secs(3), sampling_mode: SamplingMode::Auto, + quick_mode: false, }, filter: None, report: reports, @@ -797,6 +798,10 @@ impl Criterion { .long("significance-level") .takes_value(true) .help(&format!("Changes the default significance level for this run. [default: {}]", self.config.significance_level))) + .arg(Arg::with_name("quick") + .long("quick") + .conflicts_with("sample-size") + .help(&format!("Benchmark only until the significance level has been reached [default: {}]", self.config.quick_mode))) .arg(Arg::with_name("test") .hidden(true) .long("test") @@ -1060,6 +1065,10 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.config.significance_level = num_significance_level; } + if matches.is_present("quick") { + self.config.quick_mode = true; + } + self } diff --git a/src/routine.rs b/src/routine.rs index b2ad51595..b4bb7c152 100644 --- a/src/routine.rs +++ b/src/routine.rs @@ -88,6 +88,47 @@ pub(crate) trait Routine { report_context: &ReportContext, parameter: &T, ) -> (ActualSamplingMode, Box<[f64]>, Box<[f64]>) { + if config.quick_mode { + let minimum_bench_duration = Duration::from_millis(100); + let maximum_bench_duration = config.measurement_time; // default: 5 seconds + let target_rel_stdev = config.significance_level; // default: 5%, 0.05 + + use std::time::Instant; + let time_start = Instant::now(); + + let sq = |val| val * val; + let mut n = 1; + let mut t_prev = *self.bench(measurement, &[n], parameter).first().unwrap(); + + // Early exit for extremely long running benchmarks: + if time_start.elapsed() > maximum_bench_duration { + let t_prev = 1_000_000f64; + let iters = vec![n as f64, n as f64].into_boxed_slice(); + let elapsed = vec![t_prev, t_prev].into_boxed_slice(); + return (ActualSamplingMode::Flat, iters, elapsed); + } + + // Main data collection loop. + loop { + let t_now = *self + .bench(measurement, &[n * 2], parameter) + .first() + .unwrap(); + let t = (t_prev + 2. * t_now) / 5.; + let stdev = (sq(t_prev - t) + sq(t_now - 2. * t)).sqrt(); + // println!("Sample: {} {:.2}", n, stdev / t); + let elapsed = time_start.elapsed(); + if (stdev < target_rel_stdev * t && elapsed > minimum_bench_duration) + || elapsed > maximum_bench_duration + { + let iters = vec![n as f64, (n * 2) as f64].into_boxed_slice(); + let elapsed = vec![t_prev, t_now].into_boxed_slice(); + return (ActualSamplingMode::Linear, iters, elapsed); + } + n *= 2; + t_prev = t_now; + } + } let wu = config.warm_up_time; let m_ns = config.measurement_time.as_nanos(); From a9c42f01637a06bc15a4eec00d5e956df7eea6c3 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Sun, 13 Feb 2022 11:11:49 +0000 Subject: [PATCH 16/29] Compare baselines with critcmp. (#540) --- Cargo.toml | 3 + book/src/SUMMARY.md | 1 + book/src/user_guide/tabulating_results.md | 294 ++++++++++++++++++++++ plot/src/lib.rs | 2 +- src/connection.rs | 2 +- src/critcmp/app.rs | 148 +++++++++++ src/critcmp/data.rs | 227 +++++++++++++++++ src/critcmp/main.rs | 131 ++++++++++ src/critcmp/mod.rs | 5 + src/critcmp/output.rs | 234 +++++++++++++++++ src/lib.rs | 93 +++++++ src/plot/gnuplot_backend/mod.rs | 2 +- src/plot/plotters_backend/summary.rs | 7 +- 13 files changed, 1142 insertions(+), 7 deletions(-) create mode 100644 book/src/user_guide/tabulating_results.md create mode 100644 src/critcmp/app.rs create mode 100644 src/critcmp/data.rs create mode 100644 src/critcmp/main.rs create mode 100644 src/critcmp/mod.rs create mode 100644 src/critcmp/output.rs diff --git a/Cargo.toml b/Cargo.toml index 42d8f4360..dbeb28c08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,9 @@ cast = "0.2" num-traits = { version = "0.2", default-features = false, features = ["std"] } oorandom = "11.1" regex = { version = "1.3", default-features = false, features = ["std"] } +tabwriter = "1.2.1" +termcolor = "1.1.2" +unicode-width = "0.1.9" # Optional dependencies rayon = { version = "1.3", optional = true } diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 0ee2005da..5bdfce2bc 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -20,6 +20,7 @@ - [Custom Test Framework](./user_guide/custom_test_framework.md) - [Benchmarking async functions](./user_guide/benchmarking_async.md) - [Quick Mode](./user_guide/quick_mode.md) + - [Tabulating Results](./user_guide/tabulating_results.md) - [cargo-criterion](./cargo_criterion/cargo_criterion.md) - [Configuring cargo-criterion](./cargo_criterion/configuring_cargo_criterion.md) - [External Tools](./cargo_criterion/external_tools.md) diff --git a/book/src/user_guide/tabulating_results.md b/book/src/user_guide/tabulating_results.md new file mode 100644 index 000000000..2e86f0d0b --- /dev/null +++ b/book/src/user_guide/tabulating_results.md @@ -0,0 +1,294 @@ + + + +# Tabulating Results + +Criterion can save the results of different benchmark runs and +tabulate the results, making it easier to spot performance changes. + +The set of results from a benchmark run is called a `baseline` and each +`baseline` has a name. By default, the most recent run is named `base` but this +can be changed with the `--save-baseline {name}` flag. There's also a special +baseline called `new` which refers to the most recent set of results. + +## Comparing profiles + +Cargo supports custom +[profiles](https://doc.rust-lang.org/cargo/reference/profiles.html) for +controlling the level of optimizations, debug assertions, overflow checks, and +link-time-optmizations. We can use criterion to benchmark different profiles and +tabulate the results to visualize the changes. Let's use the `base64` crate as +an example: + +```bash +> git clone https://github.com/KokaKiwi/rust-hex.git +> cd rust-hex/ +``` + +Now that we've clone the repository, we can generate the first set of benchmark results: + +```bash +> cargo bench --profile=release `# Use the 'release' profile` \ + --bench=hex `# Select the 'hex' binary` \ + -- `# Switch args from cargo to criterion` \ + --save-baseline release `# Save the baseline under 'release'` +``` + +Once the run is complete (this should take a few minutes), we can benchmark the other profile: + +```bash +> cargo bench --profile=dev `# Use the 'dev' profile` \ + --bench=benchmarks `# Select the 'hex' binary` \ + -- `# Switch args from cargo to criterion` \ + --save-baseline dev `# Save the baseline under 'dev'` +``` + +Finally we can compare the two benchmark runs (scroll to the right to see all columns): + +```bash +> cargo bench --bench=hex -- --compare --baselines=dev,release +``` + +
group                          dev                                               release
+-----                          ---                                               -------
+faster_hex_decode              239.50  847.6±16.54µs        ? ?/sec    1.00      3.5±0.01µs        ? ?/sec
+faster_hex_decode_fallback     52.58   567.7±8.36µs        ? ?/sec     1.00     10.8±0.04µs        ? ?/sec
+faster_hex_decode_unchecked    400.98   503.7±3.48µs        ? ?/sec    1.00   1256.2±1.57ns        ? ?/sec
+faster_hex_encode              259.95   244.5±2.04µs        ? ?/sec    1.00    940.5±4.64ns        ? ?/sec
+faster_hex_encode_fallback     50.60   565.1±3.41µs        ? ?/sec     1.00     11.2±0.02µs        ? ?/sec
+hex_decode                     25.27     3.0±0.01ms        ? ?/sec     1.00    119.3±0.17µs        ? ?/sec
+hex_encode                     23.99 1460.8±18.11µs        ? ?/sec     1.00     60.9±0.08µs        ? ?/sec
+rustc_hex_decode               28.79     3.1±0.02ms        ? ?/sec     1.00    107.4±0.40µs        ? ?/sec
+rustc_hex_encode               25.80  1385.4±4.37µs        ? ?/sec     1.00    53.7±15.63µs        ? ?/sec
+
+ +The first column in the above results has the names of each individual +benchmark. The two other columns (`dev` and `release`) contain the actual +benchmark results. Each baseline column starts with a performance index relative +to the fastest run (eg. `faster_hex_decode` for `dev` has a performance index of +239.50 because it is 239.50 times slower than the `release` build). Next is the +mean execution time plus the standard deviation (eg. 847.6±16.54µs). Lastly +there's an optional throughput. If no throughput data is available, it will be +printed as `? ?/sec`. + +## Compact, list view. + +If horizontal space is limited or if you're comparing more than two baselines, +it can be convenient to arrange the results in a vertical list rather than in a +table. This can be enabled with the `--compare-list` flag: + +``` +faster_hex_decode +----------------- +release 1.00 3.5±0.01µs ? ?/sec +dev 239.50 847.6±16.54µs ? ?/sec + +faster_hex_decode_fallback +-------------------------- +release 1.00 10.8±0.04µs ? ?/sec +dev 52.58 567.7±8.36µs ? ?/sec + +faster_hex_decode_unchecked +--------------------------- +release 1.00 1256.2±1.57ns ? ?/sec +dev 400.98 503.7±3.48µs ? ?/sec + +faster_hex_encode +----------------- +release 1.00 940.5±4.64ns ? ?/sec +dev 259.95 244.5±2.04µs ? ?/sec + +faster_hex_encode_fallback +-------------------------- +release 1.00 11.2±0.02µs ? ?/sec +dev 50.60 565.1±3.41µs ? ?/sec + +hex_decode +---------- +release 1.00 119.3±0.17µs ? ?/sec +dev 25.27 3.0±0.01ms ? ?/sec + +hex_encode +---------- +release 1.00 60.9±0.08µs ? ?/sec +dev 23.99 1460.8±18.11µs ? ?/sec + +rustc_hex_decode +---------------- +release 1.00 107.4±0.40µs ? ?/sec +dev 28.79 3.1±0.02ms ? ?/sec + +rustc_hex_encode +---------------- +release 1.00 53.7±15.63µs ? ?/sec +dev 25.80 1385.4±4.37µs ? ?/sec +``` + +## Filtering results + +Some projects have dozens or even hundreds of benchmarks which can be +overwhelming if you're only interested in the performance of a single +feature/function. + +Let's clone the `hex` crate and change just a single function: + +```bash +> git clone https://github.com/KokaKiwi/rust-hex.git +> cd rust-hex/ +``` + +Save a baseline for the `main` branch: + +```bash +> cargo bench --bench=hex `# Select the 'hex' binary` \ + -- `# Switch args from cargo to criterion` \ + --save-baseline main `# Save the baseline under 'main'` +``` + +Create a new branch: + +```bash +> git checkout -b new-feature +``` + +For testing, let's modify the `hex_decode` benchmark to run twice: + +```diff +--- a/benches/hex.rs ++++ b/benches/hex.rs + c.bench_function("hex_decode", |b| { + let hex = hex::encode(DATA); +- b.iter(|| hex::decode(&hex).unwrap()) ++ b.iter(|| (hex::decode(&hex).unwrap(),hex::decode(&hex).unwrap())) + }); +``` + +Now we can benchmark just the `hex_decode` function: + +```bash +> cargo bench --bench=hex `# Select the 'hex' binary` \ + -- `# Switch args from cargo to criterion` \ + --save-baseline new-feature `# Save the baseline under 'new-feature'` \ + ^hex_decode `# Select the 'hex_decode' benchmark` +``` + +And compare it to the `main` branch, verifying that we've introduced a 2x +performance regression: + +```bash +> cargo bench --bench=hex -- --compare --baselines=main,new-feature ^hex_decode +``` + +
group                   main                                      new-feature
+-----                   ----                                      -----------
+hex_decode    1.00    119.1±1.30µs        ? ?/sec    2.06    245.5±2.21µs        ? ?/sec
+
+ +## Thresholds + +If we don't know which benchmarks are of interest, we can filter the results +based on how much they've changed. + +In the previous section, we only generated results for the `hex_decode` +benchmark. For this run, we need a complete set of results: + +```bash +> cargo bench --bench=hex `# Select the 'hex' binary` \ + -- `# Switch args from cargo to criterion` \ + --save-baseline new-feature `# Save the baseline under 'new-feature'` \ +``` + +Now we can compare the results that differ by more than 10%: + +```bash +> cargo bench --bench=hex -- --compare --baselines=main,new-feature --compare-threshold=10 +``` + +
group                   main                                      new-feature
+-----                   ----                                      -----------
+hex_decode    1.00    119.1±1.30µs        ? ?/sec    2.02    240.0±1.05µs        ? ?/sec
+
+ +The above console output shows that only a single benchmark changed by more than +10%. + +## Importing/Exporting JSON + +Baselines can be saved in JSON files for later use with the `--export` flag. Continuing with the `hex` crate example, here's how to +save the `release` and `dev` baselines as JSON: + +```bash +> cargo bench --bench=hex -- --export release > release.json +``` + +```bash +> cargo bench --bench=hex -- --export dev > dev.json +``` + +Baselines stored as JSON can be referenced directly when comparing results: + +```bash +> cargo bench --bench=hex -- --compare --baselines dev.json,release.json +``` + +
group                          dev                                               release
+-----                          ---                                               -------
+faster_hex_decode              239.50  847.6±16.54µs        ? ?/sec    1.00      3.5±0.01µs        ? ?/sec
+faster_hex_decode_fallback     52.58   567.7±8.36µs        ? ?/sec     1.00     10.8±0.04µs        ? ?/sec
+faster_hex_decode_unchecked    400.98   503.7±3.48µs        ? ?/sec    1.00   1256.2±1.57ns        ? ?/sec
+faster_hex_encode              259.95   244.5±2.04µs        ? ?/sec    1.00    940.5±4.64ns        ? ?/sec
+faster_hex_encode_fallback     50.60   565.1±3.41µs        ? ?/sec     1.00     11.2±0.02µs        ? ?/sec
+hex_decode                     25.27     3.0±0.01ms        ? ?/sec     1.00    119.3±0.17µs        ? ?/sec
+hex_encode                     23.99 1460.8±18.11µs        ? ?/sec     1.00     60.9±0.08µs        ? ?/sec
+rustc_hex_decode               28.79     3.1±0.02ms        ? ?/sec     1.00    107.4±0.40µs        ? ?/sec
+rustc_hex_encode               25.80  1385.4±4.37µs        ? ?/sec     1.00    53.7±15.63µs        ? ?/sec
+
+ +Note that the JSON format is not stable across criterion versions. diff --git a/plot/src/lib.rs b/plot/src/lib.rs index 151f01924..0ba04dc89 100644 --- a/plot/src/lib.rs +++ b/plot/src/lib.rs @@ -443,7 +443,7 @@ impl Figure { s.push_str(&format!( "set output '{}'\n", - self.output.display().to_string().replace("'", "''") + self.output.display().to_string().replace('\'', "''") )); if let Some(width) = self.box_width { diff --git a/src/connection.rs b/src/connection.rs index 2edc40262..c6d77532b 100644 --- a/src/connection.rs +++ b/src/connection.rs @@ -72,7 +72,7 @@ impl InnerConnection { let mut hello_buf = [0u8; RUNNER_HELLO_SIZE]; socket.read_exact(&mut hello_buf)?; assert!( - !(&hello_buf[0..RUNNER_MAGIC_NUMBER.len()] != RUNNER_MAGIC_NUMBER.as_bytes()), + (&hello_buf[0..RUNNER_MAGIC_NUMBER.len()] == RUNNER_MAGIC_NUMBER.as_bytes()), "Not connected to cargo-criterion." ); diff --git a/src/critcmp/app.rs b/src/critcmp/app.rs new file mode 100644 index 000000000..78b73f857 --- /dev/null +++ b/src/critcmp/app.rs @@ -0,0 +1,148 @@ +use std::collections::BTreeSet; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use regex::Regex; +use tabwriter::TabWriter; +use termcolor::{self, WriteColor}; + +use crate::critcmp::data::{BaseBenchmarks, Benchmarks}; +use crate::critcmp::main::Result; + +#[derive(Clone, Debug, Default)] +pub struct Args { + pub baselines: Vec, + pub output_list: bool, + pub threshold: Option, + pub color: bool, + pub filter: Option, +} + +impl Args { + pub fn benchmarks(&self) -> Result { + // First, load benchmark data from command line parameters. If a + // baseline name is given and is not a file path, then it is added to + // our whitelist of baselines. + let mut from_cli: Vec = vec![]; + let mut whitelist = BTreeSet::new(); + for arg in self.baselines.iter() { + let p = Path::new(arg); + if p.is_file() { + let baseb = BaseBenchmarks::from_path(p) + .map_err(|err| format!("{}: {}", p.display(), err))?; + whitelist.insert(baseb.name.clone()); + from_cli.push(baseb); + } else { + whitelist.insert(arg.clone()); + } + } + + let mut from_crit: Vec = vec![]; + match self.criterion_dir() { + Err(err) => { + // If we've loaded specific benchmarks from arguments, then it + // shouldn't matter whether we can find a Criterion directory. + // If we haven't loaded anything explicitly though, and if + // Criterion detection fails, then we won't have loaded + // anything and so we should return an error. + if from_cli.is_empty() { + return Err(err); + } + } + Ok(critdir) => { + let data = Benchmarks::gather(critdir)?; + from_crit.extend(data.by_baseline.into_iter().map(|(_, v)| v)); + } + } + if from_cli.is_empty() && from_crit.is_empty() { + fail!("could not find any benchmark data"); + } + + let mut data = Benchmarks::default(); + for basebench in from_crit.into_iter().chain(from_cli) { + if !whitelist.is_empty() && !whitelist.contains(&basebench.name) { + continue; + } + data.by_baseline.insert(basebench.name.clone(), basebench); + } + Ok(data) + } + + pub fn filter(&self) -> Option<&'_ Regex> { + self.filter.as_ref() + } + + pub fn group(&self) -> Result> { + // TODO + Ok(None) + // let pattern_os = match self.0.value_of_os("group") { + // None => return Ok(None), + // Some(pattern) => pattern, + // }; + // let pattern = cli::pattern_from_os(pattern_os)?; + // let re = Regex::new(pattern)?; + // if re.captures_len() <= 1 { + // fail!( + // "pattern '{}' has no capturing groups, by grouping \ + // benchmarks by a regex requires the use of at least \ + // one capturing group", + // pattern + // ); + // } + // Ok(Some(re)) + } + + pub fn threshold(&self) -> Result> { + Ok(self.threshold) + } + + pub fn list(&self) -> bool { + self.output_list + } + + pub fn criterion_dir(&self) -> Result { + let target_dir = self.target_dir()?; + let crit_dir = target_dir.join("criterion"); + if !crit_dir.exists() { + fail!( + "\ + no criterion data exists at {}\n\ + try running your benchmarks before tabulating results\ + ", + crit_dir.display() + ); + } + Ok(crit_dir) + } + + pub fn stdout(&self) -> Box { + if self.color { + Box::new(termcolor::Ansi::new(TabWriter::new(io::stdout()))) + } else { + Box::new(termcolor::NoColor::new(TabWriter::new(io::stdout()))) + } + } + + fn target_dir(&self) -> Result { + // FIXME: Use the same code as criterion + let mut cwd = fs::canonicalize(".")?; + loop { + let candidate = cwd.join("target"); + if candidate.exists() { + return Ok(candidate); + } + cwd = match cwd.parent() { + Some(p) => p.to_path_buf(), + None => { + fail!( + "\ + could not find Criterion output directory\n\ + try using --target-dir or set CARGO_TARGET_DIR\ + " + ); + } + } + } + } +} diff --git a/src/critcmp/data.rs b/src/critcmp/data.rs new file mode 100644 index 000000000..4ea321906 --- /dev/null +++ b/src/critcmp/data.rs @@ -0,0 +1,227 @@ +use std::collections::BTreeMap; +use std::fs::File; +use std::io; +use std::path::Path; + +use serde::de::DeserializeOwned; +// use serde::{Deserialize, Serialize}; +use serde_json as json; +use walkdir::WalkDir; + +use crate::critcmp::main::Result; + +#[derive(Clone, Debug, Default)] +pub struct Benchmarks { + pub by_baseline: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct BaseBenchmarks { + pub name: String, + pub benchmarks: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Benchmark { + pub baseline: String, + pub fullname: String, + #[serde(rename = "criterion_benchmark_v1")] + pub info: CBenchmark, + #[serde(rename = "criterion_estimates_v1")] + pub estimates: CEstimates, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct CBenchmark { + pub group_id: String, + pub function_id: Option, + pub value_str: Option, + pub throughput: Option, + pub full_id: String, + pub directory_name: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "PascalCase")] +pub struct CThroughput { + pub bytes: Option, + pub elements: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct CEstimates { + pub mean: CStats, + pub median: CStats, + pub median_abs_dev: CStats, + pub slope: Option, + pub std_dev: CStats, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct CStats { + pub confidence_interval: CConfidenceInterval, + pub point_estimate: f64, + pub standard_error: f64, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct CConfidenceInterval { + pub confidence_level: f64, + pub lower_bound: f64, + pub upper_bound: f64, +} + +impl Benchmarks { + pub fn gather>(criterion_dir: P) -> Result { + let mut benchmarks = Benchmarks::default(); + for result in WalkDir::new(criterion_dir) { + let dent = result?; + let b = match Benchmark::from_path(dent.path())? { + None => continue, + Some(b) => b, + }; + benchmarks + .by_baseline + .entry(b.baseline.clone()) + .or_insert_with(|| BaseBenchmarks { + name: b.baseline.clone(), + benchmarks: BTreeMap::new(), + }) + .benchmarks + .insert(b.benchmark_name().to_string(), b); + } + Ok(benchmarks) + } +} + +impl Benchmark { + fn from_path>(path: P) -> Result> { + let path = path.as_ref(); + Benchmark::from_path_imp(path).map_err(|err| { + if let Some(parent) = path.parent() { + err!("{}: {}", parent.display(), err) + } else { + err!("unknown path: {}", err) + } + }) + } + + fn from_path_imp(path: &Path) -> Result> { + match path.file_name() { + None => return Ok(None), + Some(filename) => { + if filename != "estimates.json" { + return Ok(None); + } + } + } + // Criterion's directory structure looks like this: + // + // criterion/{group}/{name}/{baseline}/estimates.json + // + // In the same directory as `estimates.json`, there is also a + // `benchmark.json` which contains most of the info we need about + // a benchmark, including its name. From the path, we only extract the + // baseline name. + let parent = path + .parent() + .ok_or_else(|| err!("{}: could not find parent dir", path.display()))?; + let baseline = parent + .file_name() + .map(|p| p.to_string_lossy().into_owned()) + .ok_or_else(|| err!("{}: could not find baseline name", path.display()))?; + if baseline == "change" { + // This isn't really a baseline, but special state emitted by + // Criterion to reflect its own comparison between baselines. We + // don't use it. + return Ok(None); + } + + let info = CBenchmark::from_path(parent.join("benchmark.json"))?; + let estimates = CEstimates::from_path(path)?; + let fullname = format!("{}/{}", baseline, info.full_id); + Ok(Some(Benchmark { + baseline, + fullname, + info, + estimates, + })) + } + + pub fn nanoseconds(&self) -> f64 { + self.estimates.mean.point_estimate + } + + pub fn stddev(&self) -> f64 { + self.estimates.std_dev.point_estimate + } + + pub fn fullname(&self) -> &str { + &self.fullname + } + + pub fn baseline(&self) -> &str { + &self.baseline + } + + pub fn benchmark_name(&self) -> &str { + &self.info.full_id + } + + pub fn throughput(&self) -> Option { + const NANOS_PER_SECOND: f64 = 1_000_000_000.0; + + let scale = NANOS_PER_SECOND / self.nanoseconds(); + + self.info.throughput.as_ref().and_then(|t| { + let scaled_bytes = t.bytes.map(|num| Throughput::Bytes(num as f64 * scale)); + let scaled_elements = t + .elements + .map(|num| Throughput::Elements(num as f64 * scale)); + scaled_bytes.or(scaled_elements) + }) + } +} + +#[derive(Clone, Copy, Debug)] +pub enum Throughput { + Bytes(f64), + Elements(f64), +} + +impl BaseBenchmarks { + pub fn from_path>(path: P) -> Result { + deserialize_json_path(path.as_ref()) + } +} + +impl CBenchmark { + fn from_path>(path: P) -> Result { + deserialize_json_path(path.as_ref()) + } +} + +impl CEstimates { + fn from_path>(path: P) -> Result { + deserialize_json_path(path.as_ref()) + } +} + +fn deserialize_json_path(path: &Path) -> Result { + let file = File::open(path).map_err(|err| { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + err!("{}: {}", name, err) + } else { + err!("{}: {}", path.display(), err) + } + })?; + let buf = io::BufReader::new(file); + let b = json::from_reader(buf).map_err(|err| { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + err!("{}: {}", name, err) + } else { + err!("{}: {}", path.display(), err) + } + })?; + Ok(b) +} diff --git a/src/critcmp/main.rs b/src/critcmp/main.rs new file mode 100644 index 000000000..1fb447031 --- /dev/null +++ b/src/critcmp/main.rs @@ -0,0 +1,131 @@ +use std::collections::BTreeMap; +use std::error::Error; +use std::io::Write; +use std::process; +use std::result; + +use regex::Regex; + +use crate::critcmp::app::Args; +use crate::critcmp::data::{Benchmark, Benchmarks}; + +use crate::critcmp::output; + +macro_rules! err { + ($($tt:tt)*) => { Box::::from(format!($($tt)*)) } +} + +macro_rules! fail { + ($($tt:tt)*) => { return Err(err!($($tt)*)) } +} + +pub type Result = result::Result>; + +pub fn main(args: Args) { + if let Err(err) = try_main(args) { + eprintln!("{}", err); + process::exit(1); + } +} + +fn try_main(args: Args) -> Result<()> { + let benchmarks = args.benchmarks()?; + + let mut comps = match args.group()? { + None => group_by_baseline(&benchmarks, args.filter()), + Some(re) => group_by_regex(&benchmarks, &re, args.filter()), + }; + if let Some(threshold) = args.threshold()? { + comps.retain(|comp| comp.biggest_difference() > threshold); + } + if comps.is_empty() { + fail!("no benchmark comparisons to show"); + } + + let mut wtr = args.stdout(); + if args.list() { + output::rows(&mut wtr, &comps)?; + } else { + output::columns(&mut wtr, &comps)?; + } + wtr.flush()?; + Ok(()) +} + +fn group_by_baseline(benchmarks: &Benchmarks, filter: Option<&Regex>) -> Vec { + let mut byname: BTreeMap> = BTreeMap::new(); + for base_benchmarks in benchmarks.by_baseline.values() { + for (name, benchmark) in base_benchmarks.benchmarks.iter() { + if filter.map_or(false, |re| !re.is_match(name)) { + continue; + } + let output_benchmark = + output::Benchmark::from_data(benchmark).name(benchmark.baseline()); + byname + .entry(name.to_string()) + .or_insert_with(Vec::new) + .push(output_benchmark); + } + } + byname + .into_iter() + .map(|(name, benchmarks)| output::Comparison::new(&name, benchmarks)) + .collect() +} + +fn group_by_regex( + benchmarks: &Benchmarks, + group_by: &Regex, + filter: Option<&Regex>, +) -> Vec { + let mut byname: BTreeMap> = BTreeMap::new(); + for base_benchmarks in benchmarks.by_baseline.values() { + for (name, benchmark) in base_benchmarks.benchmarks.iter() { + if filter.map_or(false, |re| !re.is_match(name)) { + continue; + } + let (bench, cmp) = match benchmark_names(benchmark, group_by) { + None => continue, + Some((bench, cmp)) => (bench, cmp), + }; + let output_benchmark = output::Benchmark::from_data(benchmark).name(&bench); + byname + .entry(cmp) + .or_insert_with(Vec::new) + .push(output_benchmark); + } + } + byname + .into_iter() + .map(|(name, benchmarks)| output::Comparison::new(&name, benchmarks)) + .collect() +} + +fn benchmark_names(benchmark: &Benchmark, group_by: &Regex) -> Option<(String, String)> { + assert!(group_by.captures_len() > 1); + + let caps = match group_by.captures(benchmark.benchmark_name()) { + None => return None, + Some(caps) => caps, + }; + + let mut bench_name = benchmark.benchmark_name().to_string(); + let mut cmp_name = String::new(); + let mut offset = 0; + for option in caps.iter().skip(1) { + let m = match option { + None => continue, + Some(m) => m, + }; + cmp_name.push_str(m.as_str()); + // Strip everything that doesn't match capturing groups. The leftovers + // are our benchmark name. + bench_name.drain((m.start() - offset)..(m.end() - offset)); + offset += m.end() - m.start(); + } + // Add the baseline name to the benchmark to disambiguate it from + // benchmarks with the same name in other baselines. + bench_name.insert_str(0, &format!("{}/", benchmark.baseline())); + + Some((bench_name, cmp_name)) +} diff --git a/src/critcmp/mod.rs b/src/critcmp/mod.rs new file mode 100644 index 000000000..17f6ee447 --- /dev/null +++ b/src/critcmp/mod.rs @@ -0,0 +1,5 @@ +#[macro_use] +pub mod main; +pub mod app; +pub mod data; +pub mod output; diff --git a/src/critcmp/output.rs b/src/critcmp/output.rs new file mode 100644 index 000000000..f6c68d91b --- /dev/null +++ b/src/critcmp/output.rs @@ -0,0 +1,234 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::iter; + +use termcolor::{Color, ColorSpec, WriteColor}; +use unicode_width::UnicodeWidthStr; + +use crate::critcmp::data; +use crate::critcmp::main::Result; + +#[derive(Clone, Debug)] +pub struct Comparison { + name: String, + benchmarks: Vec, + name_to_index: BTreeMap, +} + +#[derive(Clone, Debug)] +pub struct Benchmark { + name: String, + nanoseconds: f64, + stddev: Option, + throughput: Option, + /// Whether this is the best benchmark in a group. This is only populated + /// when a `Comparison` is built. + best: bool, + /// The rank of this benchmark in a group. The best is always `1.0`. This + /// is only populated when a `Comparison` is built. + rank: f64, +} + +impl Comparison { + pub fn new(name: &str, benchmarks: Vec) -> Comparison { + let mut comp = Comparison { + name: name.to_string(), + benchmarks, + name_to_index: BTreeMap::new(), + }; + if comp.benchmarks.is_empty() { + return comp; + } + + comp.benchmarks + .sort_by(|a, b| a.nanoseconds.partial_cmp(&b.nanoseconds).unwrap()); + comp.benchmarks[0].best = true; + + let top = comp.benchmarks[0].nanoseconds; + for (i, b) in comp.benchmarks.iter_mut().enumerate() { + comp.name_to_index.insert(b.name.to_string(), i); + b.rank = b.nanoseconds / top; + } + comp + } + + /// Return the biggest difference, percentage wise, between benchmarks + /// in this comparison. + /// + /// If this comparison has fewer than two benchmarks, then 0 is returned. + pub fn biggest_difference(&self) -> f64 { + if self.benchmarks.len() < 2 { + return 0.0; + } + let best = self.benchmarks[0].nanoseconds; + let worst = self.benchmarks.last().unwrap().nanoseconds; + ((worst - best) / best) * 100.0 + } + + fn get(&self, name: &str) -> Option<&Benchmark> { + self.name_to_index + .get(name) + .and_then(|&i| self.benchmarks.get(i)) + } +} + +impl Benchmark { + pub fn from_data(b: &data::Benchmark) -> Benchmark { + Benchmark { + name: b.fullname().to_string(), + nanoseconds: b.nanoseconds(), + stddev: Some(b.stddev()), + throughput: b.throughput(), + best: false, + rank: 0.0, + } + } + + pub fn name(self, name: &str) -> Benchmark { + Benchmark { + name: name.to_string(), + ..self + } + } +} + +pub fn columns(mut wtr: W, groups: &[Comparison]) -> Result<()> { + let mut columns = BTreeSet::new(); + for group in groups { + for b in &group.benchmarks { + columns.insert(b.name.to_string()); + } + } + + write!(wtr, "group")?; + for column in &columns { + write!(wtr, "\t {}", column)?; + } + writeln!(wtr)?; + + write_divider(&mut wtr, '-', "group".width())?; + for column in &columns { + write!(wtr, "\t ")?; + write_divider(&mut wtr, '-', column.width())?; + } + writeln!(wtr)?; + + for group in groups { + if group.benchmarks.is_empty() { + continue; + } + + write!(wtr, "{}", group.name)?; + for column_name in &columns { + let b = match group.get(column_name) { + Some(b) => b, + None => { + write!(wtr, "\t")?; + continue; + } + }; + + if b.best { + let mut spec = ColorSpec::new(); + spec.set_fg(Some(Color::Green)).set_bold(true); + wtr.set_color(&spec)?; + } + write!( + wtr, + "\t {:<5.2} {:>14} {:>14}", + b.rank, + time(b.nanoseconds, b.stddev), + throughput(b.throughput), + )?; + if b.best { + wtr.reset()?; + } + } + writeln!(wtr)?; + } + Ok(()) +} + +pub fn rows(mut wtr: W, groups: &[Comparison]) -> Result<()> { + for (i, group) in groups.iter().enumerate() { + if i > 0 { + writeln!(wtr)?; + } + rows_one(&mut wtr, group)?; + } + Ok(()) +} + +fn rows_one(mut wtr: W, group: &Comparison) -> Result<()> { + writeln!(wtr, "{}", group.name)?; + write_divider(&mut wtr, '-', group.name.width())?; + writeln!(wtr)?; + + if group.benchmarks.is_empty() { + writeln!(wtr, "NOTHING TO SHOW")?; + return Ok(()); + } + + for b in &group.benchmarks { + writeln!( + wtr, + "{}\t{:>7.2}\t{:>15}\t{:>12}", + b.name, + b.rank, + time(b.nanoseconds, b.stddev), + throughput(b.throughput), + )?; + } + Ok(()) +} + +fn write_divider(mut wtr: W, divider: char, width: usize) -> Result<()> { + let div: String = iter::repeat(divider).take(width).collect(); + write!(wtr, "{}", div)?; + Ok(()) +} + +fn time(nanos: f64, stddev: Option) -> String { + const MIN_MICRO: f64 = 2_000.0; + const MIN_MILLI: f64 = 2_000_000.0; + const MIN_SEC: f64 = 2_000_000_000.0; + + let (div, label) = if nanos < MIN_MICRO { + (1.0, "ns") + } else if nanos < MIN_MILLI { + (1_000.0, "µs") + } else if nanos < MIN_SEC { + (1_000_000.0, "ms") + } else { + (1_000_000_000.0, "s") + }; + if let Some(stddev) = stddev { + format!("{:.1}±{:.2}{}", nanos / div, stddev / div, label) + } else { + format!("{:.1}{}", nanos / div, label) + } +} + +fn throughput(throughput: Option) -> String { + use data::Throughput::*; + match throughput { + Some(Bytes(num)) => throughput_per(num, "B"), + Some(Elements(num)) => throughput_per(num, "Elem"), + _ => "? ?/sec".to_string(), + } +} + +fn throughput_per(per: f64, unit: &str) -> String { + const MIN_K: f64 = (2 * (1 << 10) as u64) as f64; + const MIN_M: f64 = (2 * (1 << 20) as u64) as f64; + const MIN_G: f64 = (2 * (1 << 30) as u64) as f64; + + if per < MIN_K { + format!("{} {}/sec", per as u64, unit) + } else if per < MIN_M { + format!("{:.1} K{}/sec", (per / (1 << 10) as f64), unit) + } else if per < MIN_G { + format!("{:.1} M{}/sec", (per / (1 << 20) as f64), unit) + } else { + format!("{:.1} G{}/sec", (per / (1 << 30) as f64), unit) + } +} diff --git a/src/lib.rs b/src/lib.rs index ac0a4e5a5..dd9ac0853 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -57,6 +57,7 @@ mod benchmark_group; pub mod async_executor; mod bencher; mod connection; +mod critcmp; #[cfg(feature = "csv_output")] mod csv_report; mod error; @@ -77,6 +78,7 @@ use std::cell::RefCell; use std::collections::HashSet; use std::default::Default; use std::env; +use std::io::Write; use std::net::TcpStream; use std::path::{Path, PathBuf}; use std::process::Command; @@ -454,6 +456,7 @@ impl Criterion { } } + #[must_use] /// Changes the internal profiler for benchmarks run with this runner. See /// the Profiler trait for more details. pub fn with_profiler(self, p: P) -> Criterion { @@ -463,6 +466,7 @@ impl Criterion { } } + #[must_use] /// Set the plotting backend. By default, Criterion will use gnuplot if available, or plotters /// if not. /// @@ -481,6 +485,7 @@ impl Criterion { self } + #[must_use] /// Changes the default size of the sample for benchmarks run with this runner. /// /// A bigger sample should yield more accurate results if paired with a sufficiently large @@ -498,6 +503,7 @@ impl Criterion { self } + #[must_use] /// Changes the default warm up time for benchmarks run with this runner. /// /// # Panics @@ -510,6 +516,7 @@ impl Criterion { self } + #[must_use] /// Changes the default measurement time for benchmarks run with this runner. /// /// With a longer time, the measurement will become more resilient to transitory peak loads @@ -527,6 +534,7 @@ impl Criterion { self } + #[must_use] /// Changes the default number of resamples for benchmarks run with this runner. /// /// Number of resamples to use for the @@ -548,6 +556,7 @@ impl Criterion { self } + #[must_use] /// Changes the default noise threshold for benchmarks run with this runner. The noise threshold /// is used to filter out small changes in performance, even if they are statistically /// significant. Sometimes benchmarking the same code twice will result in small but @@ -567,6 +576,7 @@ impl Criterion { self } + #[must_use] /// Changes the default confidence level for benchmarks run with this runner. The confidence /// level is the desired probability that the true runtime lies within the estimated /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is @@ -585,6 +595,7 @@ impl Criterion { self } + #[must_use] /// Changes the default [significance level](https://en.wikipedia.org/wiki/Statistical_significance) /// for benchmarks run with this runner. This is used to perform a /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if @@ -612,6 +623,7 @@ impl Criterion { self } + #[must_use] /// Enables plotting pub fn with_plots(mut self) -> Criterion { // If running under cargo-criterion then don't re-enable the reports; let it do the reporting. @@ -626,12 +638,14 @@ impl Criterion { self } + #[must_use] /// Disables plotting pub fn without_plots(mut self) -> Criterion { self.report.html = None; self } + #[must_use] /// Names an explicit baseline and enables overwriting the previous results. pub fn save_baseline(mut self, baseline: String) -> Criterion { self.baseline_directory = baseline; @@ -639,6 +653,7 @@ impl Criterion { self } + #[must_use] /// Names an explicit baseline and disables overwriting the previous results. pub fn retain_baseline(mut self, baseline: String, strict: bool) -> Criterion { self.baseline_directory = baseline; @@ -650,6 +665,7 @@ impl Criterion { self } + #[must_use] /// Filters the benchmarks. Only benchmarks with names that contain the /// given string will be executed. pub fn with_filter>(mut self, filter: S) -> Criterion { @@ -665,6 +681,7 @@ impl Criterion { self } + #[must_use] /// Override whether the CLI output will be colored or not. Usually you would use the `--color` /// CLI argument, but this is available for programmmatic use as well. pub fn with_output_color(mut self, enabled: bool) -> Criterion { @@ -673,6 +690,7 @@ impl Criterion { } /// Set the output directory (currently for testing only) + #[must_use] #[doc(hidden)] pub fn output_directory(mut self, path: &Path) -> Criterion { self.output_directory = path.to_owned(); @@ -681,6 +699,7 @@ impl Criterion { } /// Set the profile time (currently for testing only) + #[must_use] #[doc(hidden)] pub fn profile_time(mut self, profile_time: Option) -> Criterion { match profile_time { @@ -708,6 +727,7 @@ impl Criterion { /// Configure this criterion struct based on the command-line arguments to /// this process. + #[must_use] #[cfg_attr(feature = "cargo-clippy", allow(clippy::cognitive_complexity))] pub fn configure_from_args(mut self) -> Criterion { use clap::{App, Arg}; @@ -764,6 +784,30 @@ impl Criterion { .takes_value(true) .help("Iterate each benchmark for approximately the given number of seconds, doing no analysis and without storing the results. Useful for running the benchmarks in a profiler.") .conflicts_with_all(&["test", "list"])) + .arg(Arg::with_name("export") + .long("export") + .takes_value(true) + .help("Export baseline as json, printed to stdout") + .conflicts_with_all(&["list", "test", "profile-time", "compare"])) + .arg(Arg::with_name("compare") + .long("compare") + .help("Tabulate benchmark results") + .conflicts_with_all(&["list", "test", "profile-time", "export"])) + .arg(Arg::with_name("baselines") + .long("baselines") + .multiple(true) + .value_name("baselines") + .requires("compare") + .require_delimiter(true) + .help("Limit the baselines used in tabulated results.") + .help("")) + .arg(Arg::with_name("compare-threshold") + .long("compare-threshold") + .takes_value(true) + .help("Hide results that differ by less than the threshold percentage. By default, all results are shown.")) + .arg(Arg::with_name("compare-list") + .long("compare-list") + .help("Show benchmark results in a list rather than in a table. Useful when horizontal space is limited.")) .arg(Arg::with_name("load-baseline") .long("load-baseline") .takes_value(true) @@ -1065,6 +1109,54 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.config.significance_level = num_significance_level; } + // XXX: Comparison functionality should ideally live in 'cargo-criterion'. + if matches.is_present("compare") { + if self.connection.is_some() { + eprintln!( + "Error: tabulating results is not supported when running with cargo-criterion." + ); + std::process::exit(1); + } + // Other arguments: compare-threshold, compare-list. + + let stdout_isatty = atty::is(atty::Stream::Stdout); + let enable_text_coloring = match matches.value_of("color") { + Some("always") => true, + Some("never") => false, + _ => stdout_isatty, + }; + + let args = critcmp::app::Args { + baselines: matches.values_of_lossy("baselines").unwrap_or_default(), + output_list: matches.is_present("compare-list"), + threshold: value_t!(matches.value_of("compare-threshold"), f64).ok(), // FIXME: Print error message if parsing fails. + color: enable_text_coloring, + filter: self.filter, + }; + critcmp::main::main(args); + std::process::exit(0); + } + + if let Some(baseline) = matches.value_of("export") { + let benchmarks = critcmp::app::Args { + baselines: matches.values_of_lossy("baselines").unwrap_or_default(), + ..Default::default() + } + .benchmarks() + .expect("failed to find baselines"); + let mut stdout = std::io::stdout(); + let basedata = match benchmarks.by_baseline.get(baseline) { + Some(basedata) => basedata, + None => { + eprintln!("failed to find baseline '{}'", baseline); + std::process::exit(1); + } + }; + serde_json::to_writer_pretty(&mut stdout, basedata).unwrap(); + writeln!(stdout).unwrap(); + std::process::exit(0); + } + if matches.is_present("quick") { self.config.quick_mode = true; } @@ -1258,6 +1350,7 @@ impl Default for PlotConfiguration { } impl PlotConfiguration { + #[must_use] /// Set the axis scale (linear or logarithmic) for the summary plots. Typically, you would /// set this to logarithmic if benchmarking over a range of inputs which scale exponentially. /// Defaults to linear. diff --git a/src/plot/gnuplot_backend/mod.rs b/src/plot/gnuplot_backend/mod.rs index e7f2ae7ed..27cc48be3 100644 --- a/src/plot/gnuplot_backend/mod.rs +++ b/src/plot/gnuplot_backend/mod.rs @@ -26,7 +26,7 @@ use super::{PlotContext, PlotData, Plotter}; use crate::format; fn gnuplot_escape(string: &str) -> String { - string.replace("_", "\\_").replace("'", "''") + string.replace('_', "\\_").replace('\'', "''") } static DEFAULT_FONT: &str = "Helvetica"; diff --git a/src/plot/plotters_backend/summary.rs b/src/plot/plotters_backend/summary.rs index dad8a5bfa..d967828aa 100644 --- a/src/plot/plotters_backend/summary.rs +++ b/src/plot/plotters_backend/summary.rs @@ -31,9 +31,9 @@ pub fn line_comparison( let (unit, series_data) = line_comparison_series_data(formatter, all_curves); let x_range = - plotters::data::fitting_range(series_data.iter().map(|(_, xs, _)| xs.iter()).flatten()); + plotters::data::fitting_range(series_data.iter().flat_map(|(_, xs, _)| xs.iter())); let y_range = - plotters::data::fitting_range(series_data.iter().map(|(_, _, ys)| ys.iter()).flatten()); + plotters::data::fitting_range(series_data.iter().flat_map(|(_, _, ys)| ys.iter())); let root_area = SVGBackend::new(&path, SIZE) .into_drawing_area() .titled(&format!("{}: Comparison", title), (DEFAULT_FONT, 20)) @@ -196,8 +196,7 @@ pub fn violin( formatter.scale_values(max, xs); }); - let mut x_range = - plotters::data::fitting_range(kdes.iter().map(|(_, xs, _)| xs.iter()).flatten()); + let mut x_range = plotters::data::fitting_range(kdes.iter().flat_map(|(_, xs, _)| xs.iter())); x_range.start = 0.0; let y_range = -0.5..all_curves.len() as f64 - 0.5; From 08bdb9ace9c1d237c0a5a79a1844dc0a894056b2 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Sun, 13 Feb 2022 12:05:07 +0000 Subject: [PATCH 17/29] Improved WASI support (#550) * Document WASM support with wasmer, wasmtime, nodejs, and browsers. --- book/src/SUMMARY.md | 1 + book/src/user_guide/wasi.md | 230 ++++++++++++++++++++++++++++++++++++ src/critcmp/app.rs | 4 +- src/lib.rs | 3 + 4 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 book/src/user_guide/wasi.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 5bdfce2bc..25bcd1f8f 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -21,6 +21,7 @@ - [Benchmarking async functions](./user_guide/benchmarking_async.md) - [Quick Mode](./user_guide/quick_mode.md) - [Tabulating Results](./user_guide/tabulating_results.md) + - [WebAssembly/WASI](./user_guide/wasi.md) - [cargo-criterion](./cargo_criterion/cargo_criterion.md) - [Configuring cargo-criterion](./cargo_criterion/configuring_cargo_criterion.md) - [External Tools](./cargo_criterion/external_tools.md) diff --git a/book/src/user_guide/wasi.md b/book/src/user_guide/wasi.md new file mode 100644 index 000000000..734e369c1 --- /dev/null +++ b/book/src/user_guide/wasi.md @@ -0,0 +1,230 @@ +# WebAssebly/WASI benchmarking + +Criterion benchmarks can be compiled to WebAssembly/WASI. This lets you performance test your code with runtimes such as [wasmer](https://wasmer.io/) and [wasmtime](https://wasmtime.dev/), as well as JavaScript environments such as [NodeJS](https://nodejs.org/en/) or web browsers ([FireFox](https://www.mozilla.org/en-US/firefox/new/), [Chrome](https://www.google.com/chrome/), [Safari](https://www.apple.com/safari/)). + +## Adding the `wasm32-wasi` target + +We're cross-compiling to WebAssembly System Interface (aka WASI) so we have to add the right target. If you forget this step then your benchmarks will not compile and you'll be shown a gentle reminder. We'll use the [rustup](https://rustup.rs/) tool: + +```properties +rustup target add wasm32-wasi +``` + +## Install cargo-wasi + +Next we'll install the `cargo-wasi` command. While this is not required by Criterion, it does make it a lot easier to build WASI programs. + +```properties +cargo install cargo-wasi +``` + +## Building + +With `wasm32-wasi` and `cargo-wasi` installed, we're almost set to compile our benchmarks. Just one thing left: Telling Criterion not to use default features such as rayon which are not yet available in WASI: + +```diff +[dev-dependencies] +-criterion = "0.4" ++criterion = { version = "0.4", default-features = false } +``` + +Compiling the benchmark with `cargo-wasi` will automatically select the right target and optimize the resulting file. Here I'm using the [hex](https://crates.io/crates/hex) crate as an example: + +```properties +cargo wasi build --bench=hex --release +``` + +But it is also possible to compile it without `cargo-wasi`: + +```properties +cargo build --bench=hex --release --target wasm32-wasi +``` + +There should now be a `.wasm` file in `target/wasm32-wasi/release/deps/`. If you used `cargo-wasi` then there'll be both an optimized and an un-optimized version. Let's copy the newest WASM file out to the top-level directory for convenience: + +```console +cp `ls -t target/wasm32-wasi/release/deps/*.wasm | head -n 1` hex.wasm +``` + +## Running with wasmer/wasmtime + +```properties +wasmer run --dir=. hex.wasm -- --bench +``` + +```properties +wasmtime run --dir=. hex.wasm -- --bench +``` + +## Running with nodejs + +Running in NodeJS can be done via wasmer-cli: + +```properties +npm install -g @wasmer/cli +``` + +Once `wasmer-js` is installed, the interface is identical to plain `wasmer`: + +```properties +wasmer-js run --dir=. hex.wasm -- --bench +``` + +## Running in a browser with webassembly.sh + +Browsers do not natively support WASI but there are workarounds. The easiest solution is [webassembly.sh](https://webassembly.sh/). This website shims WASI using an in-memory filesystem. + +To use the website, go to https://webassembly.sh/, drag-and-drop the `hex.wasm` file into the browser window, and type: + +```properties +hex --bench +``` + +Once you start the benchmark, the browser window will freeze until the results are ready. This is an unfortunate limitation of running WebAssembly in the browser. + +### Exporting results + +Writing benchmark results to an in-memory filesystem in the browser is not very useful on its own. Luckily the results are easy to export and download as JSON: + +```properties +hex --bench --export=base | download +``` + +## Comparing results + +Let's run the same benchmark with native, wasmer, wasmtime, nodejs, firefox, and chromium, and see how they compare. Step 1 is the generate the json files: + +```properties +wasmer run --dir=. hex.wasm -- --bench --save-baseline wasmer +wasmer run --dir=. hex.wasm -- --bench --export wasmer > wasmer.json +``` + +```properties +wasmtime run --dir=. hex.wasm -- --bench --save-baseline wasmtime +wasmtime run --dir=. hex.wasm -- --bench --export wasmtime > wasmtime.json +``` + +```properties +wasmer-js run --dir=. hex.wasm -- --bench --save-baseline nodejs +wasmer-js run --dir=. hex.wasm -- --bench --export nodejs > nodejs.json +``` + +```properties +hex --bench --save-baseline firefox +hex --bench --export firefox | download +``` + +```properties +hex --bench --save-baseline chromium +hex --bench --export chromium | download +``` + +Step 2 is to tabulate the json files: + +```properties +cargo bench --bench=hex -- --compare --baselines=native.json,wasmer.json,wasmtime.json,nodejs.json,firefox.json,chromium.json --compare-list +``` + +Console output: + +```bash +faster_hex_decode +----------------- +native 1.00 3.6±0.02µs ? ?/sec +wasmer 14.72 52.6±0.49µs ? ?/sec +wasmtime 16.83 60.1±0.53µs ? ?/sec +chromium 17.66 63.1±0.70µs ? ?/sec +nodejs 20.76 74.2±0.34µs ? ?/sec +firefox 23.42 83.7±2.51µs ? ?/sec + +faster_hex_decode_fallback +-------------------------- +native 1.00 10.9±0.12µs ? ?/sec +wasmer 1.49 16.2±0.04µs ? ?/sec +wasmtime 1.65 18.1±0.73µs ? ?/sec +chromium 1.87 20.4±0.16µs ? ?/sec +nodejs 2.30 25.1±0.56µs ? ?/sec +firefox 2.36 25.7±1.03µs ? ?/sec + +faster_hex_decode_unchecked +--------------------------- +native 1.00 1239.7±16.97ns ? ?/sec +wasmer 14.27 17.7±0.35µs ? ?/sec +wasmtime 14.36 17.8±0.23µs ? ?/sec +firefox 14.73 18.3±0.75µs ? ?/sec +chromium 16.53 20.5±0.28µs ? ?/sec +nodejs 20.36 25.2±0.15µs ? ?/sec + +faster_hex_encode +----------------- +native 1.00 948.3±5.47ns ? ?/sec +wasmer 19.17 18.2±0.36µs ? ?/sec +chromium 21.25 20.2±0.17µs ? ?/sec +nodejs 22.85 21.7±0.09µs ? ?/sec +wasmtime 24.01 22.8±0.53µs ? ?/sec +firefox 32.68 31.0±3.43µs ? ?/sec + +faster_hex_encode_fallback +-------------------------- +native 1.00 11.1±0.20µs ? ?/sec +chromium 2.04 22.7±0.20µs ? ?/sec +wasmtime 2.05 22.8±0.13µs ? ?/sec +wasmer 2.06 22.8±0.15µs ? ?/sec +firefox 2.11 23.4±1.79µs ? ?/sec +nodejs 2.38 26.4±0.09µs ? ?/sec + +hex_decode +---------- +native 1.00 244.6±2.36µs ? ?/sec +wasmer 1.72 421.4±9.65µs ? ?/sec +wasmtime 1.73 423.0±3.00µs ? ?/sec +firefox 1.74 426.4±18.61µs ? ?/sec +nodejs 2.00 490.3±3.49µs ? ?/sec +chromium 2.81 688.5±12.23µs ? ?/sec + +hex_encode +---------- +native 1.00 69.2±0.40µs ? ?/sec +wasmtime 1.18 81.7±0.38µs ? ?/sec +wasmer 1.46 100.9±1.22µs ? ?/sec +nodejs 2.20 152.5±1.93µs ? ?/sec +chromium 4.08 282.7±4.19µs ? ?/sec + +rustc_hex_decode +---------------- +native 1.00 103.1±2.78µs ? ?/sec +wasmer 1.33 136.8±4.06µs ? ?/sec +wasmtime 1.38 142.3±3.31µs ? ?/sec +firefox 1.55 160.3±6.43µs ? ?/sec +nodejs 1.78 183.3±2.02µs ? ?/sec +chromium 2.04 210.0±3.37µs ? ?/sec + +rustc_hex_encode +---------------- +native 1.00 30.9±0.42µs ? ?/sec +wasmtime 2.24 69.1±0.36µs ? ?/sec +wasmer 2.25 69.6±0.74µs ? ?/sec +nodejs 2.40 74.2±1.94µs ? ?/sec +chromium 2.67 82.6±2.61µs ? ?/sec +firefox 3.45 106.7±10.13µs ? ?/sec +``` + +# Caveats and pitfalls + +## Warm-up and JIT + +Most WebAssembly environments don't reach peak performance until the code has been running for a little while. This means the warm-up step is essential and skipping it (by setting it to 0 seconds or using the `--quick` flag) will lead to inaccurate results. + +## Re-running benchmarks in [webassembly.sh](https://webassembly.sh/) + +The WebAssembly.sh website shims the WebAssembly System Interface (WASI) required by Criterion. But this shim is not perfect and causes criterion to fail spectacularly when run more then once. Should this happen to you, reloading your browser window should work around the problem. + +## Wasm and default-features. + +Criterion's default features have to be disabled when compiling to wasm. Failing to do so will trigger a compilation error. If see an error saying a feature is incompatible with wasm, open your `Cargo.toml` file and make this change: + +```diff +[dev-dependencies] +-criterion = "0.4" ++criterion = { version = "0.4", default-features = false } +``` \ No newline at end of file diff --git a/src/critcmp/app.rs b/src/critcmp/app.rs index 78b73f857..e6b3e5b8c 100644 --- a/src/critcmp/app.rs +++ b/src/critcmp/app.rs @@ -126,7 +126,9 @@ impl Args { fn target_dir(&self) -> Result { // FIXME: Use the same code as criterion - let mut cwd = fs::canonicalize(".")?; + let mut cwd = fs::canonicalize(".") + .ok() + .unwrap_or_else(|| PathBuf::from(".")); loop { let candidate = cwd.join("target"); if candidate.exists() { diff --git a/src/lib.rs b/src/lib.rs index dd9ac0853..9ae9eaf64 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,6 +27,9 @@ ) )] +#[cfg(all(feature = "rayon", target_arch = "wasm32"))] +compile_error!("Rayon cannot be used when targeting wasi32. Try disabling default features."); + #[cfg(test)] extern crate approx; From 6929f48e4828360de038f4e9732066d85b9a57c6 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Sun, 13 Feb 2022 19:13:15 +0100 Subject: [PATCH 18/29] WASI: Update missing firefox results. --- book/src/user_guide/wasi.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/book/src/user_guide/wasi.md b/book/src/user_guide/wasi.md index 734e369c1..e844ceca0 100644 --- a/book/src/user_guide/wasi.md +++ b/book/src/user_guide/wasi.md @@ -134,24 +134,24 @@ native 1.00 3.6±0.02µs ? ?/sec wasmer 14.72 52.6±0.49µs ? ?/sec wasmtime 16.83 60.1±0.53µs ? ?/sec chromium 17.66 63.1±0.70µs ? ?/sec +firefox 19.82 70.8±6.53µs ? ?/sec nodejs 20.76 74.2±0.34µs ? ?/sec -firefox 23.42 83.7±2.51µs ? ?/sec faster_hex_decode_fallback -------------------------- native 1.00 10.9±0.12µs ? ?/sec wasmer 1.49 16.2±0.04µs ? ?/sec +firefox 1.61 17.6±0.51µs ? ?/sec wasmtime 1.65 18.1±0.73µs ? ?/sec chromium 1.87 20.4±0.16µs ? ?/sec nodejs 2.30 25.1±0.56µs ? ?/sec -firefox 2.36 25.7±1.03µs ? ?/sec faster_hex_decode_unchecked --------------------------- native 1.00 1239.7±16.97ns ? ?/sec wasmer 14.27 17.7±0.35µs ? ?/sec wasmtime 14.36 17.8±0.23µs ? ?/sec -firefox 14.73 18.3±0.75µs ? ?/sec +firefox 14.38 17.8±1.83µs ? ?/sec chromium 16.53 20.5±0.28µs ? ?/sec nodejs 20.36 25.2±0.15µs ? ?/sec @@ -162,23 +162,23 @@ wasmer 19.17 18.2±0.36µs ? ?/sec chromium 21.25 20.2±0.17µs ? ?/sec nodejs 22.85 21.7±0.09µs ? ?/sec wasmtime 24.01 22.8±0.53µs ? ?/sec -firefox 32.68 31.0±3.43µs ? ?/sec +firefox 30.68 29.1±0.89µs ? ?/sec faster_hex_encode_fallback -------------------------- native 1.00 11.1±0.20µs ? ?/sec +firefox 1.98 21.9±0.57µs ? ?/sec chromium 2.04 22.7±0.20µs ? ?/sec wasmtime 2.05 22.8±0.13µs ? ?/sec wasmer 2.06 22.8±0.15µs ? ?/sec -firefox 2.11 23.4±1.79µs ? ?/sec nodejs 2.38 26.4±0.09µs ? ?/sec hex_decode ---------- native 1.00 244.6±2.36µs ? ?/sec +firefox 1.66 405.7±14.22µs ? ?/sec wasmer 1.72 421.4±9.65µs ? ?/sec wasmtime 1.73 423.0±3.00µs ? ?/sec -firefox 1.74 426.4±18.61µs ? ?/sec nodejs 2.00 490.3±3.49µs ? ?/sec chromium 2.81 688.5±12.23µs ? ?/sec @@ -188,6 +188,7 @@ native 1.00 69.2±0.40µs ? ?/sec wasmtime 1.18 81.7±0.38µs ? ?/sec wasmer 1.46 100.9±1.22µs ? ?/sec nodejs 2.20 152.5±1.93µs ? ?/sec +firefox 3.25 224.8±7.53µs ? ?/sec chromium 4.08 282.7±4.19µs ? ?/sec rustc_hex_decode @@ -195,7 +196,7 @@ rustc_hex_decode native 1.00 103.1±2.78µs ? ?/sec wasmer 1.33 136.8±4.06µs ? ?/sec wasmtime 1.38 142.3±3.31µs ? ?/sec -firefox 1.55 160.3±6.43µs ? ?/sec +firefox 1.50 154.7±4.80µs ? ?/sec nodejs 1.78 183.3±2.02µs ? ?/sec chromium 2.04 210.0±3.37µs ? ?/sec @@ -206,7 +207,7 @@ wasmtime 2.24 69.1±0.36µs ? ?/sec wasmer 2.25 69.6±0.74µs ? ?/sec nodejs 2.40 74.2±1.94µs ? ?/sec chromium 2.67 82.6±2.61µs ? ?/sec -firefox 3.45 106.7±10.13µs ? ?/sec +firefox 3.31 102.2±2.66µs ? ?/sec ``` # Caveats and pitfalls From d4593f4a5501931770e38ff00b926a8b6927c51e Mon Sep 17 00:00:00 2001 From: Marco Neumann Date: Sat, 2 Jul 2022 12:35:17 +0200 Subject: [PATCH 19/29] replace `serde_cbor` with `ciborium` (#560) * Bump MSRV to 1.56 as required by ciborium --- .github/workflows/ci.yaml | 2 +- CHANGELOG.md | 1 + Cargo.toml | 2 +- README.md | 2 +- src/connection.rs | 38 +++++++++++++++++++++++++------------- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1b1074dc2..40b413da4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ jobs: rust: - stable - beta - - 1.49.0 # MSRV + - 1.56.0 # MSRV steps: - uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index f96b2f79c..3b846c583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - HTML report hidden behind non-default feature flag: 'html_reports' - Standalone support (ie without cargo-criterion) feature flag: 'cargo_bench_support' +- MSRV bumped to 1.56 ## [Unreleased] ### Changed diff --git a/Cargo.toml b/Cargo.toml index dbeb28c08..56e7d459d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ itertools = "0.10" serde = "1.0" serde_json = "1.0" serde_derive = "1.0" -serde_cbor = "0.11" +ciborium = "0.2.0" atty = "0.2" clap = { version = "2.33", default-features = false } walkdir = "2.3" diff --git a/README.md b/README.md index 2f5262e19..ac4ac727e 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ For more details, see the [CONTRIBUTING.md file](https://github.com/bheisler/cri Criterion.rs supports the last three stable minor releases of Rust. At time of writing, this means Rust 1.50 or later. Older versions may work, but are not guaranteed. -Currently, the oldest version of Rust believed to work is 1.49. Future versions of Criterion.rs may +Currently, the oldest version of Rust believed to work is 1.56. Future versions of Criterion.rs may break support for such old versions, and this will not be considered a breaking change. If you require Criterion.rs to work on old versions of Rust, you will need to stick to a specific patch version of Criterion.rs. diff --git a/src/connection.rs b/src/connection.rs index 0adf20f7b..5de173e27 100644 --- a/src/connection.rs +++ b/src/connection.rs @@ -8,28 +8,39 @@ use std::net::TcpStream; #[derive(Debug)] pub enum MessageError { - SerializationError(serde_cbor::Error), - IoError(std::io::Error), + Deserialization(ciborium::de::Error), + Serialization(ciborium::ser::Error), + Io(std::io::Error), } -impl From for MessageError { - fn from(other: serde_cbor::Error) -> Self { - MessageError::SerializationError(other) +impl From> for MessageError { + fn from(other: ciborium::de::Error) -> Self { + MessageError::Deserialization(other) + } +} +impl From> for MessageError { + fn from(other: ciborium::ser::Error) -> Self { + MessageError::Serialization(other) } } impl From for MessageError { fn from(other: std::io::Error) -> Self { - MessageError::IoError(other) + MessageError::Io(other) } } impl std::fmt::Display for MessageError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - MessageError::SerializationError(error) => write!( + MessageError::Deserialization(error) => write!( + f, + "Failed to deserialize message to Criterion.rs benchmark:\n{}", + error + ), + MessageError::Serialization(error) => write!( f, - "Failed to serialize or deserialize message to Criterion.rs benchmark:\n{}", + "Failed to serialize message to Criterion.rs benchmark:\n{}", error ), - MessageError::IoError(error) => write!( + MessageError::Io(error) => write!( f, "Failed to read or write message to Criterion.rs benchmark:\n{}", error @@ -40,8 +51,9 @@ impl std::fmt::Display for MessageError { impl std::error::Error for MessageError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { - MessageError::SerializationError(err) => Some(err), - MessageError::IoError(err) => Some(err), + MessageError::Deserialization(err) => Some(err), + MessageError::Serialization(err) => Some(err), + MessageError::Io(err) => Some(err), } } } @@ -112,13 +124,13 @@ impl InnerConnection { let length = u32::from_be_bytes(length_buf); self.receive_buffer.resize(length as usize, 0u8); self.socket.read_exact(&mut self.receive_buffer)?; - let value = serde_cbor::from_slice(&self.receive_buffer)?; + let value = ciborium::de::from_reader(&self.receive_buffer[..])?; Ok(value) } pub fn send(&mut self, message: &OutgoingMessage) -> Result<(), MessageError> { self.send_buffer.truncate(0); - serde_cbor::to_writer(&mut self.send_buffer, message)?; + ciborium::ser::into_writer(message, &mut self.send_buffer)?; let size = u32::try_from(self.send_buffer.len()).unwrap(); let length_buf = size.to_be_bytes(); self.socket.write_all(&length_buf)?; From a00c592d703e9ed1b26dfbed6b600eb98c36e5b7 Mon Sep 17 00:00:00 2001 From: Nikhil Benesch Date: Sat, 2 Jul 2022 06:37:04 -0400 Subject: [PATCH 20/29] Upgrade to clap v3 (#543) Clap v3 is a new major version of clap with several deprecations and breaking changes, all of which are addressed within this commit. --- Cargo.toml | 4 +- src/lib.rs | 154 +++++++++++++++++++++-------------------------------- 2 files changed, 64 insertions(+), 94 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 56e7d459d..d64ab4c8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ serde_json = "1.0" serde_derive = "1.0" ciborium = "0.2.0" atty = "0.2" -clap = { version = "2.33", default-features = false } +clap = { version = "3.1", default-features = false, features = ["std"] } walkdir = "2.3" tinytemplate = "1.1" cast = "0.2" @@ -78,7 +78,7 @@ real_blackbox = [] # Enable async/await support async = ["futures"] -# These features enable built-in support for running async benchmarks on each different async +# These features enable built-in support for running async benchmarks on each different async # runtime. async_futures = ["futures/executor", "async"] async_smol = ["smol", "async"] diff --git a/src/lib.rs b/src/lib.rs index 9ae9eaf64..27c193b1c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,7 +36,6 @@ extern crate approx; #[cfg(test)] extern crate quickcheck; -use clap::value_t; use regex::Regex; #[macro_use] @@ -733,148 +732,148 @@ impl Criterion { #[must_use] #[cfg_attr(feature = "cargo-clippy", allow(clippy::cognitive_complexity))] pub fn configure_from_args(mut self) -> Criterion { - use clap::{App, Arg}; - let matches = App::new("Criterion Benchmark") - .arg(Arg::with_name("FILTER") + use clap::{Arg, Command}; + let matches = Command::new("Criterion Benchmark") + .arg(Arg::new("FILTER") .help("Skip benchmarks whose names do not contain FILTER.") .index(1)) - .arg(Arg::with_name("color") - .short("c") + .arg(Arg::new("color") + .short('c') .long("color") .alias("colour") .takes_value(true) .possible_values(&["auto", "always", "never"]) .default_value("auto") .help("Configure coloring of output. always = always colorize output, never = never colorize output, auto = colorize output if output is a tty and compiled for unix.")) - .arg(Arg::with_name("verbose") - .short("v") + .arg(Arg::new("verbose") + .short('v') .long("verbose") .help("Print additional statistical information.")) - .arg(Arg::with_name("quiet") + .arg(Arg::new("quiet") .long("quiet") .conflicts_with("verbose") .help("Print only the benchmark results.")) - .arg(Arg::with_name("noplot") - .short("n") + .arg(Arg::new("noplot") + .short('n') .long("noplot") .help("Disable plot and HTML generation.")) - .arg(Arg::with_name("save-baseline") - .short("s") + .arg(Arg::new("save-baseline") + .short('s') .long("save-baseline") .default_value("base") .help("Save results under a named baseline.")) - .arg(Arg::with_name("discard-baseline") + .arg(Arg::new("discard-baseline") .long("discard-baseline") .conflicts_with_all(&["save-baseline", "baseline", "baseline-lenient"]) .help("Discard benchmark results.")) - .arg(Arg::with_name("baseline") - .short("b") + .arg(Arg::new("baseline") + .short('b') .long("baseline") .takes_value(true) .conflicts_with_all(&["save-baseline", "baseline-lenient"]) .help("Compare to a named baseline. If any benchmarks do not have the specified baseline this command fails.")) - .arg(Arg::with_name("baseline-lenient") + .arg(Arg::new("baseline-lenient") .long("baseline-lenient") .takes_value(true) .conflicts_with_all(&["save-baseline", "baseline"]) .help("Compare to a named baseline. If any benchmarks do not have the specified baseline then just those benchmarks are not compared against the baseline while every other benchmark is compared against the baseline.")) - .arg(Arg::with_name("list") + .arg(Arg::new("list") .long("list") .help("List all benchmarks") .conflicts_with_all(&["test", "profile-time"])) - .arg(Arg::with_name("profile-time") + .arg(Arg::new("profile-time") .long("profile-time") .takes_value(true) .help("Iterate each benchmark for approximately the given number of seconds, doing no analysis and without storing the results. Useful for running the benchmarks in a profiler.") .conflicts_with_all(&["test", "list"])) - .arg(Arg::with_name("export") + .arg(Arg::new("export") .long("export") .takes_value(true) .help("Export baseline as json, printed to stdout") .conflicts_with_all(&["list", "test", "profile-time", "compare"])) - .arg(Arg::with_name("compare") + .arg(Arg::new("compare") .long("compare") .help("Tabulate benchmark results") .conflicts_with_all(&["list", "test", "profile-time", "export"])) - .arg(Arg::with_name("baselines") + .arg(Arg::new("baselines") .long("baselines") - .multiple(true) + .multiple_occurrences(true) .value_name("baselines") .requires("compare") - .require_delimiter(true) + .require_value_delimiter(true) .help("Limit the baselines used in tabulated results.") .help("")) - .arg(Arg::with_name("compare-threshold") + .arg(Arg::new("compare-threshold") .long("compare-threshold") .takes_value(true) .help("Hide results that differ by less than the threshold percentage. By default, all results are shown.")) - .arg(Arg::with_name("compare-list") + .arg(Arg::new("compare-list") .long("compare-list") .help("Show benchmark results in a list rather than in a table. Useful when horizontal space is limited.")) - .arg(Arg::with_name("load-baseline") + .arg(Arg::new("load-baseline") .long("load-baseline") .takes_value(true) .conflicts_with("profile-time") .requires("baseline") .help("Load a previous baseline instead of sampling new data.")) - .arg(Arg::with_name("sample-size") + .arg(Arg::new("sample-size") .long("sample-size") .takes_value(true) - .help(&format!("Changes the default size of the sample for this run. [default: {}]", self.config.sample_size))) - .arg(Arg::with_name("warm-up-time") + .help(&*format!("Changes the default size of the sample for this run. [default: {}]", self.config.sample_size))) + .arg(Arg::new("warm-up-time") .long("warm-up-time") .takes_value(true) - .help(&format!("Changes the default warm up time for this run. [default: {}]", self.config.warm_up_time.as_secs()))) - .arg(Arg::with_name("measurement-time") + .help(&*format!("Changes the default warm up time for this run. [default: {}]", self.config.warm_up_time.as_secs()))) + .arg(Arg::new("measurement-time") .long("measurement-time") .takes_value(true) - .help(&format!("Changes the default measurement time for this run. [default: {}]", self.config.measurement_time.as_secs()))) - .arg(Arg::with_name("nresamples") + .help(&*format!("Changes the default measurement time for this run. [default: {}]", self.config.measurement_time.as_secs()))) + .arg(Arg::new("nresamples") .long("nresamples") .takes_value(true) - .help(&format!("Changes the default number of resamples for this run. [default: {}]", self.config.nresamples))) - .arg(Arg::with_name("noise-threshold") + .help(&*format!("Changes the default number of resamples for this run. [default: {}]", self.config.nresamples))) + .arg(Arg::new("noise-threshold") .long("noise-threshold") .takes_value(true) - .help(&format!("Changes the default noise threshold for this run. [default: {}]", self.config.noise_threshold))) - .arg(Arg::with_name("confidence-level") + .help(&*format!("Changes the default noise threshold for this run. [default: {}]", self.config.noise_threshold))) + .arg(Arg::new("confidence-level") .long("confidence-level") .takes_value(true) - .help(&format!("Changes the default confidence level for this run. [default: {}]", self.config.confidence_level))) - .arg(Arg::with_name("significance-level") + .help(&*format!("Changes the default confidence level for this run. [default: {}]", self.config.confidence_level))) + .arg(Arg::new("significance-level") .long("significance-level") .takes_value(true) - .help(&format!("Changes the default significance level for this run. [default: {}]", self.config.significance_level))) - .arg(Arg::with_name("quick") + .help(&*format!("Changes the default significance level for this run. [default: {}]", self.config.significance_level))) + .arg(Arg::new("quick") .long("quick") .conflicts_with("sample-size") - .help(&format!("Benchmark only until the significance level has been reached [default: {}]", self.config.quick_mode))) - .arg(Arg::with_name("test") - .hidden(true) + .help(&*format!("Benchmark only until the significance level has been reached [default: {}]", self.config.quick_mode))) + .arg(Arg::new("test") + .hide(true) .long("test") .help("Run the benchmarks once, to verify that they execute successfully, but do not measure or report the results.") .conflicts_with_all(&["list", "profile-time"])) - .arg(Arg::with_name("bench") - .hidden(true) + .arg(Arg::new("bench") + .hide(true) .long("bench")) - .arg(Arg::with_name("plotting-backend") + .arg(Arg::new("plotting-backend") .long("plotting-backend") .takes_value(true) .possible_values(&["gnuplot", "plotters"]) .help("Set the plotting backend. By default, Criterion.rs will use the gnuplot backend if gnuplot is available, or the plotters backend if it isn't.")) - .arg(Arg::with_name("output-format") + .arg(Arg::new("output-format") .long("output-format") .takes_value(true) .possible_values(&["criterion", "bencher"]) .default_value("criterion") .help("Change the CLI output format. By default, Criterion.rs will use its own format. If output format is set to 'bencher', Criterion.rs will print output in a format that resembles the 'bencher' crate.")) - .arg(Arg::with_name("nocapture") + .arg(Arg::new("nocapture") .long("nocapture") - .hidden(true) + .hide(true) .help("Ignored, but added for compatibility with libtest.")) - .arg(Arg::with_name("version") - .hidden(true) - .short("V") + .arg(Arg::new("version") + .hide(true) + .short('V') .long("version")) .after_help(" This executable is a Criterion.rs benchmark. @@ -937,10 +936,7 @@ https://bheisler.github.io/criterion.rs/book/faq.html } else if matches.is_present("list") { Mode::List } else if matches.is_present("profile-time") { - let num_seconds = value_t!(matches.value_of("profile-time"), f64).unwrap_or_else(|e| { - println!("{}", e); - std::process::exit(1) - }); + let num_seconds = matches.value_of_t_or_exit("profile-time"); if num_seconds < 1.0 { eprintln!("Profile time must be at least one second."); @@ -1036,19 +1032,13 @@ https://bheisler.github.io/criterion.rs/book/faq.html } if matches.is_present("sample-size") { - let num_size = value_t!(matches.value_of("sample-size"), usize).unwrap_or_else(|e| { - println!("{}", e); - std::process::exit(1) - }); + let num_size = matches.value_of_t_or_exit("sample-size"); assert!(num_size >= 10); self.config.sample_size = num_size; } if matches.is_present("warm-up-time") { - let num_seconds = value_t!(matches.value_of("warm-up-time"), f64).unwrap_or_else(|e| { - println!("{}", e); - std::process::exit(1) - }); + let num_seconds = matches.value_of_t_or_exit("warm-up-time"); let dur = std::time::Duration::from_secs_f64(num_seconds); assert!(dur.as_nanos() > 0); @@ -1056,11 +1046,7 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.config.warm_up_time = dur; } if matches.is_present("measurement-time") { - let num_seconds = - value_t!(matches.value_of("measurement-time"), f64).unwrap_or_else(|e| { - println!("{}", e); - std::process::exit(1) - }); + let num_seconds = matches.value_of_t_or_exit("measurement-time"); let dur = std::time::Duration::from_secs_f64(num_seconds); assert!(dur.as_nanos() > 0); @@ -1068,44 +1054,28 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.config.measurement_time = dur; } if matches.is_present("nresamples") { - let num_resamples = - value_t!(matches.value_of("nresamples"), usize).unwrap_or_else(|e| { - println!("{}", e); - std::process::exit(1) - }); + let num_resamples = matches.value_of_t_or_exit("nresamples"); assert!(num_resamples > 0); self.config.nresamples = num_resamples; } if matches.is_present("noise-threshold") { - let num_noise_threshold = value_t!(matches.value_of("noise-threshold"), f64) - .unwrap_or_else(|e| { - println!("{}", e); - std::process::exit(1) - }); + let num_noise_threshold = matches.value_of_t_or_exit("noise-threshold"); assert!(num_noise_threshold > 0.0); self.config.noise_threshold = num_noise_threshold; } if matches.is_present("confidence-level") { - let num_confidence_level = value_t!(matches.value_of("confidence-level"), f64) - .unwrap_or_else(|e| { - println!("{}", e); - std::process::exit(1) - }); + let num_confidence_level = matches.value_of_t_or_exit("confidence-level"); assert!(num_confidence_level > 0.0 && num_confidence_level < 1.0); self.config.confidence_level = num_confidence_level; } if matches.is_present("significance-level") { - let num_significance_level = value_t!(matches.value_of("significance-level"), f64) - .unwrap_or_else(|e| { - println!("{}", e); - std::process::exit(1) - }); + let num_significance_level = matches.value_of_t_or_exit("significance-level"); assert!(num_significance_level > 0.0 && num_significance_level < 1.0); @@ -1132,7 +1102,7 @@ https://bheisler.github.io/criterion.rs/book/faq.html let args = critcmp::app::Args { baselines: matches.values_of_lossy("baselines").unwrap_or_default(), output_list: matches.is_present("compare-list"), - threshold: value_t!(matches.value_of("compare-threshold"), f64).ok(), // FIXME: Print error message if parsing fails. + threshold: matches.value_of_t("compare-threshold").ok(), // FIXME: Print error message if parsing fails. color: enable_text_coloring, filter: self.filter, }; From 1a66bb7e429b9503b24160f2939f1a5382d0b6b7 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Sat, 2 Jul 2022 14:07:13 +0200 Subject: [PATCH 21/29] Bump MSRV to 1.56.1 --- .github/workflows/ci.yaml | 2 +- CHANGELOG.md | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 40b413da4..d4f3d417e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ jobs: rust: - stable - beta - - 1.56.0 # MSRV + - 1.56.1 # MSRV steps: - uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b846c583..e0d966af3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - HTML report hidden behind non-default feature flag: 'html_reports' - Standalone support (ie without cargo-criterion) feature flag: 'cargo_bench_support' -- MSRV bumped to 1.56 +- MSRV bumped to 1.56.1 ## [Unreleased] ### Changed diff --git a/README.md b/README.md index ac4ac727e..5d0c433ec 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ For more details, see the [CONTRIBUTING.md file](https://github.com/bheisler/cri Criterion.rs supports the last three stable minor releases of Rust. At time of writing, this means Rust 1.50 or later. Older versions may work, but are not guaranteed. -Currently, the oldest version of Rust believed to work is 1.56. Future versions of Criterion.rs may +Currently, the oldest version of Rust believed to work is 1.56.1. Future versions of Criterion.rs may break support for such old versions, and this will not be considered a breaking change. If you require Criterion.rs to work on old versions of Rust, you will need to stick to a specific patch version of Criterion.rs. From 5cb91ab57691601039c0d25666faedc506dfa899 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Sat, 2 Jul 2022 14:07:23 +0200 Subject: [PATCH 22/29] Set required clap parameter. --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 1e31abd49..b1a128598 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -801,6 +801,7 @@ impl Criterion { .value_name("baselines") .requires("compare") .require_value_delimiter(true) + .use_value_delimiter(true) .help("Limit the baselines used in tabulated results.") .help("")) .arg(Arg::new("compare-threshold") From 6127032c8e82888757adecb8dc5843d5b1700c49 Mon Sep 17 00:00:00 2001 From: guanqun Date: Sat, 2 Jul 2022 22:09:15 +0800 Subject: [PATCH 23/29] bump the regex to 1.5 (#525) --- Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d64ab4c8d..c85e1ffda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,10 +32,10 @@ tinytemplate = "1.1" cast = "0.2" num-traits = { version = "0.2", default-features = false, features = ["std"] } oorandom = "11.1" -regex = { version = "1.3", default-features = false, features = ["std"] } -tabwriter = "1.2.1" -termcolor = "1.1.2" -unicode-width = "0.1.9" +regex = { version = "1.5", default-features = false, features = ["std"] } +tabwriter = "1.2.1" +termcolor = "1.1.2" +unicode-width = "0.1.9" # Optional dependencies rayon = { version = "1.3", optional = true } From 758b324615da6ef8eb63e55579edd57a9cd64a3b Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Sat, 2 Jul 2022 16:27:52 +0200 Subject: [PATCH 24/29] Bump minimum version of 'anes' to 0.1.4 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1d147b5c1..c7896289a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ license = "Apache-2.0/MIT" exclude = ["book/*"] [dependencies] -anes = "0.1" +anes = "0.1.4" lazy_static = "1.4" criterion-plot = { path = "plot", version = "0.4.4" } itertools = "0.10" From ecfd1d15b0def59a4b6e313553153c26a755f694 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Sat, 2 Jul 2022 16:47:51 +0200 Subject: [PATCH 25/29] Update CHANGELOG with v0.4 changes. --- CHANGELOG.md | 139 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 108 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0d966af3..d2eb65312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # Changelog + All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) @@ -7,35 +8,50 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [0.4.0] - IN DEVELOPMENT ### Removed + - The `Criterion::can_plot` function has been removed. - The `Criterion::bench_function_over_inputs` function has been removed. - The `Criterion::bench_functions` function has been removed. - The `Criterion::bench` function has been removed. ### Changed + - HTML report hidden behind non-default feature flag: 'html_reports' - Standalone support (ie without cargo-criterion) feature flag: 'cargo_bench_support' - MSRV bumped to 1.56.1 +- `rayon` and `plotters` are optional (and default) dependencies. +- Status messages ('warming up', 'analyzing', etc) are printed to stderr, benchmark results are printed to stdout. +- Accept subsecond durations for `--warm-up-time`, `--measurement-time` and `--profile-time`. +- Replaced serde_cbor with ciborium because the former is no longer maintained. +- Upgrade clap to v3 and regex to v1.5. -## [Unreleased] -### Changed -- MSRV bumped to 1.49 +### Added + +- A `--discard-baseline` flag for discarding rather than saving benchmark results. +- Formal support for benchmarking code compiled to web-assembly. +- A `--quiet` flag for printing just a single line per benchmark. ## [0.3.5] - 2021-07-26 + ### Fixed + - Corrected `Criterion.toml` in the book. - Corrected configuration typo in the book. ### Changed + - Bump plotters dependency to always include a bug-fix. - MSRV bumped to 1.46. ## [0.3.4] - 2021-01-24 + ### Added + - Added support for benchmarking async functions - Added `with_output_color` for enabling or disabling CLI output coloring programmatically. ### Fixed + - Criterion.rs will now give a clear error message in case of benchmarks that take zero time. - Added some extra code to ensure that every sample has at least one iteration. - Added a notice to the `--help` output regarding "unrecognized option" errors. @@ -44,19 +60,20 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Criterion.rs will now automatically detect the right output directory. ### Deprecated + - `Criterion::can_plot` is no longer useful and is deprecated pending deletion in 0.4.0. -- `Benchmark` and `ParameterizedBenchmark` were already hidden from documentation, but are now +- `Benchmark` and `ParameterizedBenchmark` were already hidden from documentation, but are now formally deprecated pending deletion in 0.4.0. Callers should use `BenchmarkGroup` instead. - `Criterion::bench_function_over_inputs`, `Criterion::bench_functions`, and `Criterion::bench` were already hidden from documentation, but are now formally deprecated pending deletion in 0.4.0. Callers should use `BenchmarkGroup` instead. -- Three new optional features have been added; "html_reports", "csv_output" and - "cargo_bench_support". These features currently do nothing except disable a warning message at - runtime, but in version 0.4.0 they will be used to enable HTML report generation, CSV file - generation, and the ability to run in cargo-bench (as opposed to [cargo-criterion]). +- Three new optional features have been added; "html_reports", "csv_output" and + "cargo_bench_support". These features currently do nothing except disable a warning message at + runtime, but in version 0.4.0 they will be used to enable HTML report generation, CSV file + generation, and the ability to run in cargo-bench (as opposed to [cargo-criterion]). "cargo_bench_support" is enabled by default, but "html_reports" and "csv_output" are not. If you use Criterion.rs' HTML reports, it is recommended to switch to [cargo-criterion]. - If you use CSV output, it is recommended to switch to [cargo-criterion] and use the + If you use CSV output, it is recommended to switch to [cargo-criterion] and use the `--message-format=json` option for machine-readable output instead. A warning message will be printed at the start of benchmark runs which do not have "html_reports" or "cargo_bench_support" enabled, but because CSV output is not widely used it has no warning. @@ -64,11 +81,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [cargo-criterion]: https://github.com/bheisler/cargo-criterion ## [0.3.3] - 2020-06-29 + ### Added + - Added `CRITERION_HOME` environment variable to set the directory for Criterion to store - its results and charts in. -- Added support for [cargo-criterion]. The long-term goal here is to remove code from Criterion-rs - itself to improve compile times, as well as to add features to `cargo-criterion` that are + its results and charts in. +- Added support for [cargo-criterion]. The long-term goal here is to remove code from Criterion-rs + itself to improve compile times, as well as to add features to `cargo-criterion` that are difficult to implement in Criterion-rs. - Add sampling mode option for benchmarks. This allows the user to change how Criterion.rs chooses the iteration counts in each sample. By default, nothing will change for most benchmarks, but @@ -76,11 +95,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. This affects the statistics and plots generated. ### Changed + - The serialization format for some of the files has changed. This may cause your first benchmark run after updating to produce errors, but they're harmless and will go away after running the benchmarks once. ### Fixed + - Fixed a bug where the current measurement was not shown on the relative regression plot. - Fixed rare panic in the plotters backend. - Panic with a clear error message (rather than panicking messily later on) when the user sets the @@ -88,7 +109,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Escape single quotes in benchmark names when generating Gnuplot scripts. ## [0.3.2] - 2020-04-26 + ### Added + - Added `?Sized` bound to benchmark parameter types, which allows dynamically sized types like `&str` and `&[T]` to be used as benchmark parameters. - Added the `--output-format ` command-line option. If `--output-format bencher` is passed, @@ -100,15 +123,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. when running tests or benchmarks and allowing stdout output from other tests. ### Fixed + - Fixed panic when environment variables contains non-UTF8 characters. -- Fixed panic when `CRITERION_DEBUG` or `CRITERION_TARGET_DIR` environment variables contain +- Fixed panic when `CRITERION_DEBUG` or `CRITERION_TARGET_DIR` environment variables contain non-UTF8 characters. ## [0.3.1] - 2020-01-25 + ### Added -- Added new plotting backend using the `plotters` crate. Implementation generously provided by Hao + +- Added new plotting backend using the `plotters` crate. Implementation generously provided by Hao Hou, author of the `plotters` crate. -- Added `--plotting-backend` command-line option to select the plotting backend. The existing +- Added `--plotting-backend` command-line option to select the plotting backend. The existing gnuplot backend will be used by default when available, and the plotters backend will be used when gnuplot is not available or when requested. - Added `Criterion::plotting_backend()` function to configure the plotting backend in code. @@ -117,6 +143,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Benchmark filters can now be regular expressions. ### Fixed + - Fixed `fibonacci` functions. - Fixed `#[criterion]` benchmarks ignoring the command-line options. - Fixed incorrect scaling of the violin plots. @@ -124,11 +151,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. sample count. - Fix potential panic when `nresamples` is set too low. Also added a warning against setting `nresamples` too low. -- Fixed issue where a slow outer closure would cause Criterion.rs to calculate +- Fixed issue where a slow outer closure would cause Criterion.rs to calculate the wrong estimated time and number of iterations in the warm-up phase. ## [0.3.0] - 2019-08-25 + ### Added + - Added support for plugging in custom measurements (eg. processor counters) into Criterion.rs' measurement and analysis. - Added support for plugging in instrumentation for internal profilers such as @@ -139,7 +168,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. `BenchmarkGroup` performs the same function as all of the above, but is cleaner to use and more powerful and flexible. All of these types/functions are now soft-deprecated (meaning they're hidden from the documentation and should not be used in new code). They will be fully deprecated - at some point in the 0.3.* series and removed in 0.4.0. + at some point in the 0.3.\* series and removed in 0.4.0. - `iter_custom` - a "timing loop" that allows the caller to perform their own measurements. This is useful for complex measurements that don't fit into the usual mode of calling a lambda in a loop. - If the benchmark cannot be completed in approximately the requested measurement time, @@ -148,67 +177,84 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Added command-line options to set the defaults for warm-up time, measurement-time, etc. ### Changed + - The `raw.csv` file format has been changed slightly. The `sample_time_nanos` field has been split into `sample_measured_value` and `unit` fields to accommodate custom measurements. - Throughput has been expanded from u32 to u64 to accommodate very large input sizes. ### Fixed + - Fixed possible invalid file name error on Windows - Fixed potential case where data for two different benchmarks would be stored in the same directory. ### Removed + - Removed the `--measure-only` command-line argument; it was deprecated in favor of `--profile-time` in 0.2.6. -- External program benchmarks have been removed; they were deprecated in 0.2.6. The new +- External program benchmarks have been removed; they were deprecated in 0.2.6. The new `iter_custom` timing loop can be used as a substitute; see `benches/external_process.rs` for an example of this. ### Deprecated + - The `--test` argument is now deprecated. To test benchmarks, use `cargo test --benches`. ## [0.2.11] - 2019-04-08 + ### Added + - Enabled automatic text-coloring on Windows. ### Fixed + - Fixed panic caused by outdated files after benchmark names or types were changed. - Reduced timing overhead of `Criterion::iter_batched/iter_batched_ref`. ## [0.2.10] - 2019-02-09 + ### Added -- Added `iter_batched/iter_batched_ref` timing loops, which allow for setup (like + +- Added `iter_batched/iter_batched_ref` timing loops, which allow for setup (like `iter_with_setup/iter_with_large_setup`) and exclude drop (like `iter_with_large_drop`) but measure the runtime more accurately, use less memory and are more flexible. ### Deprecated + - `iter_with_setup/iter_with_large_setup` are now deprecated in favor of `iter_batched`. ## [0.2.9] - 2019-01-24 + ### Changed + - Criterion.rs no longer depends on the default features of the `rand-core` crate. This fixes some downstream crates which use `rand` in a `no_std` context. ## [0.2.8] - 2019-01-20 + ### Changed + - Criterion.rs now uses `rayon` internally instead of manual `unsafe` code built with thread-scoped. - Replaced handlebars templates with [TinyTemplate](https://github.com/bheisler/TinyTemplate) - Merged `criterion-stats` crate into `criterion` crate. `criterion-stats` will no longer receive updates. -- Replaced or removed various other dependencies to reduce the size of Criterion.rs' dependency +- Replaced or removed various other dependencies to reduce the size of Criterion.rs' dependency tree. ## [0.2.7] - 2018-12-29 ### Fixed + - Fixed version numbers to prevent incompatibilities between `criterion` and `criterion-stats` crates. ## [0.2.6] - 2018-12-27 - Yanked + ### Added + - Added `--list` command line option, which lists the benchmarks but does not run them, to match `cargo test -- --list`. - Added README/CONTRIBUTING/LICENSE files to sub-crates. -- Displays change in throughput in the command-line and HTML output as well as change in iteration +- Displays change in throughput in the command-line and HTML output as well as change in iteration time. - Benchmarks with multiple functions and multiple values will now generate a per-value summary report file in addition to the existing per-function one. @@ -217,8 +263,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. (now-deprecated) `--measure-only` argument. ### Fixed -- Functions passed to `Bencher::iter_with_large_setup` can now return output. This is necessary to - prevent the compiler from optimizing away the benchmark. This is technically a breaking change - + +- Functions passed to `Bencher::iter_with_large_setup` can now return output. This is necessary to + prevent the compiler from optimizing away the benchmark. This is technically a breaking change - that function requires a new type parameter. It's so unlikely to break existing code that I decided not to delay this for a breaking-change release. - Reduced measurement overhead for the `iter_with_large_setup` and `iter_with_drop` methods. @@ -228,16 +275,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Criterion.rs macros no longer require user to `use criterion::Criterion;` - Criterion.rs no longer initializes a logger, meaning that it will no longer conflict with user code which does. -- Criterion.rs no longer fails to parse gnuplot version numbers like +- Criterion.rs no longer fails to parse gnuplot version numbers like `gnuplot 5.2 patchlevel 5a (Gentoo revision r0)` -- Criterion.rs no longer prints an error message that gnuplot couldn't be found when chart - generation is disabled (either by `Criterion::without_plots`, `--noplot` or disabling the +- Criterion.rs no longer prints an error message that gnuplot couldn't be found when chart + generation is disabled (either by `Criterion::without_plots`, `--noplot` or disabling the HTML reports feature) - Benchmark names are now automatically truncated to 100 characters and a number may be added to make them unique. This fixes a problem where gnuplot would crash if the title was extremely long, and also improves the general usability of Criterion.rs. ### Changed + - Changed timing model of `iter_with_large_setup` to exclude time spent dropping values returned by the routine. Time measurements taken with 0.2.6 using these methods may differ from those taken with 0.2.5. @@ -245,6 +293,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. tree in the benchmark index. This is to accommodate the new per-value summary reports. ### Deprecated + - Deprecated the `--measure-only` command-line-argument in favor of `--profile-time`. This will be removed in 0.3.0. - External-program benchmarks are now deprecated. They will be removed in 0.3.0. @@ -254,11 +303,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. may be breaking changes that are not listed here. ## [0.2.5] - 2018-08-27 + ### Fixed + - Fixed links from generated report files to documentation. - Fixed formatting for very large percentage changes (>1000%) - Sorted the benchmarks in the index report by name -- Fixed case where benchmark ID with special characters would cause Criterion.rs to open the wrong +- Fixed case where benchmark ID with special characters would cause Criterion.rs to open the wrong file and log an error message. - Fixed case where running `cargo clean; cargo bench -- ` would cause Criterion.rs to log an error message. @@ -269,11 +320,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Criterion.rs now honors the `CARGO_TARGET_DIR` environment variable. ### Added + - Criterion.rs will generate a chart showing the effects of changes in input (or input size) for all benchmarks with numeric inputs or throughput, not just for those which compare multiple functions. ## [0.2.4] 2018-07-08 + ### Added + - Added a pair of flags, `--save-baseline` and `--baseline`, which change how benchmark results are stored and compared. This is useful for working against a fixed baseline(eg. comparing progress on an @@ -294,7 +348,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. private implementation details. ### Fixed -- The `sample_size` method on the `Criterion`, `Benchmark` and + +- The `sample_size` method on the `Criterion`, `Benchmark` and `ParameterizedBenchmark` structs has been changed to panic if the sample size is less than 2. Other parts of the code require this and will panic if the sample size is 1, so this is not considered to be a breaking change. @@ -303,13 +358,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. directory paths, to avoid generating invalid or unexpected paths. ## [0.2.3] - 2018-04-14 + ### Fixed + - Criterion.rs will now panic with a clear error message if the user attempts to run a benchmark which doesn't call the `Bencher::iter` function or a related function, rather than failing in an uncontrolled manner later. - Fixed broken links in some more summary reports. ### Added + - Added a `--measure-only` argument which causes the benchmark executable to run the warmup and measurement and then move on to the next benchmark without analyzing or saving data. This is useful to prevent Criterion.rs' analysis code from appearing @@ -318,12 +376,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. the other reports for easy navigation. ## [0.2.2] - 2018-03-25 + ### Fixed + - Fixed broken links in some summary reports. - Work around apparent rustc bug in >= 1.24.0. ## [0.2.1] - 2018-02-24 + ### Added + - HTML reports are now a default Cargo feature. If you wish to disable HTML reports, disable Criterion.rs' default features. Doing so will allow compatibility with older Rust versions such as 1.20. If you wish to continue using HTML reports, you @@ -332,14 +394,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. inputs. ### Changed + - The plots and HTML reports are now generated in a `report` folder. ### Fixed + - Underscores in benchmark names will no longer cause subscripted characters to appear in generated plots. ## [0.2.0] - 2018-02-05 + ### Added + - Added `Criterion.bench` function, which accepts either a `Benchmark` or `ParameterizedBenchmark`. These new structures allow for custom per-benchmark configuration as well as more complex benchmark grouping (eg. comparing a Rust @@ -352,6 +418,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Added `--noplot` command line option to disable plot generation. ### Changed + - The builder methods on the Criterion struct now take and return self by value for easier chaining. Functions which configure a Criterion structure will need to be updated accordingly, or will need to be changed to work with the @@ -365,16 +432,20 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - The generated plots are stored in `target/criterion` rather than `.criterion`. ### Removed + - The hidden `criterion::ConfidenceInterval` and`criterion::Estimate` types are no longer publicly accessible. - The `Criterion.summarize` function has been removed. ### Fixed + - Fixed the relative mean and median reports. - Fixed panic while summarizing benchmarks. ## [0.1.2] - 2018-01-12 + ### Changed + - Criterion.rs is now stable-compatible! - Criterion.rs now includes its own stable-compatible `black_box` function. Some benchmarks may now be affected by dead-code-elimination where they @@ -385,34 +456,40 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. and reduce noise. ### Added + - Running benchmarks with the variable "CRITERION_DEBUG" in the environment will cause Criterion.rs to generate extra debug output and save the gnuplot scripts alongside the generated plots. ### Fixed + - Don't panic on IO errors or gnuplot failures - Fix generation of invalid gnuplot scripts when benchmarking over inputs and inputs include values <= 0. - Bug where benchmarks would run one sample fewer than was configured. ### Removed + - Generated plots will no longer use log-scale. ## [0.1.1] - 2017-12-12 + ### Added + - A changelog file. - Added a chapter to the book on how Criterion.rs collects and analyzes data. - Added macro rules to generate a test harness for use with `cargo bench`. Benchmarks defined without these macros should continue to work. - New contribution guidelines - Criterion.rs can selectively run benchmarks. See the Command-line page for -more details + more details ## 0.1.0 - 2017-12-02 + ### Added -- Initial release on Crates.io. +- Initial release on Crates.io. -[Unreleased]: https://github.com/bheisler/criterion.rs/compare/0.3.4...HEAD +[unreleased]: https://github.com/bheisler/criterion.rs/compare/0.3.4...HEAD [0.1.1]: https://github.com/bheisler/criterion.rs/compare/0.1.0...0.1.1 [0.1.2]: https://github.com/bheisler/criterion.rs/compare/0.1.1...0.1.2 [0.2.0]: https://github.com/bheisler/criterion.rs/compare/0.1.2...0.2.0 From 2934163518a75bf037c7467c0a1d28443203eabf Mon Sep 17 00:00:00 2001 From: Brook Heisler Date: Thu, 14 Jul 2022 18:29:49 -0600 Subject: [PATCH 26/29] Add missing black_box for bench_with_input parameters. Fixes 566. --- CHANGELOG.md | 4 ++++ src/routine.rs | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2eb65312..6b5a11531 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Formal support for benchmarking code compiled to web-assembly. - A `--quiet` flag for printing just a single line per benchmark. +### Fixed +- When using `bench_with_input`, the input parameter will now be passed through `black_box` before + passing it to the benchmark. + ## [0.3.5] - 2021-07-26 ### Fixed diff --git a/src/routine.rs b/src/routine.rs index b4bb7c152..9567fb438 100644 --- a/src/routine.rs +++ b/src/routine.rs @@ -2,7 +2,7 @@ use crate::benchmark::BenchmarkConfig; use crate::connection::OutgoingMessage; use crate::measurement::Measurement; use crate::report::{BenchmarkId, Report, ReportContext}; -use crate::{ActualSamplingMode, Bencher, Criterion}; +use crate::{black_box, ActualSamplingMode, Bencher, Criterion}; use std::marker::PhantomData; use std::time::Duration; @@ -247,7 +247,7 @@ where .iter() .map(|iters| { b.iters = *iters; - (*f)(&mut b, parameter); + (*f)(&mut b, black_box(parameter)); b.assert_iterated(); m.to_f64(&b.value) }) @@ -267,7 +267,7 @@ where let mut total_iters = 0; let mut elapsed_time = Duration::from_millis(0); loop { - (*f)(&mut b, parameter); + (*f)(&mut b, black_box(parameter)); b.assert_iterated(); From f82ce59d710e8b5cc30b9a9b07a9da3e8e715768 Mon Sep 17 00:00:00 2001 From: David Himmelstrup Date: Tue, 23 Aug 2022 15:25:17 +0200 Subject: [PATCH 27/29] Remove critcmp code (it belongs in cargo-criterion) (#610) * Delete critcmp code (it belongs in cargo-criterion) * Bump MSRV to 1.57 due to os_str_bytes. * Mention MSRV bump in CHANGELOG. --- .github/workflows/ci.yaml | 2 +- CHANGELOG.md | 2 +- Cargo.toml | 3 - README.md | 2 +- book/src/SUMMARY.md | 3 +- book/src/user_guide/tabulating_results.md | 294 ---------------------- src/critcmp/app.rs | 150 ----------- src/critcmp/data.rs | 227 ----------------- src/critcmp/main.rs | 131 ---------- src/critcmp/mod.rs | 5 - src/critcmp/output.rs | 234 ----------------- src/lib.rs | 75 ------ 12 files changed, 4 insertions(+), 1124 deletions(-) delete mode 100644 book/src/user_guide/tabulating_results.md delete mode 100644 src/critcmp/app.rs delete mode 100644 src/critcmp/data.rs delete mode 100644 src/critcmp/main.rs delete mode 100644 src/critcmp/mod.rs delete mode 100644 src/critcmp/output.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d4f3d417e..d7f39e495 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ jobs: rust: - stable - beta - - 1.56.1 # MSRV + - 1.57 # MSRV steps: - uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 89f4d80a2..053bda9a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - HTML report hidden behind non-default feature flag: 'html_reports' - Standalone support (ie without cargo-criterion) feature flag: 'cargo_bench_support' -- MSRV bumped to 1.56.1 +- MSRV bumped to 1.57 - `rayon` and `plotters` are optional (and default) dependencies. - Status messages ('warming up', 'analyzing', etc) are printed to stderr, benchmark results are printed to stdout. - Accept subsecond durations for `--warm-up-time`, `--measurement-time` and `--profile-time`. diff --git a/Cargo.toml b/Cargo.toml index 75b379502..6470b9623 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,9 +33,6 @@ cast = "0.3" num-traits = { version = "0.2", default-features = false, features = ["std"] } oorandom = "11.1" regex = { version = "1.5", default-features = false, features = ["std"] } -tabwriter = "1.2.1" -termcolor = "1.1.2" -unicode-width = "0.1.9" # Optional dependencies rayon = { version = "1.3", optional = true } diff --git a/README.md b/README.md index a243e9831..a4dfc091b 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ For more details, see the [CONTRIBUTING.md file](https://github.com/bheisler/cri Criterion.rs supports the last three stable minor releases of Rust. At time of writing, this means Rust 1.59 or later. Older versions may work, but are not guaranteed. -Currently, the oldest version of Rust believed to work is 1.56.1. Future versions of Criterion.rs may +Currently, the oldest version of Rust believed to work is 1.57. Future versions of Criterion.rs may break support for such old versions, and this will not be considered a breaking change. If you require Criterion.rs to work on old versions of Rust, you will need to stick to a specific patch version of Criterion.rs. diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 25bcd1f8f..b7268009b 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -20,7 +20,6 @@ - [Custom Test Framework](./user_guide/custom_test_framework.md) - [Benchmarking async functions](./user_guide/benchmarking_async.md) - [Quick Mode](./user_guide/quick_mode.md) - - [Tabulating Results](./user_guide/tabulating_results.md) - [WebAssembly/WASI](./user_guide/wasi.md) - [cargo-criterion](./cargo_criterion/cargo_criterion.md) - [Configuring cargo-criterion](./cargo_criterion/configuring_cargo_criterion.md) @@ -30,4 +29,4 @@ - [Comparison to Criterion.rs](./iai/comparison.md) - [Analysis Process](./analysis.md) - [Frequently Asked Questions](./faq.md) -- [Migrating from 0.2.* to 0.3.*](./migrating_0_2_to_0_3.md) \ No newline at end of file +- [Migrating from 0.2.* to 0.3.*](./migrating_0_2_to_0_3.md) diff --git a/book/src/user_guide/tabulating_results.md b/book/src/user_guide/tabulating_results.md deleted file mode 100644 index 2e86f0d0b..000000000 --- a/book/src/user_guide/tabulating_results.md +++ /dev/null @@ -1,294 +0,0 @@ - - - -# Tabulating Results - -Criterion can save the results of different benchmark runs and -tabulate the results, making it easier to spot performance changes. - -The set of results from a benchmark run is called a `baseline` and each -`baseline` has a name. By default, the most recent run is named `base` but this -can be changed with the `--save-baseline {name}` flag. There's also a special -baseline called `new` which refers to the most recent set of results. - -## Comparing profiles - -Cargo supports custom -[profiles](https://doc.rust-lang.org/cargo/reference/profiles.html) for -controlling the level of optimizations, debug assertions, overflow checks, and -link-time-optmizations. We can use criterion to benchmark different profiles and -tabulate the results to visualize the changes. Let's use the `base64` crate as -an example: - -```bash -> git clone https://github.com/KokaKiwi/rust-hex.git -> cd rust-hex/ -``` - -Now that we've clone the repository, we can generate the first set of benchmark results: - -```bash -> cargo bench --profile=release `# Use the 'release' profile` \ - --bench=hex `# Select the 'hex' binary` \ - -- `# Switch args from cargo to criterion` \ - --save-baseline release `# Save the baseline under 'release'` -``` - -Once the run is complete (this should take a few minutes), we can benchmark the other profile: - -```bash -> cargo bench --profile=dev `# Use the 'dev' profile` \ - --bench=benchmarks `# Select the 'hex' binary` \ - -- `# Switch args from cargo to criterion` \ - --save-baseline dev `# Save the baseline under 'dev'` -``` - -Finally we can compare the two benchmark runs (scroll to the right to see all columns): - -```bash -> cargo bench --bench=hex -- --compare --baselines=dev,release -``` - -
group                          dev                                               release
------                          ---                                               -------
-faster_hex_decode              239.50  847.6±16.54µs        ? ?/sec    1.00      3.5±0.01µs        ? ?/sec
-faster_hex_decode_fallback     52.58   567.7±8.36µs        ? ?/sec     1.00     10.8±0.04µs        ? ?/sec
-faster_hex_decode_unchecked    400.98   503.7±3.48µs        ? ?/sec    1.00   1256.2±1.57ns        ? ?/sec
-faster_hex_encode              259.95   244.5±2.04µs        ? ?/sec    1.00    940.5±4.64ns        ? ?/sec
-faster_hex_encode_fallback     50.60   565.1±3.41µs        ? ?/sec     1.00     11.2±0.02µs        ? ?/sec
-hex_decode                     25.27     3.0±0.01ms        ? ?/sec     1.00    119.3±0.17µs        ? ?/sec
-hex_encode                     23.99 1460.8±18.11µs        ? ?/sec     1.00     60.9±0.08µs        ? ?/sec
-rustc_hex_decode               28.79     3.1±0.02ms        ? ?/sec     1.00    107.4±0.40µs        ? ?/sec
-rustc_hex_encode               25.80  1385.4±4.37µs        ? ?/sec     1.00    53.7±15.63µs        ? ?/sec
-
- -The first column in the above results has the names of each individual -benchmark. The two other columns (`dev` and `release`) contain the actual -benchmark results. Each baseline column starts with a performance index relative -to the fastest run (eg. `faster_hex_decode` for `dev` has a performance index of -239.50 because it is 239.50 times slower than the `release` build). Next is the -mean execution time plus the standard deviation (eg. 847.6±16.54µs). Lastly -there's an optional throughput. If no throughput data is available, it will be -printed as `? ?/sec`. - -## Compact, list view. - -If horizontal space is limited or if you're comparing more than two baselines, -it can be convenient to arrange the results in a vertical list rather than in a -table. This can be enabled with the `--compare-list` flag: - -``` -faster_hex_decode ------------------ -release 1.00 3.5±0.01µs ? ?/sec -dev 239.50 847.6±16.54µs ? ?/sec - -faster_hex_decode_fallback --------------------------- -release 1.00 10.8±0.04µs ? ?/sec -dev 52.58 567.7±8.36µs ? ?/sec - -faster_hex_decode_unchecked ---------------------------- -release 1.00 1256.2±1.57ns ? ?/sec -dev 400.98 503.7±3.48µs ? ?/sec - -faster_hex_encode ------------------ -release 1.00 940.5±4.64ns ? ?/sec -dev 259.95 244.5±2.04µs ? ?/sec - -faster_hex_encode_fallback --------------------------- -release 1.00 11.2±0.02µs ? ?/sec -dev 50.60 565.1±3.41µs ? ?/sec - -hex_decode ----------- -release 1.00 119.3±0.17µs ? ?/sec -dev 25.27 3.0±0.01ms ? ?/sec - -hex_encode ----------- -release 1.00 60.9±0.08µs ? ?/sec -dev 23.99 1460.8±18.11µs ? ?/sec - -rustc_hex_decode ----------------- -release 1.00 107.4±0.40µs ? ?/sec -dev 28.79 3.1±0.02ms ? ?/sec - -rustc_hex_encode ----------------- -release 1.00 53.7±15.63µs ? ?/sec -dev 25.80 1385.4±4.37µs ? ?/sec -``` - -## Filtering results - -Some projects have dozens or even hundreds of benchmarks which can be -overwhelming if you're only interested in the performance of a single -feature/function. - -Let's clone the `hex` crate and change just a single function: - -```bash -> git clone https://github.com/KokaKiwi/rust-hex.git -> cd rust-hex/ -``` - -Save a baseline for the `main` branch: - -```bash -> cargo bench --bench=hex `# Select the 'hex' binary` \ - -- `# Switch args from cargo to criterion` \ - --save-baseline main `# Save the baseline under 'main'` -``` - -Create a new branch: - -```bash -> git checkout -b new-feature -``` - -For testing, let's modify the `hex_decode` benchmark to run twice: - -```diff ---- a/benches/hex.rs -+++ b/benches/hex.rs - c.bench_function("hex_decode", |b| { - let hex = hex::encode(DATA); -- b.iter(|| hex::decode(&hex).unwrap()) -+ b.iter(|| (hex::decode(&hex).unwrap(),hex::decode(&hex).unwrap())) - }); -``` - -Now we can benchmark just the `hex_decode` function: - -```bash -> cargo bench --bench=hex `# Select the 'hex' binary` \ - -- `# Switch args from cargo to criterion` \ - --save-baseline new-feature `# Save the baseline under 'new-feature'` \ - ^hex_decode `# Select the 'hex_decode' benchmark` -``` - -And compare it to the `main` branch, verifying that we've introduced a 2x -performance regression: - -```bash -> cargo bench --bench=hex -- --compare --baselines=main,new-feature ^hex_decode -``` - -
group                   main                                      new-feature
------                   ----                                      -----------
-hex_decode    1.00    119.1±1.30µs        ? ?/sec    2.06    245.5±2.21µs        ? ?/sec
-
- -## Thresholds - -If we don't know which benchmarks are of interest, we can filter the results -based on how much they've changed. - -In the previous section, we only generated results for the `hex_decode` -benchmark. For this run, we need a complete set of results: - -```bash -> cargo bench --bench=hex `# Select the 'hex' binary` \ - -- `# Switch args from cargo to criterion` \ - --save-baseline new-feature `# Save the baseline under 'new-feature'` \ -``` - -Now we can compare the results that differ by more than 10%: - -```bash -> cargo bench --bench=hex -- --compare --baselines=main,new-feature --compare-threshold=10 -``` - -
group                   main                                      new-feature
------                   ----                                      -----------
-hex_decode    1.00    119.1±1.30µs        ? ?/sec    2.02    240.0±1.05µs        ? ?/sec
-
- -The above console output shows that only a single benchmark changed by more than -10%. - -## Importing/Exporting JSON - -Baselines can be saved in JSON files for later use with the `--export` flag. Continuing with the `hex` crate example, here's how to -save the `release` and `dev` baselines as JSON: - -```bash -> cargo bench --bench=hex -- --export release > release.json -``` - -```bash -> cargo bench --bench=hex -- --export dev > dev.json -``` - -Baselines stored as JSON can be referenced directly when comparing results: - -```bash -> cargo bench --bench=hex -- --compare --baselines dev.json,release.json -``` - -
group                          dev                                               release
------                          ---                                               -------
-faster_hex_decode              239.50  847.6±16.54µs        ? ?/sec    1.00      3.5±0.01µs        ? ?/sec
-faster_hex_decode_fallback     52.58   567.7±8.36µs        ? ?/sec     1.00     10.8±0.04µs        ? ?/sec
-faster_hex_decode_unchecked    400.98   503.7±3.48µs        ? ?/sec    1.00   1256.2±1.57ns        ? ?/sec
-faster_hex_encode              259.95   244.5±2.04µs        ? ?/sec    1.00    940.5±4.64ns        ? ?/sec
-faster_hex_encode_fallback     50.60   565.1±3.41µs        ? ?/sec     1.00     11.2±0.02µs        ? ?/sec
-hex_decode                     25.27     3.0±0.01ms        ? ?/sec     1.00    119.3±0.17µs        ? ?/sec
-hex_encode                     23.99 1460.8±18.11µs        ? ?/sec     1.00     60.9±0.08µs        ? ?/sec
-rustc_hex_decode               28.79     3.1±0.02ms        ? ?/sec     1.00    107.4±0.40µs        ? ?/sec
-rustc_hex_encode               25.80  1385.4±4.37µs        ? ?/sec     1.00    53.7±15.63µs        ? ?/sec
-
- -Note that the JSON format is not stable across criterion versions. diff --git a/src/critcmp/app.rs b/src/critcmp/app.rs deleted file mode 100644 index e6b3e5b8c..000000000 --- a/src/critcmp/app.rs +++ /dev/null @@ -1,150 +0,0 @@ -use std::collections::BTreeSet; -use std::fs; -use std::io; -use std::path::{Path, PathBuf}; - -use regex::Regex; -use tabwriter::TabWriter; -use termcolor::{self, WriteColor}; - -use crate::critcmp::data::{BaseBenchmarks, Benchmarks}; -use crate::critcmp::main::Result; - -#[derive(Clone, Debug, Default)] -pub struct Args { - pub baselines: Vec, - pub output_list: bool, - pub threshold: Option, - pub color: bool, - pub filter: Option, -} - -impl Args { - pub fn benchmarks(&self) -> Result { - // First, load benchmark data from command line parameters. If a - // baseline name is given and is not a file path, then it is added to - // our whitelist of baselines. - let mut from_cli: Vec = vec![]; - let mut whitelist = BTreeSet::new(); - for arg in self.baselines.iter() { - let p = Path::new(arg); - if p.is_file() { - let baseb = BaseBenchmarks::from_path(p) - .map_err(|err| format!("{}: {}", p.display(), err))?; - whitelist.insert(baseb.name.clone()); - from_cli.push(baseb); - } else { - whitelist.insert(arg.clone()); - } - } - - let mut from_crit: Vec = vec![]; - match self.criterion_dir() { - Err(err) => { - // If we've loaded specific benchmarks from arguments, then it - // shouldn't matter whether we can find a Criterion directory. - // If we haven't loaded anything explicitly though, and if - // Criterion detection fails, then we won't have loaded - // anything and so we should return an error. - if from_cli.is_empty() { - return Err(err); - } - } - Ok(critdir) => { - let data = Benchmarks::gather(critdir)?; - from_crit.extend(data.by_baseline.into_iter().map(|(_, v)| v)); - } - } - if from_cli.is_empty() && from_crit.is_empty() { - fail!("could not find any benchmark data"); - } - - let mut data = Benchmarks::default(); - for basebench in from_crit.into_iter().chain(from_cli) { - if !whitelist.is_empty() && !whitelist.contains(&basebench.name) { - continue; - } - data.by_baseline.insert(basebench.name.clone(), basebench); - } - Ok(data) - } - - pub fn filter(&self) -> Option<&'_ Regex> { - self.filter.as_ref() - } - - pub fn group(&self) -> Result> { - // TODO - Ok(None) - // let pattern_os = match self.0.value_of_os("group") { - // None => return Ok(None), - // Some(pattern) => pattern, - // }; - // let pattern = cli::pattern_from_os(pattern_os)?; - // let re = Regex::new(pattern)?; - // if re.captures_len() <= 1 { - // fail!( - // "pattern '{}' has no capturing groups, by grouping \ - // benchmarks by a regex requires the use of at least \ - // one capturing group", - // pattern - // ); - // } - // Ok(Some(re)) - } - - pub fn threshold(&self) -> Result> { - Ok(self.threshold) - } - - pub fn list(&self) -> bool { - self.output_list - } - - pub fn criterion_dir(&self) -> Result { - let target_dir = self.target_dir()?; - let crit_dir = target_dir.join("criterion"); - if !crit_dir.exists() { - fail!( - "\ - no criterion data exists at {}\n\ - try running your benchmarks before tabulating results\ - ", - crit_dir.display() - ); - } - Ok(crit_dir) - } - - pub fn stdout(&self) -> Box { - if self.color { - Box::new(termcolor::Ansi::new(TabWriter::new(io::stdout()))) - } else { - Box::new(termcolor::NoColor::new(TabWriter::new(io::stdout()))) - } - } - - fn target_dir(&self) -> Result { - // FIXME: Use the same code as criterion - let mut cwd = fs::canonicalize(".") - .ok() - .unwrap_or_else(|| PathBuf::from(".")); - loop { - let candidate = cwd.join("target"); - if candidate.exists() { - return Ok(candidate); - } - cwd = match cwd.parent() { - Some(p) => p.to_path_buf(), - None => { - fail!( - "\ - could not find Criterion output directory\n\ - try using --target-dir or set CARGO_TARGET_DIR\ - " - ); - } - } - } - } -} diff --git a/src/critcmp/data.rs b/src/critcmp/data.rs deleted file mode 100644 index 4ea321906..000000000 --- a/src/critcmp/data.rs +++ /dev/null @@ -1,227 +0,0 @@ -use std::collections::BTreeMap; -use std::fs::File; -use std::io; -use std::path::Path; - -use serde::de::DeserializeOwned; -// use serde::{Deserialize, Serialize}; -use serde_json as json; -use walkdir::WalkDir; - -use crate::critcmp::main::Result; - -#[derive(Clone, Debug, Default)] -pub struct Benchmarks { - pub by_baseline: BTreeMap, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct BaseBenchmarks { - pub name: String, - pub benchmarks: BTreeMap, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Benchmark { - pub baseline: String, - pub fullname: String, - #[serde(rename = "criterion_benchmark_v1")] - pub info: CBenchmark, - #[serde(rename = "criterion_estimates_v1")] - pub estimates: CEstimates, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct CBenchmark { - pub group_id: String, - pub function_id: Option, - pub value_str: Option, - pub throughput: Option, - pub full_id: String, - pub directory_name: String, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "PascalCase")] -pub struct CThroughput { - pub bytes: Option, - pub elements: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct CEstimates { - pub mean: CStats, - pub median: CStats, - pub median_abs_dev: CStats, - pub slope: Option, - pub std_dev: CStats, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct CStats { - pub confidence_interval: CConfidenceInterval, - pub point_estimate: f64, - pub standard_error: f64, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct CConfidenceInterval { - pub confidence_level: f64, - pub lower_bound: f64, - pub upper_bound: f64, -} - -impl Benchmarks { - pub fn gather>(criterion_dir: P) -> Result { - let mut benchmarks = Benchmarks::default(); - for result in WalkDir::new(criterion_dir) { - let dent = result?; - let b = match Benchmark::from_path(dent.path())? { - None => continue, - Some(b) => b, - }; - benchmarks - .by_baseline - .entry(b.baseline.clone()) - .or_insert_with(|| BaseBenchmarks { - name: b.baseline.clone(), - benchmarks: BTreeMap::new(), - }) - .benchmarks - .insert(b.benchmark_name().to_string(), b); - } - Ok(benchmarks) - } -} - -impl Benchmark { - fn from_path>(path: P) -> Result> { - let path = path.as_ref(); - Benchmark::from_path_imp(path).map_err(|err| { - if let Some(parent) = path.parent() { - err!("{}: {}", parent.display(), err) - } else { - err!("unknown path: {}", err) - } - }) - } - - fn from_path_imp(path: &Path) -> Result> { - match path.file_name() { - None => return Ok(None), - Some(filename) => { - if filename != "estimates.json" { - return Ok(None); - } - } - } - // Criterion's directory structure looks like this: - // - // criterion/{group}/{name}/{baseline}/estimates.json - // - // In the same directory as `estimates.json`, there is also a - // `benchmark.json` which contains most of the info we need about - // a benchmark, including its name. From the path, we only extract the - // baseline name. - let parent = path - .parent() - .ok_or_else(|| err!("{}: could not find parent dir", path.display()))?; - let baseline = parent - .file_name() - .map(|p| p.to_string_lossy().into_owned()) - .ok_or_else(|| err!("{}: could not find baseline name", path.display()))?; - if baseline == "change" { - // This isn't really a baseline, but special state emitted by - // Criterion to reflect its own comparison between baselines. We - // don't use it. - return Ok(None); - } - - let info = CBenchmark::from_path(parent.join("benchmark.json"))?; - let estimates = CEstimates::from_path(path)?; - let fullname = format!("{}/{}", baseline, info.full_id); - Ok(Some(Benchmark { - baseline, - fullname, - info, - estimates, - })) - } - - pub fn nanoseconds(&self) -> f64 { - self.estimates.mean.point_estimate - } - - pub fn stddev(&self) -> f64 { - self.estimates.std_dev.point_estimate - } - - pub fn fullname(&self) -> &str { - &self.fullname - } - - pub fn baseline(&self) -> &str { - &self.baseline - } - - pub fn benchmark_name(&self) -> &str { - &self.info.full_id - } - - pub fn throughput(&self) -> Option { - const NANOS_PER_SECOND: f64 = 1_000_000_000.0; - - let scale = NANOS_PER_SECOND / self.nanoseconds(); - - self.info.throughput.as_ref().and_then(|t| { - let scaled_bytes = t.bytes.map(|num| Throughput::Bytes(num as f64 * scale)); - let scaled_elements = t - .elements - .map(|num| Throughput::Elements(num as f64 * scale)); - scaled_bytes.or(scaled_elements) - }) - } -} - -#[derive(Clone, Copy, Debug)] -pub enum Throughput { - Bytes(f64), - Elements(f64), -} - -impl BaseBenchmarks { - pub fn from_path>(path: P) -> Result { - deserialize_json_path(path.as_ref()) - } -} - -impl CBenchmark { - fn from_path>(path: P) -> Result { - deserialize_json_path(path.as_ref()) - } -} - -impl CEstimates { - fn from_path>(path: P) -> Result { - deserialize_json_path(path.as_ref()) - } -} - -fn deserialize_json_path(path: &Path) -> Result { - let file = File::open(path).map_err(|err| { - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - err!("{}: {}", name, err) - } else { - err!("{}: {}", path.display(), err) - } - })?; - let buf = io::BufReader::new(file); - let b = json::from_reader(buf).map_err(|err| { - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - err!("{}: {}", name, err) - } else { - err!("{}: {}", path.display(), err) - } - })?; - Ok(b) -} diff --git a/src/critcmp/main.rs b/src/critcmp/main.rs deleted file mode 100644 index 1fb447031..000000000 --- a/src/critcmp/main.rs +++ /dev/null @@ -1,131 +0,0 @@ -use std::collections::BTreeMap; -use std::error::Error; -use std::io::Write; -use std::process; -use std::result; - -use regex::Regex; - -use crate::critcmp::app::Args; -use crate::critcmp::data::{Benchmark, Benchmarks}; - -use crate::critcmp::output; - -macro_rules! err { - ($($tt:tt)*) => { Box::::from(format!($($tt)*)) } -} - -macro_rules! fail { - ($($tt:tt)*) => { return Err(err!($($tt)*)) } -} - -pub type Result = result::Result>; - -pub fn main(args: Args) { - if let Err(err) = try_main(args) { - eprintln!("{}", err); - process::exit(1); - } -} - -fn try_main(args: Args) -> Result<()> { - let benchmarks = args.benchmarks()?; - - let mut comps = match args.group()? { - None => group_by_baseline(&benchmarks, args.filter()), - Some(re) => group_by_regex(&benchmarks, &re, args.filter()), - }; - if let Some(threshold) = args.threshold()? { - comps.retain(|comp| comp.biggest_difference() > threshold); - } - if comps.is_empty() { - fail!("no benchmark comparisons to show"); - } - - let mut wtr = args.stdout(); - if args.list() { - output::rows(&mut wtr, &comps)?; - } else { - output::columns(&mut wtr, &comps)?; - } - wtr.flush()?; - Ok(()) -} - -fn group_by_baseline(benchmarks: &Benchmarks, filter: Option<&Regex>) -> Vec { - let mut byname: BTreeMap> = BTreeMap::new(); - for base_benchmarks in benchmarks.by_baseline.values() { - for (name, benchmark) in base_benchmarks.benchmarks.iter() { - if filter.map_or(false, |re| !re.is_match(name)) { - continue; - } - let output_benchmark = - output::Benchmark::from_data(benchmark).name(benchmark.baseline()); - byname - .entry(name.to_string()) - .or_insert_with(Vec::new) - .push(output_benchmark); - } - } - byname - .into_iter() - .map(|(name, benchmarks)| output::Comparison::new(&name, benchmarks)) - .collect() -} - -fn group_by_regex( - benchmarks: &Benchmarks, - group_by: &Regex, - filter: Option<&Regex>, -) -> Vec { - let mut byname: BTreeMap> = BTreeMap::new(); - for base_benchmarks in benchmarks.by_baseline.values() { - for (name, benchmark) in base_benchmarks.benchmarks.iter() { - if filter.map_or(false, |re| !re.is_match(name)) { - continue; - } - let (bench, cmp) = match benchmark_names(benchmark, group_by) { - None => continue, - Some((bench, cmp)) => (bench, cmp), - }; - let output_benchmark = output::Benchmark::from_data(benchmark).name(&bench); - byname - .entry(cmp) - .or_insert_with(Vec::new) - .push(output_benchmark); - } - } - byname - .into_iter() - .map(|(name, benchmarks)| output::Comparison::new(&name, benchmarks)) - .collect() -} - -fn benchmark_names(benchmark: &Benchmark, group_by: &Regex) -> Option<(String, String)> { - assert!(group_by.captures_len() > 1); - - let caps = match group_by.captures(benchmark.benchmark_name()) { - None => return None, - Some(caps) => caps, - }; - - let mut bench_name = benchmark.benchmark_name().to_string(); - let mut cmp_name = String::new(); - let mut offset = 0; - for option in caps.iter().skip(1) { - let m = match option { - None => continue, - Some(m) => m, - }; - cmp_name.push_str(m.as_str()); - // Strip everything that doesn't match capturing groups. The leftovers - // are our benchmark name. - bench_name.drain((m.start() - offset)..(m.end() - offset)); - offset += m.end() - m.start(); - } - // Add the baseline name to the benchmark to disambiguate it from - // benchmarks with the same name in other baselines. - bench_name.insert_str(0, &format!("{}/", benchmark.baseline())); - - Some((bench_name, cmp_name)) -} diff --git a/src/critcmp/mod.rs b/src/critcmp/mod.rs deleted file mode 100644 index 17f6ee447..000000000 --- a/src/critcmp/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[macro_use] -pub mod main; -pub mod app; -pub mod data; -pub mod output; diff --git a/src/critcmp/output.rs b/src/critcmp/output.rs deleted file mode 100644 index f6c68d91b..000000000 --- a/src/critcmp/output.rs +++ /dev/null @@ -1,234 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::iter; - -use termcolor::{Color, ColorSpec, WriteColor}; -use unicode_width::UnicodeWidthStr; - -use crate::critcmp::data; -use crate::critcmp::main::Result; - -#[derive(Clone, Debug)] -pub struct Comparison { - name: String, - benchmarks: Vec, - name_to_index: BTreeMap, -} - -#[derive(Clone, Debug)] -pub struct Benchmark { - name: String, - nanoseconds: f64, - stddev: Option, - throughput: Option, - /// Whether this is the best benchmark in a group. This is only populated - /// when a `Comparison` is built. - best: bool, - /// The rank of this benchmark in a group. The best is always `1.0`. This - /// is only populated when a `Comparison` is built. - rank: f64, -} - -impl Comparison { - pub fn new(name: &str, benchmarks: Vec) -> Comparison { - let mut comp = Comparison { - name: name.to_string(), - benchmarks, - name_to_index: BTreeMap::new(), - }; - if comp.benchmarks.is_empty() { - return comp; - } - - comp.benchmarks - .sort_by(|a, b| a.nanoseconds.partial_cmp(&b.nanoseconds).unwrap()); - comp.benchmarks[0].best = true; - - let top = comp.benchmarks[0].nanoseconds; - for (i, b) in comp.benchmarks.iter_mut().enumerate() { - comp.name_to_index.insert(b.name.to_string(), i); - b.rank = b.nanoseconds / top; - } - comp - } - - /// Return the biggest difference, percentage wise, between benchmarks - /// in this comparison. - /// - /// If this comparison has fewer than two benchmarks, then 0 is returned. - pub fn biggest_difference(&self) -> f64 { - if self.benchmarks.len() < 2 { - return 0.0; - } - let best = self.benchmarks[0].nanoseconds; - let worst = self.benchmarks.last().unwrap().nanoseconds; - ((worst - best) / best) * 100.0 - } - - fn get(&self, name: &str) -> Option<&Benchmark> { - self.name_to_index - .get(name) - .and_then(|&i| self.benchmarks.get(i)) - } -} - -impl Benchmark { - pub fn from_data(b: &data::Benchmark) -> Benchmark { - Benchmark { - name: b.fullname().to_string(), - nanoseconds: b.nanoseconds(), - stddev: Some(b.stddev()), - throughput: b.throughput(), - best: false, - rank: 0.0, - } - } - - pub fn name(self, name: &str) -> Benchmark { - Benchmark { - name: name.to_string(), - ..self - } - } -} - -pub fn columns(mut wtr: W, groups: &[Comparison]) -> Result<()> { - let mut columns = BTreeSet::new(); - for group in groups { - for b in &group.benchmarks { - columns.insert(b.name.to_string()); - } - } - - write!(wtr, "group")?; - for column in &columns { - write!(wtr, "\t {}", column)?; - } - writeln!(wtr)?; - - write_divider(&mut wtr, '-', "group".width())?; - for column in &columns { - write!(wtr, "\t ")?; - write_divider(&mut wtr, '-', column.width())?; - } - writeln!(wtr)?; - - for group in groups { - if group.benchmarks.is_empty() { - continue; - } - - write!(wtr, "{}", group.name)?; - for column_name in &columns { - let b = match group.get(column_name) { - Some(b) => b, - None => { - write!(wtr, "\t")?; - continue; - } - }; - - if b.best { - let mut spec = ColorSpec::new(); - spec.set_fg(Some(Color::Green)).set_bold(true); - wtr.set_color(&spec)?; - } - write!( - wtr, - "\t {:<5.2} {:>14} {:>14}", - b.rank, - time(b.nanoseconds, b.stddev), - throughput(b.throughput), - )?; - if b.best { - wtr.reset()?; - } - } - writeln!(wtr)?; - } - Ok(()) -} - -pub fn rows(mut wtr: W, groups: &[Comparison]) -> Result<()> { - for (i, group) in groups.iter().enumerate() { - if i > 0 { - writeln!(wtr)?; - } - rows_one(&mut wtr, group)?; - } - Ok(()) -} - -fn rows_one(mut wtr: W, group: &Comparison) -> Result<()> { - writeln!(wtr, "{}", group.name)?; - write_divider(&mut wtr, '-', group.name.width())?; - writeln!(wtr)?; - - if group.benchmarks.is_empty() { - writeln!(wtr, "NOTHING TO SHOW")?; - return Ok(()); - } - - for b in &group.benchmarks { - writeln!( - wtr, - "{}\t{:>7.2}\t{:>15}\t{:>12}", - b.name, - b.rank, - time(b.nanoseconds, b.stddev), - throughput(b.throughput), - )?; - } - Ok(()) -} - -fn write_divider(mut wtr: W, divider: char, width: usize) -> Result<()> { - let div: String = iter::repeat(divider).take(width).collect(); - write!(wtr, "{}", div)?; - Ok(()) -} - -fn time(nanos: f64, stddev: Option) -> String { - const MIN_MICRO: f64 = 2_000.0; - const MIN_MILLI: f64 = 2_000_000.0; - const MIN_SEC: f64 = 2_000_000_000.0; - - let (div, label) = if nanos < MIN_MICRO { - (1.0, "ns") - } else if nanos < MIN_MILLI { - (1_000.0, "µs") - } else if nanos < MIN_SEC { - (1_000_000.0, "ms") - } else { - (1_000_000_000.0, "s") - }; - if let Some(stddev) = stddev { - format!("{:.1}±{:.2}{}", nanos / div, stddev / div, label) - } else { - format!("{:.1}{}", nanos / div, label) - } -} - -fn throughput(throughput: Option) -> String { - use data::Throughput::*; - match throughput { - Some(Bytes(num)) => throughput_per(num, "B"), - Some(Elements(num)) => throughput_per(num, "Elem"), - _ => "? ?/sec".to_string(), - } -} - -fn throughput_per(per: f64, unit: &str) -> String { - const MIN_K: f64 = (2 * (1 << 10) as u64) as f64; - const MIN_M: f64 = (2 * (1 << 20) as u64) as f64; - const MIN_G: f64 = (2 * (1 << 30) as u64) as f64; - - if per < MIN_K { - format!("{} {}/sec", per as u64, unit) - } else if per < MIN_M { - format!("{:.1} K{}/sec", (per / (1 << 10) as f64), unit) - } else if per < MIN_G { - format!("{:.1} M{}/sec", (per / (1 << 20) as f64), unit) - } else { - format!("{:.1} G{}/sec", (per / (1 << 30) as f64), unit) - } -} diff --git a/src/lib.rs b/src/lib.rs index b1a128598..19ecac925 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,7 +59,6 @@ mod benchmark_group; pub mod async_executor; mod bencher; mod connection; -mod critcmp; #[cfg(feature = "csv_output")] mod csv_report; mod error; @@ -80,7 +79,6 @@ use std::cell::RefCell; use std::collections::HashSet; use std::default::Default; use std::env; -use std::io::Write; use std::net::TcpStream; use std::path::{Path, PathBuf}; use std::process::Command; @@ -786,31 +784,6 @@ impl Criterion { .takes_value(true) .help("Iterate each benchmark for approximately the given number of seconds, doing no analysis and without storing the results. Useful for running the benchmarks in a profiler.") .conflicts_with_all(&["test", "list"])) - .arg(Arg::new("export") - .long("export") - .takes_value(true) - .help("Export baseline as json, printed to stdout") - .conflicts_with_all(&["list", "test", "profile-time", "compare"])) - .arg(Arg::new("compare") - .long("compare") - .help("Tabulate benchmark results") - .conflicts_with_all(&["list", "test", "profile-time", "export"])) - .arg(Arg::new("baselines") - .long("baselines") - .multiple_occurrences(true) - .value_name("baselines") - .requires("compare") - .require_value_delimiter(true) - .use_value_delimiter(true) - .help("Limit the baselines used in tabulated results.") - .help("")) - .arg(Arg::new("compare-threshold") - .long("compare-threshold") - .takes_value(true) - .help("Hide results that differ by less than the threshold percentage. By default, all results are shown.")) - .arg(Arg::new("compare-list") - .long("compare-list") - .help("Show benchmark results in a list rather than in a table. Useful when horizontal space is limited.")) .arg(Arg::new("load-baseline") .long("load-baseline") .takes_value(true) @@ -1087,54 +1060,6 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.config.significance_level = num_significance_level; } - // XXX: Comparison functionality should ideally live in 'cargo-criterion'. - if matches.is_present("compare") { - if self.connection.is_some() { - eprintln!( - "Error: tabulating results is not supported when running with cargo-criterion." - ); - std::process::exit(1); - } - // Other arguments: compare-threshold, compare-list. - - let stdout_isatty = atty::is(atty::Stream::Stdout); - let enable_text_coloring = match matches.value_of("color") { - Some("always") => true, - Some("never") => false, - _ => stdout_isatty, - }; - - let args = critcmp::app::Args { - baselines: matches.values_of_lossy("baselines").unwrap_or_default(), - output_list: matches.is_present("compare-list"), - threshold: matches.value_of_t("compare-threshold").ok(), // FIXME: Print error message if parsing fails. - color: enable_text_coloring, - filter: self.filter, - }; - critcmp::main::main(args); - std::process::exit(0); - } - - if let Some(baseline) = matches.value_of("export") { - let benchmarks = critcmp::app::Args { - baselines: matches.values_of_lossy("baselines").unwrap_or_default(), - ..Default::default() - } - .benchmarks() - .expect("failed to find baselines"); - let mut stdout = std::io::stdout(); - let basedata = match benchmarks.by_baseline.get(baseline) { - Some(basedata) => basedata, - None => { - eprintln!("failed to find baseline '{}'", baseline); - std::process::exit(1); - } - }; - serde_json::to_writer_pretty(&mut stdout, basedata).unwrap(); - writeln!(stdout).unwrap(); - std::process::exit(0); - } - if matches.is_present("quick") { self.config.quick_mode = true; } From 935c6327e152e44f2b9178797682b9b99b5123a5 Mon Sep 17 00:00:00 2001 From: Brook Heisler Date: Sat, 10 Sep 2022 11:41:59 -0600 Subject: [PATCH 28/29] Add Throughput::BytesDecimal. Fixes #581. --- CHANGELOG.md | 11 ++++------ benches/benchmarks/custom_measurement.rs | 4 ++-- benches/benchmarks/with_inputs.rs | 9 ++++++++ src/lib.rs | 5 +++++ src/measurement.rs | 28 ++++++++++++++++++++++++ src/report.rs | 5 ++++- 6 files changed, 52 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 053bda9a5..4b76ba961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,9 +32,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - A `--discard-baseline` flag for discarding rather than saving benchmark results. - Formal support for benchmarking code compiled to web-assembly. - A `--quiet` flag for printing just a single line per benchmark. +- A `Throughput::BytesDecimal` option for measuring throughput in bytes but printing them using + decimal units like kilobytes instead of binary units like kibibytes. ### Fixed -- When using `bench_with_input`, the input parameter will now be passed through `black_box` before +- When using `bench_with_input`, the input parameter will now be passed through `black_box` before passing it to the benchmark. ## [0.3.6] - 2022-07-06 @@ -502,12 +504,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Initial release on Crates.io. -<<<<<<< HEAD -[unreleased]: https://github.com/bheisler/criterion.rs/compare/0.3.4...HEAD -======= - [Unreleased]: https://github.com/bheisler/criterion.rs/compare/0.3.6...HEAD ->>>>>>> master [0.1.1]: https://github.com/bheisler/criterion.rs/compare/0.1.0...0.1.1 [0.1.2]: https://github.com/bheisler/criterion.rs/compare/0.1.1...0.1.2 [0.2.0]: https://github.com/bheisler/criterion.rs/compare/0.1.2...0.2.0 @@ -528,4 +525,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [0.3.3]: https://github.com/bheisler/criterion.rs/compare/0.3.2...0.3.3 [0.3.4]: https://github.com/bheisler/criterion.rs/compare/0.3.3...0.3.4 [0.3.5]: https://github.com/bheisler/criterion.rs/compare/0.3.4...0.3.5 -[0.3.5]: https://github.com/bheisler/criterion.rs/compare/0.3.5...0.3.6 +[0.3.5]: https://github.com/bheisler/criterion.rs/compare/0.3.5...0.3.6 \ No newline at end of file diff --git a/benches/benchmarks/custom_measurement.rs b/benches/benchmarks/custom_measurement.rs index c685f382f..449f9030c 100644 --- a/benches/benchmarks/custom_measurement.rs +++ b/benches/benchmarks/custom_measurement.rs @@ -14,7 +14,7 @@ impl ValueFormatter for HalfSecFormatter { fn format_throughput(&self, throughput: &Throughput, value: f64) -> String { match *throughput { - Throughput::Bytes(bytes) => { + Throughput::Bytes(bytes) | Throughput::BytesDecimal(bytes) => { format!("{} b/s/2", (bytes as f64) / (value * 2f64 * 10f64.powi(-9))) } Throughput::Elements(elems) => format!( @@ -39,7 +39,7 @@ impl ValueFormatter for HalfSecFormatter { values: &mut [f64], ) -> &'static str { match *throughput { - Throughput::Bytes(bytes) => { + Throughput::Bytes(bytes) | Throughput::BytesDecimal(bytes) => { for val in values { *val = (bytes as f64) / (*val * 2f64 * 10f64.powi(-9)) } diff --git a/benches/benchmarks/with_inputs.rs b/benches/benchmarks/with_inputs.rs index 8eaaf0081..b0b12a89f 100644 --- a/benches/benchmarks/with_inputs.rs +++ b/benches/benchmarks/with_inputs.rs @@ -13,6 +13,15 @@ fn from_elem(c: &mut Criterion) { }); } group.finish(); + + let mut group = c.benchmark_group("from_elem_decimal"); + for size in [KB, 2 * KB].iter() { + group.throughput(Throughput::BytesDecimal(*size as u64)); + group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| { + b.iter(|| iter::repeat(0u8).take(size).collect::>()); + }); + } + group.finish(); } criterion_group!(benches, from_elem); diff --git a/src/lib.rs b/src/lib.rs index 19ecac925..16e79cc28 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1207,6 +1207,11 @@ pub enum Throughput { /// an input string or `&[u8]`. Bytes(u64), + /// Equivalent to Bytes, but the value will be reported in terms of + /// kilobytes (1000 bytes) per second instead of kibibytes (1024 bytes) per + /// second, megabytes instead of mibibytes, and gigabytes instead of gibibytes. + BytesDecimal(u64), + /// Measure throughput in terms of elements/second. The value should be the number of elements /// processed by one iteration of the benchmarked code. Typically, this would be the size of a /// collection, but could also be the number of lines of input text or the number of values to diff --git a/src/measurement.rs b/src/measurement.rs index e577dedd4..63719753d 100644 --- a/src/measurement.rs +++ b/src/measurement.rs @@ -124,6 +124,31 @@ impl DurationFormatter { unit } + fn bytes_per_second_decimal( + &self, + bytes: f64, + typical: f64, + values: &mut [f64], + ) -> &'static str { + let bytes_per_second = bytes * (1e9 / typical); + let (denominator, unit) = if bytes_per_second < 1000.0 { + (1.0, " B/s") + } else if bytes_per_second < 1000.0 * 1000.0 { + (1000.0, "KB/s") + } else if bytes_per_second < 1000.0 * 1000.0 * 1000.0 { + (1000.0 * 1000.0, "MB/s") + } else { + (1000.0 * 1000.0 * 1000.0, "GB/s") + }; + + for val in values { + let bytes_per_second = bytes * (1e9 / *val); + *val = bytes_per_second / denominator; + } + + unit + } + fn elements_per_second(&self, elems: f64, typical: f64, values: &mut [f64]) -> &'static str { let elems_per_second = elems * (1e9 / typical); let (denominator, unit) = if elems_per_second < 1000.0 { @@ -153,6 +178,9 @@ impl ValueFormatter for DurationFormatter { ) -> &'static str { match *throughput { Throughput::Bytes(bytes) => self.bytes_per_second(bytes as f64, typical, values), + Throughput::BytesDecimal(bytes) => { + self.bytes_per_second_decimal(bytes as f64, typical, values) + } Throughput::Elements(elems) => self.elements_per_second(elems as f64, typical, values), } } diff --git a/src/report.rs b/src/report.rs index 485ce4e90..9374c3e5b 100644 --- a/src/report.rs +++ b/src/report.rs @@ -173,7 +173,9 @@ impl BenchmarkId { pub fn as_number(&self) -> Option { match self.throughput { - Some(Throughput::Bytes(n)) | Some(Throughput::Elements(n)) => Some(n as f64), + Some(Throughput::Bytes(n)) + | Some(Throughput::Elements(n)) + | Some(Throughput::BytesDecimal(n)) => Some(n as f64), None => self .value_str .as_ref() @@ -184,6 +186,7 @@ impl BenchmarkId { pub fn value_type(&self) -> Option { match self.throughput { Some(Throughput::Bytes(_)) => Some(ValueType::Bytes), + Some(Throughput::BytesDecimal(_)) => Some(ValueType::Bytes), Some(Throughput::Elements(_)) => Some(ValueType::Elements), None => self .value_str From 4d6d69a9f56744f89d6143f928074dbd47d0512b Mon Sep 17 00:00:00 2001 From: Brook Heisler Date: Sat, 10 Sep 2022 11:46:34 -0600 Subject: [PATCH 29/29] Increment version numbers. --- CHANGELOG.md | 7 ++++--- Cargo.toml | 4 ++-- bencher_compat/Cargo.toml | 4 ++-- macro/Cargo.toml | 4 ++-- plot/Cargo.toml | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b76ba961..939bc4ea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] -## [0.4.0] - IN DEVELOPMENT +## [0.4.0] - 2022-09-10 ### Removed @@ -504,7 +504,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Initial release on Crates.io. -[Unreleased]: https://github.com/bheisler/criterion.rs/compare/0.3.6...HEAD +[Unreleased]: https://github.com/bheisler/criterion.rs/compare/0.4.0...HEAD [0.1.1]: https://github.com/bheisler/criterion.rs/compare/0.1.0...0.1.1 [0.1.2]: https://github.com/bheisler/criterion.rs/compare/0.1.1...0.1.2 [0.2.0]: https://github.com/bheisler/criterion.rs/compare/0.1.2...0.2.0 @@ -525,4 +525,5 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [0.3.3]: https://github.com/bheisler/criterion.rs/compare/0.3.2...0.3.3 [0.3.4]: https://github.com/bheisler/criterion.rs/compare/0.3.3...0.3.4 [0.3.5]: https://github.com/bheisler/criterion.rs/compare/0.3.4...0.3.5 -[0.3.5]: https://github.com/bheisler/criterion.rs/compare/0.3.5...0.3.6 \ No newline at end of file +[0.3.6]: https://github.com/bheisler/criterion.rs/compare/0.3.5...0.3.6 +[0.4.0]: https://github.com/bheisler/criterion.rs/compare/0.3.6...0.4.0 \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 6470b9623..c0adc3f99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ authors = [ "Brook Heisler ", ] name = "criterion" -version = "0.3.6" +version = "0.4.0" edition = "2018" description = "Statistics-driven micro-benchmarking library" @@ -19,7 +19,7 @@ exclude = ["book/*"] [dependencies] anes = "0.1.4" lazy_static = "1.4" -criterion-plot = { path = "plot", version = "0.4.5" } +criterion-plot = { path = "plot", version = "0.5.0" } itertools = "0.10" serde = "1.0" serde_json = "1.0" diff --git a/bencher_compat/Cargo.toml b/bencher_compat/Cargo.toml index c4823f3dc..c901ca72b 100644 --- a/bencher_compat/Cargo.toml +++ b/bencher_compat/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "criterion_bencher_compat" -version = "0.3.4" +version = "0.4.0" authors = ["Brook Heisler "] edition = "2018" @@ -13,7 +13,7 @@ categories = ["development-tools::profiling"] license = "Apache-2.0/MIT" [dependencies] -criterion = { version = "0.3.4", path = "..", default-features = false } +criterion = { version = "0.4.0", path = "..", default-features = false } [features] real_blackbox = ["criterion/real_blackbox"] diff --git a/macro/Cargo.toml b/macro/Cargo.toml index accd23878..b8adf2d44 100644 --- a/macro/Cargo.toml +++ b/macro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "criterion-macro" -version = "0.3.4" +version = "0.4.0" authors = ["Brook Heisler "] edition = "2018" @@ -20,7 +20,7 @@ proc-macro2 = { version = "1.0", features = ["nightly"] } quote = "1.0" [dev-dependencies] -criterion = { version = "0.3.4", path = "..", default-features = false } +criterion = { version = "0.4.0", path = "..", default-features = false } [[bench]] name = "test_macro_bench" diff --git a/plot/Cargo.toml b/plot/Cargo.toml index d7e78e0a5..a1ce36eb3 100644 --- a/plot/Cargo.toml +++ b/plot/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["Jorge Aparicio ", "Brook Heisler "] name = "criterion-plot" -version = "0.4.5" +version = "0.5.0" edition = "2018" description = "Criterion's plotting library"