Skip to content

Commit

Permalink
match cpu score with latest substrate using blake2b_simd
Browse files Browse the repository at this point in the history 
bump sc-sysinfo/sp-core/sp-core-hashing
- https://diff.rs/sp-core-hashing/8.0.0/7.0.0/Cargo.toml
- paritytech/substrate#13548
  • Loading branch information
navigaid committed Mar 27, 2023
1 parent 5d8bb39 commit 0f5352c
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 52 deletions.
23 changes: 2 additions & 21 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "substrate-benchmark-machine"
version = "0.0.1"
version = "0.0.2"
authors = ["Parity Technologies <admin@parity.io>"]
edition = "2021"
license = "Apache-2.0"
Expand Down Expand Up @@ -28,28 +28,9 @@ log = "0.4.17"
rand = { version = "0.8.4", features = ["small_rng"] }
sc-service = { version = "0", default-features = false }
sc-cli = { version = "0", default-features = false }
sc-sysinfo = { version = "18" }
sc-sysinfo = { version = "19" }
serde = "1.0.136"
serde_json = "1.0.85"
thiserror = "1.0.30"
anyhow = "1.0.70"

[profile.release]

# Substrate runtime requires unwinding.
panic = "unwind"

[profile.release.package]
blake2 = { opt-level = 3 }
blake2b_simd = { opt-level = 3 }

[profile.production]
inherits = "release"

# Sacrifice compile speed for execution speed by using optimization flags:

# https://doc.rust-lang.org/rustc/linker-plugin-lto.html
lto = "fat"
# https://doc.rust-lang.org/rustc/codegen-options/index.html#codegen-units
codegen-units = 1
opt-level = 3
22 changes: 8 additions & 14 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,21 @@ The `benchmark machine` command archives this by measuring key metrics and makin

Invoking the command looks like this:
```sh
cargo run --profile=production -- benchmark machine --dev
cargo run https://github.com/btwiuse/substrate-benchmark-machine
```

## Output

The output on reference hardware:

```pre
+----------+----------------+---------------+--------------+-------------------+
| Category | Function | Score | Minimum | Result |
+----------+----------------+---------------+--------------+-------------------+
| CPU | BLAKE2-256 | 1023.00 MiB/s | 1.00 GiB/s | ✅ Pass ( 99.4 %) |
+----------+----------------+---------------+--------------+-------------------+
| CPU | SR25519-Verify | 665.13 KiB/s | 666.00 KiB/s | ✅ Pass ( 99.9 %) |
+----------+----------------+---------------+--------------+-------------------+
| Memory | Copy | 14.39 GiB/s | 14.32 GiB/s | ✅ Pass (100.4 %) |
+----------+----------------+---------------+--------------+-------------------+
| Disk | Seq Write | 457.00 MiB/s | 450.00 MiB/s | ✅ Pass (101.6 %) |
+----------+----------------+---------------+--------------+-------------------+
| Disk | Rnd Write | 190.00 MiB/s | 200.00 MiB/s | ✅ Pass ( 95.0 %) |
+----------+----------------+---------------+--------------+-------------------+
[INFO] 🏁 CPU score: 1.42 GiBs (✅ Blake2256: expected minimum 1.00 GiBs) [INFO] 🏁 Memory score: 17.93 GiBs (✅ MemCopy: expected minimum 14.32 GiBs) [INFO] 🏁 Disk score (seq. writes): 397.35 MiBs (❌ DiskSeqWrite: expected minimum 450.00 MiBs) [INFO] 🏁 Disk score (rand. writes): 193.44 MiBs (❌ DiskRndWrite: expected minimum 200.00 MiBs) [INFO] ⚠ The hardware does not meet the minimal requirements for role 'Authority'. [INFO] Running machine benchmarks... [INFO] +----------+----------------+-------------+-------------+-------------------+ | Category | Function | Score | Minimum | Result | +===========================================================================+ | CPU | BLAKE2-256 | 1.42 GiBs | 1.00 GiBs | ✅ Pass (141.3 %) | |----------+----------------+-------------+-------------+-------------------| | CPU | SR25519-Verify | 726.21 KiBs | 666.00 KiBs | ✅ Pass (109.0 %) | |----------+----------------+-------------+-------------+-------------------|
| Memory | Copy | 18.10 GiBs | 14.32 GiBs | ✅ Pass (126.4 %) | |----------+----------------+-------------+-------------+-------------------| | Disk | Seq Write | 398.56 MiBs | 450.00 MiBs | ❌ Fail ( 88.6 %) | |----------+----------------+-------------+-------------+-------------------|
| Disk | Rnd Write | 193.37 MiBs | 200.00 MiBs | ✅ Pass ( 96.7 %) |
+----------+----------------+-------------+-------------+-------------------+
From 5 benchmarks in total, 4 passed and 1 failed (10% fault tolerance).
[INFO] The hardware fails to meet the requirements
Error: One of the benchmarks had a score that was lower than its requirement
```

The *score* is the average result of each benchmark. It always adheres to "higher is better".
Expand Down
112 changes: 104 additions & 8 deletions lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ use log::{error, info, warn};
use sc_cli::Result;
use sc_sysinfo::{
benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes,
benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, Metric, Requirement, Requirements,
Throughput,
benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, HwBench, Metric, Requirement,
Requirements, Throughput,
};

// use crate::shared::check_build_profile;
Expand All @@ -50,6 +50,10 @@ pub struct MachineCmd {
#[arg(long, short = 'd')]
pub base_path: Option<String>,

/// Run the full benchmark and show the result as a table.
#[arg(long, short = 'f', default_value_t = true)]
pub full: bool,

/// Do not return an error if any check fails.
///
/// Should only be used for debugging.
Expand Down Expand Up @@ -145,6 +149,18 @@ impl MachineCmd {
Ok(score)
}

pub fn print_full_table(&self, dir: &Path) -> Result<()> {
info!("Running machine benchmarks...");
let requirements = &SUBSTRATE_REFERENCE_HARDWARE.clone();
let mut results = Vec::new();
for requirement in &requirements.0 {
let result = self.run_benchmark(requirement, &dir)?;
results.push(result);
}
self.print_summary(requirements.clone(), results)?;
Ok(())
}

/// Prints a human-readable summary.
pub fn print_summary(
&self,
Expand Down Expand Up @@ -181,12 +197,6 @@ impl MachineCmd {
} else {
info!("The hardware meets the requirements ");
}
// Check that the results were not created by a bad build profile.
/*
if let Err(err) = check_build_profile() {
self.check_failed(Error::BadBuildProfile(err))?;
}
*/
Ok(())
}

Expand Down Expand Up @@ -224,3 +234,89 @@ impl BenchResult {
.into()
}
}

fn status_emoji(s: bool) -> String {
if s {
"✅".into()
} else {
"❌".into()
}
}

/// Whether the hardware requirements are met by the provided benchmark results.
pub fn check_hardware(hwbench: &HwBench) -> bool {
let req = &SUBSTRATE_REFERENCE_HARDWARE;

let mut cpu_ok = true;
let mut mem_ok = true;
let mut dsk_seq_write_ok = true;
let mut dsk_rnd_write_ok = true;

for requirement in req.0.iter() {
match requirement.metric {
Metric::Blake2256 => {
if requirement.minimum > hwbench.cpu_hashrate_score {
cpu_ok = false;
}
info!(
"🏁 CPU score: {} ({})",
hwbench.cpu_hashrate_score,
format!(
"{} Blake2256: expected minimum {}",
status_emoji(cpu_ok),
requirement.minimum
)
);
}
Metric::MemCopy => {
if requirement.minimum > hwbench.memory_memcpy_score {
mem_ok = false;
}
info!(
"🏁 Memory score: {} ({})",
hwbench.memory_memcpy_score,
format!(
"{} MemCopy: expected minimum {}",
status_emoji(mem_ok),
requirement.minimum
)
);
}
Metric::DiskSeqWrite => {
if let Some(score) = hwbench.disk_sequential_write_score {
if requirement.minimum > score {
dsk_seq_write_ok = false;
}
info!(
"🏁 Disk score (seq. writes): {} ({})",
score,
format!(
"{} DiskSeqWrite: expected minimum {}",
status_emoji(dsk_seq_write_ok),
requirement.minimum
)
);
}
}
Metric::DiskRndWrite => {
if let Some(score) = hwbench.disk_random_write_score {
if requirement.minimum > score {
dsk_rnd_write_ok = false;
}
info!(
"🏁 Disk score (rand. writes): {} ({})",
score,
format!(
"{} DiskRndWrite: expected minimum {}",
status_emoji(dsk_rnd_write_ok),
requirement.minimum
)
);
}
}
Metric::Sr25519Verify => {}
}
}

cpu_ok && mem_ok && dsk_seq_write_ok && dsk_rnd_write_ok
}
22 changes: 13 additions & 9 deletions main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
use anyhow::Result;
use clap::Parser;
use log::info;
use sc_sysinfo::gather_hwbench;
use std::fs;
use std::path::Path;
use substrate_benchmark_machine::{MachineCmd, SUBSTRATE_REFERENCE_HARDWARE};
use substrate_benchmark_machine::{check_hardware, MachineCmd};

fn main() -> Result<()> {
lg::info::init()?;
Expand All @@ -12,16 +14,18 @@ fn main() -> Result<()> {

let base_path = cmd.base_path.clone().unwrap_or(".".into());
fs::create_dir_all(&base_path)?;

// info!("Running machine benchmarks...");
let requirements = &SUBSTRATE_REFERENCE_HARDWARE.clone();
let mut results = Vec::new();
let dir = Path::new(&base_path);
for requirement in &requirements.0 {
let result = cmd.run_benchmark(requirement, &dir)?;
results.push(result);

let hwbench = gather_hwbench(Some(dir));
if !check_hardware(&hwbench) {
info!("⚠ The hardware does not meet the minimal requirements for role 'Authority'.");
} else {
info!("🎉 The hardware meets the minimal requirements for role 'Authority'.");
}

if cmd.full {
cmd.print_full_table(&dir)?;
}
cmd.print_summary(requirements.clone(), results)?;

Ok(())
}

0 comments on commit 0f5352c

Please sign in to comment.