Switch sr-arithmetic benchmarking to criterion (#3902)

* Change DefaultMaxDepth from 1024 to 32

* Switch sr-arithmetic benchmarking to criterion

* Update core/sr-arithmetic/Cargo.toml

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* Update core/sr-arithmetic/benches/bench.rs

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* Test on variable limb lengths

* Change license

* Rework division

substrate/Cargo.lock

@@ -3897,6 +3897,7 @@ dependencies = [
 name = "sr-arithmetic"
 version = "2.0.0"
 dependencies = [
+ "criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "integer-sqrt 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",

substrate/core/primitives/Cargo.toml

@@ -45,7 +45,7 @@ rand = "0.7.2"
 criterion = "0.2.11"
 
 [[bench]]
-name = "benches"
+name = "bench"
 harness = false
 
 [lib]

substrate/core/sr-arithmetic/Cargo.toml

@@ -15,9 +15,9 @@ substrate-debug-derive = { path = "../primitives/debug-derive", default-features
 [dev-dependencies]
 primitive-types = "0.6.0"
 rand = "0.7.2"
+criterion = "0.3"
 
 [features]
-bench = []
 default = ["std"]
 std = [
 	"codec/std",
@@ -26,3 +26,7 @@ std = [
 	"serde",
 	"substrate-debug-derive/std",
 ]
+
+[[bench]]
+name = "bench"
+harness = false

substrate/core/sr-arithmetic/benches/bench.rs

@@ -0,0 +1,80 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use criterion::{Criterion, Throughput, BenchmarkId, criterion_group, criterion_main};
+use sr_arithmetic::biguint::{BigUint, Single};
+use rand::Rng;
+
+fn random_big_uint(size: usize) -> BigUint {
+	let mut rng = rand::thread_rng();
+	let digits: Vec<_> = (0..size).map(|_| rng.gen_range(0, Single::max_value())).collect();
+	BigUint::from_limbs(&digits)
+}
+
+fn bench_op<F: Fn(&BigUint, &BigUint)>(c: &mut Criterion, name: &str, op: F) {
+	let mut group = c.benchmark_group(name);
+
+	for size in [2, 4, 6, 8, 10].iter() {
+		group.throughput(Throughput::Elements(*size));
+		group.bench_with_input(BenchmarkId::from_parameter(size), size, |bencher, &size| {
+			let a = random_big_uint(size as usize);
+			let b = random_big_uint(size as usize);
+
+			bencher.iter(|| op(&a, &b));
+		});
+	}
+}
+
+fn bench_addition(c: &mut Criterion) {
+	bench_op(c, "addition", |a, b| {
+		let _ = a.clone().add(&b);
+	});
+}
+
+fn bench_subtraction(c: &mut Criterion) {
+	bench_op(c, "subtraction", |a, b| {
+		let _ = a.clone().sub(&b);
+	});
+}
+
+fn bench_multiplication(c: &mut Criterion) {
+	bench_op(c, "multiplication", |a, b| {
+		let _ = a.clone().mul(&b);
+	});
+}
+
+fn bench_division(c: &mut Criterion) {
+	let mut group = c.benchmark_group("division");
+
+	for size in [4, 6, 8, 10].iter() {
+		group.throughput(Throughput::Elements(*size));
+		group.bench_with_input(BenchmarkId::from_parameter(size), size, |bencher, &size| {
+			let a = random_big_uint(size as usize);
+			let b = random_big_uint(rand::thread_rng().gen_range(2, size as usize));
+
+			bencher.iter(|| {
+				let _ = a.clone().div(&b, true);
+			});
+		});
+	}
+}
+
+criterion_group!{
+    name = benches;
+    config = Criterion::default();
+    targets = bench_addition, bench_subtraction, bench_multiplication, bench_division
+}
+criterion_main!(benches);

substrate/core/sr-arithmetic/src/biguint.rs

@@ -561,8 +561,6 @@ impl From<Double> for BigUint {
 #[cfg(test)]
 pub mod tests {
 	use super::*;
-	#[cfg(feature = "bench")]
-	use test::Bencher;
 
 	fn with_limbs(n: usize) -> BigUint {
 		BigUint { digits: vec![1; n] }
@@ -734,82 +732,4 @@ pub mod tests {
 		assert_eq!(b.clone().div_unit(7), BigUint::from(((B + 100) / 7) as Single));
 
 	}
-
-	#[cfg(feature = "bench")]
-	fn random_big_uint(size: usize) -> BigUint {
-		use rand::Rng;
-		let mut rng = rand::thread_rng();
-		let digits = (0..size).map(|_| rng.gen_range(0, Single::max_value())).collect();
-		BigUint { digits }
-	}
-
-	#[cfg(feature = "bench")]
-	#[bench]
-	fn bench_addition_2_digit(bencher: &mut Bencher) {
-		let a = random_big_uint(2);
-		let b = random_big_uint(2);
-		bencher.iter(|| {
-			let _ = a.clone().add(&b);
-		});
-	}
-
-	#[cfg(feature = "bench")]
-	#[bench]
-	fn bench_addition_4_digit(bencher: &mut Bencher) {
-		let a = random_big_uint(4);
-		let b = random_big_uint(4);
-		bencher.iter(|| {
-			let _ = a.clone().add(&b);
-		});
-	}
-
-	#[cfg(feature = "bench")]
-	#[bench]
-	fn bench_subtraction_2_digit(bencher: &mut Bencher) {
-		let a = random_big_uint(2);
-		let b = random_big_uint(2);
-		bencher.iter(|| {
-			let _ = a.clone().sub(&b);
-		});
-	}
-
-	#[cfg(feature = "bench")]
-	#[bench]
-	fn bench_subtraction_4_digit(bencher: &mut Bencher) {
-		let a = random_big_uint(4);
-		let b = random_big_uint(4);
-		bencher.iter(|| {
-			let _ = a.clone().sub(&b);
-		});
-	}
-
-	#[cfg(feature = "bench")]
-	#[bench]
-	fn bench_multiplication_2_digit(bencher: &mut Bencher) {
-		let a = random_big_uint(2);
-		let b = random_big_uint(2);
-		bencher.iter(|| {
-			let _ = a.clone().mul(&b);
-		});
-	}
-
-	#[cfg(feature = "bench")]
-	#[bench]
-	fn bench_multiplication_4_digit(bencher: &mut Bencher) {
-		let a = random_big_uint(4);
-		let b = random_big_uint(4);
-		bencher.iter(|| {
-			let _ = a.clone().mul(&b);
-		});
-	}
-
-	#[cfg(feature = "bench")]
-	#[bench]
-	fn bench_division_4_digit(bencher: &mut Bencher) {
-		let a = random_big_uint(4);
-		let b = random_big_uint(2);
-		bencher.iter(|| {
-			let _ = a.clone().div(&b, true);
-		});
-	}
 }

substrate/core/sr-arithmetic/src/lib.rs

@@ -18,10 +18,6 @@
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
-// to allow benchmarking
-#![cfg_attr(feature = "bench", feature(test))]
-#[cfg(feature = "bench")] extern crate test;
-
 /// Copied from `sr-primitives` and documented there.
 #[cfg(test)]
 macro_rules! assert_eq_error_rate {