Convert linfa-preprocessing Benchmark to Criterion #274

.gitignore

@@ -36,3 +36,6 @@ poetry.lock
 # Generated artifacts of website (with Zola)
 docs/website/public/*
 docs/website/static/rustdocs/
+
+# Downloaded data for the linfa-preprocessing benches
+20news/

algorithms/linfa-preprocessing/Cargo.toml

@@ -47,7 +47,9 @@ iai = "0.1"
 curl = "0.4.35"
 flate2 = "1.0.20"
 tar = "0.4.33"
-
+linfa = { version = "0.7.0", path = "../..", features = ["benchmarks"] }
+criterion = "0.4.0"
+statrs = "0.16.0"
 
 [[bench]]
 name = "vectorizer_bench"
@@ -60,3 +62,7 @@ harness = false
 [[bench]]
 name = "whitening_bench"
 harness = false
+
+[[bench]]
+name = "norm_scaler_bench"
+harness = false

algorithms/linfa-preprocessing/benches/linear_scaler_bench.rs

@@ -1,46 +1,46 @@
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use linfa::benchmarks::config;
 use linfa::traits::{Fit, Transformer};
-use linfa_preprocessing::linear_scaling::{LinearScaler, LinearScalerParams};
-use ndarray::Array2;
-use ndarray_rand::{
-    rand::distributions::Uniform, rand::rngs::SmallRng, rand::SeedableRng, RandomExt,
-};
+use linfa_datasets::generate::make_dataset;
+use linfa_preprocessing::linear_scaling::LinearScaler;
+use statrs::distribution::{DiscreteUniform, Laplace};
 
-fn iai_standard_scaler_bench() {
-    let mut rng = SmallRng::seed_from_u64(42);
-    for nfeatures in (10..100).step_by(10) {
-        fit_transform_scaler(LinearScaler::standard(), &mut rng, 10000, nfeatures);
-    }
-}
-
-fn iai_min_max_scaler_bench() {
-    let mut rng = SmallRng::seed_from_u64(42);
-    for nfeatures in (10..100).step_by(10) {
-        fit_transform_scaler(LinearScaler::min_max(), &mut rng, 10000, nfeatures);
-    }
-}
+fn bench(c: &mut Criterion) {
+    let mut benchmark = c.benchmark_group("liner scaler");
+    config::set_default_benchmark_configs(&mut benchmark);
+    let size = 10000;
+    let feat_distr = Laplace::new(0.5, 5.).unwrap();
+    let target_distr = DiscreteUniform::new(0, 5).unwrap();
 
-fn iai_max_abs_scaler_bench() {
-    let mut rng = SmallRng::seed_from_u64(42);
-    for nfeatures in (10..100).step_by(10) {
-        fit_transform_scaler(LinearScaler::max_abs(), &mut rng, 10000, nfeatures);
+    for (liner_scaler, fn_name) in [
+        (LinearScaler::standard(), "standard scaler"),
+        (LinearScaler::min_max(), "min max scaler"),
+        (LinearScaler::max_abs(), "max abs scaler"),
+    ] {
+        for nfeatures in (10..100).step_by(10) {
+            let dataset = make_dataset(size, nfeatures, 1, feat_distr, target_distr);
+            benchmark.bench_function(
+                BenchmarkId::new(fn_name, format!("{}x{}", nfeatures, size)),
+                |bencher| {
+                    bencher.iter(|| {
+                        liner_scaler
+                            .fit(black_box(&dataset))
+                            .unwrap()
+                            .transform(black_box(dataset.view()));
+                    });
+                },
+            );
+        }
     }
 }
 
-fn fit_transform_scaler(
-    scaler: LinearScalerParams<f64>,
-    rng: &mut SmallRng,
-    size: usize,
-    nfeatures: usize,
-) {
-    let dataset = Array2::random_using((size, nfeatures), Uniform::from(-30. ..30.), rng).into();
-    scaler
-        .fit(iai::black_box(&dataset))
-        .unwrap()
-        .transform(iai::black_box(dataset));
+#[cfg(not(target_os = "windows"))]
+criterion_group! {
+    name = benches;
+    config = config::get_default_profiling_configs();
+    targets = bench
 }
+#[cfg(target_os = "windows")]
+criterion_group!(benches, bench);
 
-iai::main!(
-    iai_standard_scaler_bench,
-    iai_min_max_scaler_bench,
-    iai_max_abs_scaler_bench
-);
+criterion_main!(benches);

algorithms/linfa-preprocessing/benches/norm_scaler_bench.rs

@@ -1,39 +1,43 @@
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use linfa::benchmarks::config;
 use linfa::traits::Transformer;
+use linfa_datasets::generate::make_dataset;
 use linfa_preprocessing::norm_scaling::NormScaler;
-use ndarray::Array2;
-use ndarray_rand::{
-    rand::distributions::Uniform, rand::rngs::SmallRng, rand::SeedableRng, RandomExt,
-};
+use statrs::distribution::{DiscreteUniform, Laplace};
 
-fn iai_l2_scaler_bench() {
-    let mut rng = SmallRng::seed_from_u64(84);
-    for nfeatures in (10..100).step_by(10) {
-        transform_scaler(NormScaler::l2(), &mut rng, 10000, nfeatures);
-    }
-}
-
-fn iai_l1_scaler_bench() {
-    let mut rng = SmallRng::seed_from_u64(84);
-    for nfeatures in (10..100).step_by(10) {
-        transform_scaler(NormScaler::l1(), &mut rng, 10000, nfeatures);
-    }
-}
+fn bench(c: &mut Criterion) {
+    let mut benchmark = c.benchmark_group("norm scaler");
+    config::set_default_benchmark_configs(&mut benchmark);
+    let size = 10000;
+    let feat_distr = Laplace::new(0.5, 5.).unwrap();
+    let target_distr = DiscreteUniform::new(0, 5).unwrap();
 
-fn iai_max_scaler_bench() {
-    let mut rng = SmallRng::seed_from_u64(84);
-    for nfeatures in (10..100).step_by(10) {
-        transform_scaler(NormScaler::max(), &mut rng, 10000, nfeatures);
+    for (scaler, fn_name) in [
+        (NormScaler::l2(), "l2 scaler"),
+        (NormScaler::l1(), "l1 scaler"),
+        (NormScaler::max(), "max scaler"),
+    ] {
+        for nfeatures in (10..100).step_by(10) {
+            let dataset = make_dataset(size, nfeatures, 1, feat_distr, target_distr);
+            benchmark.bench_function(
+                BenchmarkId::new(fn_name, format!("{}x{}", nfeatures, size)),
+                |bencher| {
+                    bencher.iter(|| {
+                        scaler.transform(black_box(dataset.view()));
+                    });
+                },
+            );
+        }
     }
 }
 
-fn transform_scaler(scaler: NormScaler, rng: &mut SmallRng, size: usize, nfeatures: usize) {
-    let dataset: Array2<f64> =
-        Array2::random_using((size, nfeatures), Uniform::from(-30. ..30.), rng);
-    scaler.transform(iai::black_box(dataset));
+#[cfg(not(target_os = "windows"))]
+criterion_group! {
+    name = benches;
+    config = config::get_default_profiling_configs();
+    targets = bench
 }
+#[cfg(target_os = "windows")]
+criterion_group!(benches, bench);
 
-iai::main!(
-    iai_l2_scaler_bench,
-    iai_l1_scaler_bench,
-    iai_max_scaler_bench
-);
+criterion_main!(benches);

algorithms/linfa-preprocessing/benches/vectorizer_bench.rs

@@ -5,6 +5,9 @@ use linfa_preprocessing::CountVectorizer;
 use std::path::Path;
 use tar::Archive;
 
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use linfa::benchmarks::config;
+
 fn download_20news_bydate() {
     let mut data = Vec::new();
     let mut easy = Easy::new();
@@ -77,95 +80,110 @@ fn load_test_set(desired_targets: &[&str]) -> Result<Vec<std::path::PathBuf>, st
     load_set("./20news/20news-bydate-test", desired_targets)
 }
 
-fn iai_fit_vectorizer() {
-    let file_names = load_20news_bydate();
+fn fit_vectorizer(file_names: &Vec<std::path::PathBuf>) {
     CountVectorizer::params()
         .document_frequency(0.05, 0.75)
         .n_gram_range(1, 2)
         .fit_files(
-            &file_names,
+            file_names,
             encoding::all::ISO_8859_1,
             encoding::DecoderTrap::Strict,
         )
         .unwrap();
 }
 
-fn iai_fit_tf_idf() {
-    let file_names = load_20news_bydate();
+fn fit_tf_idf(file_names: &Vec<std::path::PathBuf>) {
     TfIdfVectorizer::default()
         .document_frequency(0.05, 0.75)
         .n_gram_range(1, 2)
         .fit_files(
-            &file_names,
+            file_names,
             encoding::all::ISO_8859_1,
             encoding::DecoderTrap::Strict,
         )
         .unwrap();
 }
 
-fn iai_fit_transform_vectorizer() {
-    let file_names = load_20news_bydate();
+fn fit_transform_vectorizer(file_names: &Vec<std::path::PathBuf>) {
     CountVectorizer::params()
         .document_frequency(0.05, 0.75)
         .n_gram_range(1, 2)
         .fit_files(
-            &file_names,
+            file_names,
             encoding::all::ISO_8859_1,
             encoding::DecoderTrap::Strict,
         )
         .unwrap()
         .transform_files(
-            &file_names,
+            file_names,
             encoding::all::ISO_8859_1,
             encoding::DecoderTrap::Strict,
         );
 }
-fn iai_fit_transform_tf_idf() {
-    let file_names = load_20news_bydate();
+fn fit_transform_tf_idf(file_names: &Vec<std::path::PathBuf>) {
     TfIdfVectorizer::default()
         .document_frequency(0.05, 0.75)
         .n_gram_range(1, 2)
         .fit_files(
-            &file_names,
+            file_names,
             encoding::all::ISO_8859_1,
             encoding::DecoderTrap::Strict,
         )
         .unwrap()
         .transform_files(
-            &file_names,
+            file_names,
             encoding::all::ISO_8859_1,
             encoding::DecoderTrap::Strict,
         );
 }
 
-macro_rules! iai_main {
-    ( $( $func_name:ident ),+ $(,)* ) => {
-        mod iai_wrappers {
-            $(
-                pub fn $func_name() {
-                    let _ = iai::black_box(super::$func_name());
-                }
-            )+
-        }
+fn bench(c: &mut Criterion) {
+    let mut benchmark = c.benchmark_group("Linfa_preprocessing_vectorizer");
+    config::set_default_benchmark_configs(&mut benchmark);
 
-        fn main() {
-            load_20news_bydate();
-            let benchmarks : &[&(&'static str, fn())]= &[
+    let file_names = load_20news_bydate();
 
-                $(
-                    &(stringify!($func_name), iai_wrappers::$func_name),
-                )+
-            ];
+    benchmark.bench_function(
+        BenchmarkId::new("Fit-Vectorizer", "20news_bydate"),
+        |bencher| {
+            bencher.iter(|| {
+                fit_vectorizer(black_box(&file_names));
+            });
+        },
+    );
 
-            iai::runner(benchmarks);
-            std::fs::remove_dir_all("./20news").unwrap_or(());
-        }
-    }
+    benchmark.bench_function(BenchmarkId::new("Fit-Tf-Idf", "20news_bydate"), |bencher| {
+        bencher.iter(|| {
+            fit_tf_idf(black_box(&file_names));
+        });
+    });
+
+    benchmark.bench_function(
+        BenchmarkId::new("Fit-Transfor-Vectorizer", "20news_bydate"),
+        |bencher| {
+            bencher.iter(|| {
+                fit_transform_vectorizer(black_box(&file_names));
+            });
+        },
+    );
+
+    benchmark.bench_function(
+        BenchmarkId::new("Fit-Transfor-Tf-Idf", "20news_bydate"),
+        |bencher| {
+            bencher.iter(|| {
+                fit_transform_tf_idf(black_box(&file_names));
+            });
+        },
+    );
+}
+
+#[cfg(not(target_os = "windows"))]
+criterion_group! {
+    name = benches;
+    config = config::get_default_profiling_configs();
+    targets = bench
 }
+#[cfg(target_os = "windows")]
+criterion_group!(benches, bench);
 
-iai_main!(
-    iai_fit_vectorizer,
-    iai_fit_transform_vectorizer,
-    iai_fit_tf_idf,
-    iai_fit_transform_tf_idf
-);
+criterion_main!(benches);