From aea3930bda6719ab3653518cb3f03c7af37ffc1a Mon Sep 17 00:00:00 2001 From: martinvuyk Date: Wed, 23 Oct 2024 12:27:19 -0300 Subject: [PATCH] add comment on benchmarking approach Signed-off-by: martinvuyk --- stdlib/benchmarks/collections/bench_string.mojo | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/stdlib/benchmarks/collections/bench_string.mojo b/stdlib/benchmarks/collections/bench_string.mojo index 3c9f01925c..240eada336 100644 --- a/stdlib/benchmarks/collections/bench_string.mojo +++ b/stdlib/benchmarks/collections/bench_string.mojo @@ -223,8 +223,17 @@ fn bench_string_is_valid_utf8[ # ===----------------------------------------------------------------------===# def main(): seed() - var m = Bench(BenchConfig(num_repetitions=1)) - m.bench_function[bench_string_init](BenchId("bench_string_init")) + var m = Bench(BenchConfig(num_repetitions=5)) + # NOTE: A proper way to run a benchmark like this is: + # 1. Run the benchmark on nightly branch with num_repetitions=5 and take the + # **median** value for each function, length, and language that is to be + # measured. + # 2. Then run the benchmark on num_repetitions=1 if you want faster results + # during development of your branch. + # 3. When ready to make statements about speed improvements, first run the + # benchmark again with num_repetitions=5 and take the **median** of that. + # 4. Make a table and report the new **median** numbers and the markdown + # percentage improvement over nightly version (new - nightly)/nightly. alias filenames = ( "UN_charter_EN", "UN_charter_ES", @@ -236,6 +245,8 @@ def main(): alias new_chars = ("A", "Ó", "ل", "И", "一") alias lengths = (10, 30, 50, 100, 1000, 10_000, 100_000, 1_000_000) + m.bench_function[bench_string_init](BenchId("bench_string_init")) + @parameter for i in range(len(lengths)): alias length = lengths.get[i, Int]()