allow jmh benchmarks to be cached for faster ci runs

ben-manes · Oct 21, 2024 · aa490e2 · aa490e2
1 parent 47a0101
commit aa490e2
Show file tree

Hide file tree

Showing 11 changed files with 113 additions and 29 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -46,15 +46,15 @@ jobs:
           java: ${{ matrix.java }}
           cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
           arguments: >
-            caffeine:jmh --no-daemon -q
+            caffeine:jmh
+            --no-configuration-cache
             -PincludePattern=ComputeBenchmark
             -PbenchmarkParameters=computeType=Caffeine,Guava,ConcurrentHashMap
       - name: Upload Compute JMH Results to Gist
         uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0
         if: >
           github.event_name == 'push'
           && github.event.repository.fork == false
-          && endsWith(github.ref, github.event.repository.default_branch)
         continue-on-error: true
         with:
           auth: ${{ secrets.GIST_TOKEN }}
@@ -68,15 +68,15 @@ jobs:
           java: ${{ matrix.java }}
           cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
           arguments: >
-            caffeine:jmh --no-daemon -q
+            caffeine:jmh
+            --no-configuration-cache
             -PincludePattern=GetPutBenchmark
             -PbenchmarkParameters=cacheType=Caffeine,Guava,ConcurrentHashMap
       - name: Upload Get/Put JMH Results to Gist
         uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0
         if: >
           github.event_name == 'push'
           && github.event.repository.fork == false
-          && endsWith(github.ref, github.event.repository.default_branch)
         continue-on-error: true
         with:
           auth: ${{ secrets.GIST_TOKEN }}
@@ -90,39 +90,116 @@ jobs:
           java: ${{ matrix.java }}
           cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
           arguments: >
-            caffeine:jmh --no-daemon -q
+            caffeine:jmh
+            --no-configuration-cache
             -PincludePattern=PutRemoveBenchmark
             -PbenchmarkParameters=cacheType=Caffeine,Guava,ConcurrentHashMap
+      - name: Upload Put/Remove JMH Results to Gist
+        uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0
+        if: >
+          github.event_name == 'push'
+          && github.event.repository.fork == false
+        continue-on-error: true
+        with:
+          auth: ${{ secrets.GIST_TOKEN }}
+          gist_description: Put/Remove JMH Results
+          gist_title: putremove_results_${{ matrix.java }}.json
+          github_file: ./caffeine/build/reports/jmh/results.json
+          gist_url: https://gist.githubusercontent.com/ben-manes/1359c399cb717e81976ee635c513340b
+      - name: EvictionBenchmark JMH Benchmark
+        uses: ./.github/actions/run-gradle
+        with:
+          java: ${{ matrix.java }}
+          cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
+          arguments: >
+            caffeine:jmh
+            --no-configuration-cache
+            -PincludePattern=EvictionBenchmark
+            -PbenchmarkParameters=cacheType=Caffeine,Guava
+      - name: Upload EvictionBenchmark JMH Results to Gist
+        uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0
+        if: >
+          github.event_name == 'push'
+          && github.event.repository.fork == false
+        continue-on-error: true
+        with:
+          auth: ${{ secrets.GIST_TOKEN }}
+          gist_description: Eviction JMH Results
+          gist_title: eviction_results_${{ matrix.java }}.json
+          github_file: ./caffeine/build/reports/jmh/results.json
+          gist_url: https://gist.githubusercontent.com/ben-manes/2f5b54540c2e67edc0d1a7f28a1139f5
       - name: FrequencySketch JMH Benchmark
         uses: ./.github/actions/run-gradle
         with:
           java: ${{ matrix.java }}
           cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
           arguments: >
-            caffeine:jmh --no-daemon -q
+            caffeine:jmh
+            --no-configuration-cache
             -PincludePattern=FrequencySketchBenchmark
-            -PbenchmarkParameters=tableSize=134217728
+      - name: Upload FrequencySketch JMH Results to Gist
+        uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0
+        if: >
+          github.event_name == 'push'
+          && github.event.repository.fork == false
+        continue-on-error: true
+        with:
+          auth: ${{ secrets.GIST_TOKEN }}
+          gist_description: FrequencySketch JMH Results
+          gist_title: freq_results_${{ matrix.java }}.json
+          github_file: ./caffeine/build/reports/jmh/results.json
+          gist_url: https://gist.githubusercontent.com/ben-manes/f93df1b66aa210bc23a1e672ef9bee16
       - name: TimerWheelBenchmark JMH Benchmark
         uses: ./.github/actions/run-gradle
         with:
           java: ${{ matrix.java }}
           cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
-          arguments: caffeine:jmh --no-daemon -q -PincludePattern=TimerWheelBenchmark
+          arguments: >
+            caffeine:jmh
+            --no-configuration-cache
+            -PincludePattern=TimerWheelBenchmark
+      - name: Upload TimerWheelBenchmark JMH Results to Gist
+        uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0
+        if: >
+          github.event_name == 'push'
+          && github.event.repository.fork == false
+        continue-on-error: true
+        with:
+          auth: ${{ secrets.GIST_TOKEN }}
+          gist_description: TimerWheel JMH Results
+          gist_title: timerwheel_results_${{ matrix.java }}.json
+          github_file: ./caffeine/build/reports/jmh/results.json
+          gist_url: https://gist.githubusercontent.com/ben-manes/9abafdbc76fe34c0e63cef386b0b9ac0
       - name: Publish JMH benchmarks
         if: >
           matrix.java == env.PUBLISH_JDK
           && github.event_name == 'push'
-          && endsWith(github.ref, github.event.repository.default_branch)
         run: |
           {
             echo '#### [Compute](https://jmh.morethan.io/?gists=511298014cc5629cbc5e57f09fd4c430)'
-            echo -n 'This benchmark that evaluates the overhead due to locking when the entry is'
-            echo -n 'present. The scenarios graphed are all threads retrieving a single entry'
-            echo -n '("sameKey") and threads retrieving different keys based on a Zipf distribution'
-            echo -n '("spread").'
+            echo -n 'This benchmark that evaluates the overhead due to locking when the entry is '
+            echo -n 'present. The scenarios graphed are all threads retrieving a single entry '
+            echo -n '("sameKey") and threads retrieving different keys based on a Zipf '
+            echo -n 'distribution ("spread").'
             echo -e '\n'
             echo '#### [Get/Put](https://jmh.morethan.io/?gists=b231cf57cf8e144e2247716e777edcf3)'
-            echo -n 'A benchmark that evaluates the read/write performance of a cache. The cache is'
-            echo -n 'pre-populated for a 100% hit rate and a Zipf distribution of keys is used to'
-            echo -n 'mimic application usage patterns.'
+            echo -n 'A benchmark that evaluates the read/write performance of a cache. The cache '
+            echo -n 'is pre-populated for a 100% hit rate and a Zipf distribution of keys is used '
+            echo -n 'to mimic application usage patterns.'
+            echo -e '\n'
+            echo '#### [Put/Remove](https://jmh.morethan.io/?gists=1359c399cb717e81976ee635c513340b)'
+            echo -n 'A benchmark that evaluates the insert/update/remove performance of a cache. '
+            echo -n 'A Zipf distribution of keys is used but it is unpredictable if the entry is '
+            echo -n 'present given the competing operations.'
+            echo -e '\n'
+            echo '#### [Eviction](https://jmh.morethan.io/?gists=2f5b54540c2e67edc0d1a7f28a1139f5)'
+            echo -n 'A benchmark that evaluates the evictions rate by always inserting a new entry.'
+            echo -e '\n'
+            echo '#### [FrequencySketch](https://jmh.morethan.io/?gists=f93df1b66aa210bc23a1e672ef9bee16)'
+            echo -n 'A benchmark that evaluates the performance of a frequency sketch for use by '
+            echo -n 'size eviction.'
+            echo -e '\n'
+            echo '#### [TimerWheel](https://jmh.morethan.io/?gists=9abafdbc76fe34c0e63cef386b0b9ac0)'
+            echo -n 'A benchmark that evaluates the performance of the timer wheel for use by '
+            echo -n 'variable expiration.'
           } >> $GITHUB_STEP_SUMMARY
diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/FactoryBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/FactoryBenchmark.java
@@ -30,7 +30,7 @@
  * This benchmark can be run by optionally specifying the target jvm in the command.
  * <p>
  * <pre>{@code
- *   JAVA_VERSION=20 ./gradlew jmh -PincludePattern=FactoryBenchmark --no-daemon
+ *   ./gradlew jmh -PincludePattern=FactoryBenchmark --rerun
  * }</pre>
  *
  * @author [email protected] (Ben Manes)

diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/BuilderBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/BuilderBenchmark.java
@@ -31,7 +31,7 @@
 /**
  * <p>
  * <pre>{@code
- * ./gradlew jmh -PincludePattern=BuilderBenchmark --no-daemon
+ *   ./gradlew jmh -PincludePattern=BuilderBenchmark --rerun
  * }</pre>
  *
  * @author [email protected] (Ben Manes)

diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/EvictionBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/EvictionBenchmark.java
@@ -28,7 +28,7 @@
  * a 100% eviction rate to mimic worst case behavior.
  * <p>
  * <pre>{@code
- *   ./gradlew jmh -PincludePattern=EvictionBenchmark
+ *   ./gradlew jmh -PincludePattern=EvictionBenchmark --rerun
  * }</pre>
  *
  * @author [email protected] (Ben Manes)

diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/FrequencySketchBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/FrequencySketchBenchmark.java
@@ -29,7 +29,7 @@
 
 /**
  * <pre>{@code
- *   ./gradlew jmh -PincludePattern=FrequencySketchBenchmark
+ *   ./gradlew jmh -PincludePattern=FrequencySketchBenchmark --rerun
  * }</pre>
  *
  * @author [email protected] (Ben Manes)

diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/GetPutBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/GetPutBenchmark.java
@@ -35,7 +35,8 @@
  * a 100% hit rate and a Zipf distribution of keys is used to mimic application usage patterns.
  * <p>
  * <pre>{@code
- *   ./gradlew jmh -PincludePattern=GetPutBenchmark
+ *   // JAVA_VERSION=?? for an alternative jdk
+ *   ./gradlew jmh -PincludePattern=GetPutBenchmark --rerun
  * }</pre>
  *
  * @author [email protected] (Ben Manes)

diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/PutRemoveBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/PutRemoveBenchmark.java
@@ -41,7 +41,7 @@
  * concerns and should not be used to compare implementations.
  * <p>
  * <pre>{@code
- *   ./gradlew jmh -PincludePattern=PutRemoveBenchmark
+ *   ./gradlew jmh -PincludePattern=PutRemoveBenchmark --rerun
  * }</pre>
  *
  * @author [email protected] (Ben Manes)

diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/TimerWheelBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/TimerWheelBenchmark.java
@@ -27,7 +27,7 @@
 
 /**
  * <pre>{@code
- *   ./gradlew jmh -PincludePattern=TimerWheelBenchmark
+ *   ./gradlew jmh -PincludePattern=TimerWheelBenchmark --rerun
  * }</pre>
  *
  * @author [email protected] (Ben Manes)

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -23,11 +23,11 @@ dependency-check = "10.0.4"
 eclipse-collections = "12.0.0.M3"
 ehcache3 = "3.10.8"
 errorprone = "2.34.0"
-errorprone-plugin = "4.0.1"
+errorprone-plugin = "4.1.0"
 errorprone-support = "0.18.0"
 expiring-map = "0.5.11"
 fast-filter = "1.0.2"
-fastutil = "8.5.14"
+fastutil = "8.5.15"
 felix-framework = "7.0.5"
 felix-scr = "2.2.12"
 findsecbugs = "1.13.0"
@@ -62,14 +62,14 @@ json-bind = "1.0"
 jsoup = "1.18.1"
 junit-testng = "1.0.5"
 junit4 = "4.13.2"
-junit5 = "5.11.2"
+junit5 = "5.11.3"
 jvm-dependency-conflict-resolution = "2.1.2"
 kotlin = "2.0.21"
 lincheck = "2.34"
 mockito = "5.14.2"
 nexus-publish = "2.0.0"
 nullaway = "0.12.0"
-nullaway-plugin = "2.0.0"
+nullaway-plugin = "2.1.0"
 okhttp-bom = "4.12.0"
 okio-bom = "3.9.1"
 osgi-annotations = "1.5.1"

diff --git a/gradle/plugins/src/main/kotlin/analyze/jmh.caffeine.gradle.kts b/gradle/plugins/src/main/kotlin/analyze/jmh.caffeine.gradle.kts
@@ -59,11 +59,16 @@ jmhReport {
   jmhReportOutput = file(layout.buildDirectory.file("reports/jmh")).toString()
 }
 
+// Use --rerun for repeated executions
 tasks.withType<JmhTask>().configureEach {
   group = "Benchmarks"
   description = "Executes a Java microbenchmark"
   incompatibleWithConfigurationCache()
-  outputs.upToDateWhen { false }
+
+  inputs.property("benchmarkParameters", jmh.benchmarkParameters)
+  inputs.property("includes", includes)
+  outputs.file(jmh.resultsFile)
+  outputs.cacheIf { true }
 
   doFirst {
     if (!project.hasProperty("includePattern")) {
@@ -81,6 +86,7 @@ tasks.withType<JmhBytecodeGeneratorTask>().configureEach {
 
 tasks.named("jmhJar").configure {
   incompatibleWithConfigurationCache()
+  outputs.cacheIf { true }
 }
 
 tasks.named("jmhReport").configure {

diff --git a/jcache/build.gradle.kts b/jcache/build.gradle.kts
@@ -102,7 +102,7 @@ tasks.named<Javadoc>("javadoc").configure {
 tasks.withType<Test>().configureEach {
   useJUnitPlatform()
   dependsOn(unzipTestKit)
-  testClassesDirs += layout.buildDirectory.files("tck")
+  testClassesDirs = files(testClassesDirs, layout.buildDirectory.files("tck"))
 
   project(":caffeine").plugins.withId("java-library") {
     val caffeineJar = project(":caffeine").tasks.named<Jar>("jar")