diff --git a/build.gradle b/build.gradle index c12b9edc12..26989b697d 100644 --- a/build.gradle +++ b/build.gradle @@ -34,7 +34,6 @@ allprojects { repositories { mavenCentral() - maven { url 'https://jitpack.io' } } group = 'com.github.ben-manes.caffeine' diff --git a/checksum.xml b/checksum.xml index 8f2d824724..cbbe8b47a4 100644 --- a/checksum.xml +++ b/checksum.xml @@ -99,6 +99,7 @@ + diff --git a/gradle/codeQuality.gradle b/gradle/codeQuality.gradle index 40ee19531f..191ff380c3 100644 --- a/gradle/codeQuality.gradle +++ b/gradle/codeQuality.gradle @@ -107,7 +107,7 @@ tasks.withType(JavaExec) { tasks.withType(JavaCompile).configureEach { dependsOn downloadCaffeineLocal options.compilerArgs << [ - '-Xlint:all,-exports', '-auxiliaryclass', + '-Xlint:all,-processing,-exports', '-auxiliaryclass', '-requires-automatic', '-requires-transitive-automatic', ].join(',') options.encoding = 'UTF-8' diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index febbf38746..c194f51d58 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -40,7 +40,7 @@ ext { errorproneJavac: '9+181-r4173-1', elasticSearch: '7.13.2', expiringMap: '0.5.9', - fastfilter: '1.0', + fastfilter: '1.0.2', fastutil: '8.5.4', flipTables: '1.1.0', googleJavaFormat: '1.10.0', @@ -72,18 +72,18 @@ ext { jcacheTck: '1.1.1', jctools: '3.3.0', junit: '4.13.2', - mockito: '3.11.1', + mockito: '3.11.2', paxExam: '4.13.4', testng: '7.4.0', truth: '1.1.3', - felix: '7.0.0', + felix: '7.0.1', felixScr: '2.1.26', osgiUtilFunction: '1.1.0', osgiUtilPromise: '1.1.1', ] pluginVersions = [ bnd: '5.3.0', - checkstyle: '8.43', + checkstyle: '8.44', coveralls: '2.12.0', errorprone: '2.0.1', findsecbugs: '1.11.0', @@ -92,7 +92,7 @@ ext { jmhReport: '0.9.0', nexusPublish: '1.1.0', nullaway: '1.1.0', - pmd: '6.35.0', + pmd: '6.36.0', semanticVersioning: '1.1.0', shadow: '7.0.0', sonarqube: '3.3', @@ -126,7 +126,7 @@ ext { exclude group: 'org.apache.lucene' }, expiringMap: "net.jodah:expiringmap:${versions.expiringMap}", - fastfilter: "com.github.FastFilter:fastfilter_java:${versions.fastfilter}", + fastfilter: "io.github.fastfilter:fastfilter:${versions.fastfilter}", fastutil: "it.unimi.dsi:fastutil-core:${versions.fastutil}", flipTables: "com.jakewharton.fliptables:fliptables:${versions.flipTables}", googleJavaFormat: "com.google.googlejavaformat:google-java-format:${versions.googleJavaFormat}", diff --git a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/BinaryTraceWriter.java b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/BinaryTraceWriter.java new file mode 100644 index 0000000000..c98ca5a2b2 --- /dev/null +++ b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/BinaryTraceWriter.java @@ -0,0 +1,42 @@ +/* + * Copyright 2021 Ben Manes. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.benmanes.caffeine.cache.simulator.parser; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * A skeletal implementation that writes to the trace file as binary data. + * + * @author ben.manes@gmail.com (Ben Manes) + */ +public abstract class BinaryTraceWriter implements TraceWriter { + private final DataOutputStream writer; + + protected BinaryTraceWriter(OutputStream output) { + this.writer = new DataOutputStream(output); + } + + protected DataOutputStream writer() { + return writer; + } + + @Override + public void close() throws IOException { + writer.close(); + } +} diff --git a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/OutputFormat.java b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/OutputFormat.java index 14553e551c..4b18672a75 100644 --- a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/OutputFormat.java +++ b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/OutputFormat.java @@ -20,7 +20,8 @@ import java.util.function.Function; import com.github.benmanes.caffeine.cache.simulator.parser.adapt_size.AdaptSizeTraceWriter; -import com.github.benmanes.caffeine.cache.simulator.parser.climb.ClimbTraceWriter; +import com.github.benmanes.caffeine.cache.simulator.parser.cloud_physics.CloudPhysicsTraceWriter; +import com.github.benmanes.caffeine.cache.simulator.parser.lirs.LirsTraceWriter; /** * The trace output format. @@ -30,7 +31,8 @@ @SuppressWarnings("ImmutableEnumChecker") public enum OutputFormat { ADAPT_SIZE(AdaptSizeTraceWriter::new), - CLIMB(ClimbTraceWriter::new); + CLOUD_PHYSICS(CloudPhysicsTraceWriter::new), + LIRS(LirsTraceWriter::new); private final Function factory; diff --git a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/Rewriter.java b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/Rewriter.java index 0723b19c01..c635bd3f5d 100644 --- a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/Rewriter.java +++ b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/Rewriter.java @@ -17,15 +17,11 @@ import java.io.BufferedOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; -import java.util.stream.Stream; -import com.github.benmanes.caffeine.cache.simulator.policy.AccessEvent; import com.google.common.base.Stopwatch; import picocli.CommandLine; @@ -38,7 +34,7 @@ * into Java. *

* 

{@code
- *   ./gradlew :simulator:rewrite \
+ *   ./gradlew :simulator:rewrite -q \
  *      -PinputFormat=? \
  *      -PinputFiles=? \
  *      -PoutputFile=? \
@@ -47,11 +43,10 @@
  *
  * @author ben.manes@gmail.com (Ben Manes)
  */
-@SuppressWarnings({"PMD.ImmutableField", "FieldCanBeFinal"})
 public final class Rewriter implements Runnable {
-  @Option(names = "--inputFiles", required = true, description = "The trace input files. To use "
-      + "a mix of formats, specify the entry as format:path, e.g. lirs:loop.trace.gz")
-  private List inputFiles = new ArrayList<>();
+  @Option(names = "--inputFiles", required = true, split = ",", description = "The trace input "
+      + "files. To use a mix of formats, specify the entry as format:path, e.g. lirs:loop.trace.gz")
+  private List inputFiles;
   @Option(names = "--inputFormat", required = true, description = "The default trace input format")
   private TraceFormat inputFormat;
 
@@ -62,10 +57,10 @@ public final class Rewriter implements Runnable {
 
   @Override
   public void run() {
-    Stopwatch stopwatch = Stopwatch.createStarted();
-    try (OutputStream output = new BufferedOutputStream(Files.newOutputStream(outputFile));
-         Stream events = inputFormat.readFiles(inputFiles).events();
-         TraceWriter writer = outputFormat.writer(output)) {
+    var stopwatch = Stopwatch.createStarted();
+    try (var output = new BufferedOutputStream(Files.newOutputStream(outputFile));
+         var events = inputFormat.readFiles(inputFiles).events();
+         var writer = outputFormat.writer(output)) {
       int[] tick = { 0 };
       writer.writeHeader();
       events.forEach(event -> {
@@ -77,7 +72,8 @@ public void run() {
         }
       });
       writer.writeFooter();
-      System.out.printf("Rewrote %,d events in %s%n", tick[0], stopwatch);
+      System.out.printf("Rewrote %,d events from %,d inputs in %s%n",
+          tick[0], inputFiles.size(), stopwatch);
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }
diff --git a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/TraceFormat.java b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/TraceFormat.java
index d6c6030196..03edfadb7b 100644
--- a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/TraceFormat.java
+++ b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/TraceFormat.java
@@ -29,7 +29,7 @@
 import com.github.benmanes.caffeine.cache.simulator.parser.arc.ArcTraceReader;
 import com.github.benmanes.caffeine.cache.simulator.parser.cache2k.Cache2kTraceReader;
 import com.github.benmanes.caffeine.cache.simulator.parser.camelab.CamelabTraceReader;
-import com.github.benmanes.caffeine.cache.simulator.parser.climb.ClimbTraceReader;
+import com.github.benmanes.caffeine.cache.simulator.parser.cloud_physics.CloudPhysicsTraceReader;
 import com.github.benmanes.caffeine.cache.simulator.parser.corda.CordaTraceReader;
 import com.github.benmanes.caffeine.cache.simulator.parser.gradle.GradleTraceReader;
 import com.github.benmanes.caffeine.cache.simulator.parser.kaggle.OutbrainTraceReader;
@@ -65,7 +65,7 @@ public enum TraceFormat {
   ARC(ArcTraceReader::new),
   CACHE2K(Cache2kTraceReader::new),
   CAMELAB(CamelabTraceReader::new),
-  CLIMB(ClimbTraceReader::new),
+  CLOUD_PHYSICS(CloudPhysicsTraceReader::new),
   CORDA(CordaTraceReader::new),
   GRADLE(GradleTraceReader::new),
   LIRS(LirsTraceReader::new),
diff --git a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/climb/ClimbTraceReader.java b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/climb/ClimbTraceReader.java
deleted file mode 100644
index 57bafb5296..0000000000
--- a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/climb/ClimbTraceReader.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2019 Ben Manes. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.github.benmanes.caffeine.cache.simulator.parser.climb;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.InputStream;
-import java.util.PrimitiveIterator;
-import java.util.Scanner;
-import java.util.Spliterator;
-import java.util.Spliterators;
-import java.util.stream.LongStream;
-import java.util.stream.StreamSupport;
-
-import com.github.benmanes.caffeine.cache.simulator.parser.TextTraceReader;
-import com.github.benmanes.caffeine.cache.simulator.parser.TraceReader.KeyOnlyTraceReader;
-
-/**
- * A reader for the trace files provided by the authors of the AdaptiveClimb algorithm.
- *
- * @author ben.manes@gmail.com (Ben Manes)
- */
-public final class ClimbTraceReader extends TextTraceReader implements KeyOnlyTraceReader {
-
-  public ClimbTraceReader(String filePath) {
-    super(filePath);
-  }
-
-  @Override
-  public LongStream keys() {
-    TraceIterator iterator = new TraceIterator(readFile());
-    return StreamSupport.longStream(Spliterators.spliteratorUnknownSize(
-        iterator, Spliterator.ORDERED), /* parallel */ false).onClose(iterator::close);
-  }
-
-  private static final class TraceIterator implements PrimitiveIterator.OfLong {
-    private final Scanner scanner;
-
-    TraceIterator(InputStream input) {
-      scanner = new Scanner(input, UTF_8.name());
-    }
-
-    @Override
-    public boolean hasNext() {
-      return scanner.hasNextLong();
-    }
-
-    @Override
-    public long nextLong() {
-      return scanner.nextLong();
-    }
-
-    public void close() {
-      scanner.close();
-    }
-  }
-}
diff --git a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/cloud_physics/CloudPhysicsTraceReader.java b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/cloud_physics/CloudPhysicsTraceReader.java
new file mode 100644
index 0000000000..f0aac3a80e
--- /dev/null
+++ b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/cloud_physics/CloudPhysicsTraceReader.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 Ben Manes. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.benmanes.caffeine.cache.simulator.parser.cloud_physics;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Set;
+
+import com.github.benmanes.caffeine.cache.simulator.parser.BinaryTraceReader;
+import com.github.benmanes.caffeine.cache.simulator.policy.AccessEvent;
+import com.github.benmanes.caffeine.cache.simulator.policy.Policy.Characteristic;
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * A reader for the trace files provided by the author of LIRS2. See
+ * traces.
+ *
+ * @author ben.manes@gmail.com (Ben Manes)
+ */
+public final class CloudPhysicsTraceReader extends BinaryTraceReader {
+
+  public CloudPhysicsTraceReader(String filePath) {
+    super(filePath);
+  }
+
+  @Override
+  public Set characteristics() {
+    return ImmutableSet.of();
+  }
+
+  @Override
+  protected AccessEvent readEvent(DataInputStream input) throws IOException {
+    return AccessEvent.forKey(Integer.toUnsignedLong(input.readInt()));
+  }
+}
diff --git a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/cloud_physics/CloudPhysicsTraceWriter.java b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/cloud_physics/CloudPhysicsTraceWriter.java
new file mode 100644
index 0000000000..d074f4f825
--- /dev/null
+++ b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/cloud_physics/CloudPhysicsTraceWriter.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Ben Manes. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.benmanes.caffeine.cache.simulator.parser.cloud_physics;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import com.github.benmanes.caffeine.cache.simulator.parser.BinaryTraceWriter;
+import com.github.benmanes.caffeine.cache.simulator.policy.AccessEvent;
+import com.google.common.hash.Hashing;
+
+/**
+ * A writer for the trace format used by the authors of the LIRS2 algorithm.
+ *
+ * @author ben.manes@gmail.com (Ben Manes)
+ */
+public final class CloudPhysicsTraceWriter extends BinaryTraceWriter {
+
+  public CloudPhysicsTraceWriter(OutputStream output) {
+    super(output);
+  }
+
+  @Override
+  public void writeEvent(int tick, AccessEvent event) throws IOException {
+    int key = Hashing.murmur3_128().hashLong(event.key()).asInt();
+    writer().writeInt(key);
+  }
+}
diff --git a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/climb/ClimbTraceWriter.java b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/lirs/LirsTraceWriter.java
similarity index 80%
rename from simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/climb/ClimbTraceWriter.java
rename to simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/lirs/LirsTraceWriter.java
index 33bcd57b69..059f7e2196 100644
--- a/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/climb/ClimbTraceWriter.java
+++ b/simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/lirs/LirsTraceWriter.java
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.github.benmanes.caffeine.cache.simulator.parser.climb;
+package com.github.benmanes.caffeine.cache.simulator.parser.lirs;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -22,13 +22,13 @@
 import com.github.benmanes.caffeine.cache.simulator.policy.AccessEvent;
 
 /**
- * A writer for the trace format used by the authors of the AdaptiveClimb simulator.
+ * A writer for the trace format used by the authors of the LIRS algorithm.
  *
  * @author ben.manes@gmail.com (Ben Manes)
  */
-public final class ClimbTraceWriter extends TextTraceWriter {
+public final class LirsTraceWriter extends TextTraceWriter {
 
-  public ClimbTraceWriter(OutputStream output) {
+  public LirsTraceWriter(OutputStream output) {
     super(output);
   }
 
diff --git a/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/cloud_physics/w106.trace.xz b/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/cloud_physics/w106.trace.xz
new file mode 100755
index 0000000000..80ed76b183
Binary files /dev/null and b/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/cloud_physics/w106.trace.xz differ
diff --git a/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs/backf.trace.gz b/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs/backf.trace.gz
new file mode 100755
index 0000000000..9552814dc8
Binary files /dev/null and b/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs/backf.trace.gz differ
diff --git a/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs/scan.trace.gz b/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs/scan.trace.gz
new file mode 100644
index 0000000000..2b29389cde
Binary files /dev/null and b/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs/scan.trace.gz differ
diff --git a/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs/zigzag.trace.gz b/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs/zigzag.trace.gz
new file mode 100644
index 0000000000..6d5f2a3d72
Binary files /dev/null and b/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs/zigzag.trace.gz differ
diff --git a/simulator/src/main/resources/reference.conf b/simulator/src/main/resources/reference.conf
index 283aeee48c..dd594ef354 100644
--- a/simulator/src/main/resources/reference.conf
+++ b/simulator/src/main/resources/reference.conf
@@ -478,24 +478,24 @@ caffeine.simulator {
     # address: format of UCSD program address traces
     # address-penalties: format of UCSD program address traces with hit & miss penalties
     # cache2k: format from the author of the Cache2k library
-    # camelab: format from the Camelab storage traces
-    # climb: format from the authors of the AdaptiveClimb algorithm
+    # camelab: format of the Camelab storage traces
+    # cloud-physics: format of the Cloud Physics traces
     # corda: format of Corda traces
     # gradle: format from the authors of the Gradle build tool
     # lirs: format from the authors of the LIRS algorithm
     # lrb: format from the authors of the LRB algorithm
-    # outbrain: format from Outbrain trace provided on Kaggle
+    # outbrain: format of Outbrain's trace provided on Kaggle
     # scarab: format of Scarab Research traces
-    # snia-cambridge: format from the SNIA MSR Cambridge traces
-    # snia-k5cloud: format from the SNIA K5cloud traces
-    # snia-object-store: format from the SNIA IBM ObjectStore traces
-    # snia-systor: format from the SNIA SYSTOR '17 traces
-    # snia-tencent-block: format from the SNIA Tencent Block traces
-    # snia-tencent-photo: format from the SNIA Tencent Photo traces
-    # twitter: format from the Twitter Cache Cluster traces
-    # umass-storage: format from the University of Massachusetts storage traces
-    # umass-youtube: format from the University of Massachusetts youtube traces
-    # wikipedia: format from the WikiBench request traces
+    # snia-cambridge: format of the SNIA MSR Cambridge traces
+    # snia-k5cloud: format of the SNIA K5cloud traces
+    # snia-object-store: format of the SNIA IBM ObjectStore traces
+    # snia-systor: format of the SNIA SYSTOR '17 traces
+    # snia-tencent-block: format of the SNIA Tencent Block traces
+    # snia-tencent-photo: format of the SNIA Tencent Photo traces
+    # twitter: format of the Twitter Cache Cluster traces
+    # umass-storage: format of the University of Massachusetts storage traces
+    # umass-youtube: format of the University of Massachusetts youtube traces
+    # wikipedia: format of the WikiBench request traces
     format = lirs
   }