diff --git a/src/main/java/net/fabricmc/loom/decompilers/cache/CachedJarProcessor.java b/src/main/java/net/fabricmc/loom/decompilers/cache/CachedJarProcessor.java index e89d035ed..f78ff8b15 100644 --- a/src/main/java/net/fabricmc/loom/decompilers/cache/CachedJarProcessor.java +++ b/src/main/java/net/fabricmc/loom/decompilers/cache/CachedJarProcessor.java @@ -49,11 +49,13 @@ public WorkRequest prepareJob(Path inputJar) throws IOException { boolean hasSomeExisting = false; Path incompleteJar = Files.createTempFile("loom-cache-incomplete", ".jar"); - Path existingJar = Files.createTempFile("loom-cache-existing", ".jar"); + Path existingClassesJar = Files.createTempFile("loom-cache-existingClasses", ".jar"); + Path existingSourcesJar = Files.createTempFile("loom-cache-existingSources", ".jar"); // We must delete the empty files, so they can be created as a zip Files.delete(incompleteJar); - Files.delete(existingJar); + Files.delete(existingClassesJar); + Files.delete(existingSourcesJar); // Sources name -> hash Map outputNameMap = new HashMap<>(); @@ -64,12 +66,14 @@ public WorkRequest prepareJob(Path inputJar) throws IOException { try (FileSystemUtil.Delegate inputFs = FileSystemUtil.getJarFileSystem(inputJar, false); FileSystemUtil.Delegate incompleteFs = FileSystemUtil.getJarFileSystem(incompleteJar, true); - FileSystemUtil.Delegate existingFs = FileSystemUtil.getJarFileSystem(existingJar, true)) { + FileSystemUtil.Delegate existingSourcesFs = FileSystemUtil.getJarFileSystem(existingSourcesJar, true); + FileSystemUtil.Delegate existingClassesFs = FileSystemUtil.getJarFileSystem(existingClassesJar, true)) { final List inputClasses = JarWalker.findClasses(inputFs); + final Map rawEntryHashes = getEntryHashes(inputClasses, inputFs.getRoot()); for (ClassEntry entry : inputClasses) { String outputFileName = entry.sourcesFileName(); - String fullHash = baseHash + "/" + entry.hash(inputFs.getRoot()); + String fullHash = baseHash + "/" + entry.hashSuperHierarchy(rawEntryHashes); final CachedData entryData = fileStore.getEntry(fullHash); @@ -82,10 +86,12 @@ public WorkRequest prepareJob(Path inputJar) throws IOException { LOGGER.debug("Cached entry ({}) not found, going to process {}", fullHash, outputFileName); misses++; } else { - final Path outputPath = existingFs.getPath(outputFileName); + final Path outputPath = existingSourcesFs.getPath(outputFileName); Files.createDirectories(outputPath.getParent()); Files.writeString(outputPath, entryData.sources()); + entry.copyTo(inputFs.getRoot(), existingClassesFs.getRoot()); + if (entryData.lineNumbers() != null) { lineNumbersMap.put(entryData.className(), entryData.lineNumbers()); } else { @@ -110,7 +116,8 @@ public WorkRequest prepareJob(Path inputJar) throws IOException { if (isIncomplete && !hasSomeExisting) { // The cache contained nothing of use, fully process the input jar Files.delete(incompleteJar); - Files.delete(existingJar); + Files.delete(existingClassesJar); + Files.delete(existingSourcesJar); LOGGER.info("No cached entries found, going to process the whole jar"); return new FullWorkJob(inputJar, outputJar, outputNameMap) @@ -118,17 +125,33 @@ public WorkRequest prepareJob(Path inputJar) throws IOException { } else if (isIncomplete) { // The cache did not contain everything so we have some work to do LOGGER.info("Some cached entries found, using partial work job"); - return new PartialWorkJob(incompleteJar, existingJar, outputJar, outputNameMap) + return new PartialWorkJob(incompleteJar, existingSourcesJar, existingClassesJar, outputJar, outputNameMap) .asRequest(stats, lineNumbers); } else { // The cached contained everything we need, so the existing jar is the output LOGGER.info("All cached entries found, using completed work job"); Files.delete(incompleteJar); - return new CompletedWorkJob(existingJar) + Files.delete(existingClassesJar); + return new CompletedWorkJob(existingSourcesJar) .asRequest(stats, lineNumbers); } } + private static Map getEntryHashes(List entries, Path root) throws IOException { + final Map rawEntryHashes = new HashMap<>(); + + for (ClassEntry entry : entries) { + String hash = entry.hash(root); + rawEntryHashes.put(entry.name(), hash); + + for (String s : entry.innerClasses()) { + rawEntryHashes.put(s, hash); + } + } + + return Collections.unmodifiableMap(rawEntryHashes); + } + public void completeJob(Path output, WorkJob workJob, ClassLineNumbers lineNumbers) throws IOException { if (workJob instanceof CompletedWorkJob completedWorkJob) { // Fully complete, nothing new to cache @@ -189,7 +212,7 @@ public void completeJob(Path output, WorkJob workJob, ClassLineNumbers lineNumbe if (workJob instanceof PartialWorkJob partialWorkJob) { // Copy all the existing items to the output jar try (FileSystemUtil.Delegate outputFs = FileSystemUtil.getJarFileSystem(partialWorkJob.output(), false); - FileSystemUtil.Delegate existingFs = FileSystemUtil.getJarFileSystem(partialWorkJob.existing(), false); + FileSystemUtil.Delegate existingFs = FileSystemUtil.getJarFileSystem(partialWorkJob.existingSources(), false); Stream walk = Files.walk(existingFs.getRoot())) { Iterator iterator = walk.iterator(); @@ -208,7 +231,8 @@ public void completeJob(Path output, WorkJob workJob, ClassLineNumbers lineNumbe } } - Files.delete(partialWorkJob.existing()); + Files.delete(partialWorkJob.existingSources()); + Files.delete(partialWorkJob.existingClasses()); Files.move(partialWorkJob.output(), output); } else if (workJob instanceof FullWorkJob fullWorkJob) { // Nothing to merge, just use the output jar @@ -259,11 +283,12 @@ public record CompletedWorkJob(Path completed) implements WorkJob { * Some work needs to be done. * * @param incomplete A path to jar file containing all the classes to be processed - * @param existing A path pointing to a jar containing existing classes that have previously been processed + * @param existingSources A path pointing to a jar containing existing sources that have previously been processed + * @param existingClasses A path pointing to a jar containing existing classes that have previously been processed * @param output A path to a temporary jar where work output should be written to * @param outputNameMap A map of sources name to hash */ - public record PartialWorkJob(Path incomplete, Path existing, Path output, Map outputNameMap) implements WorkToDoJob { + public record PartialWorkJob(Path incomplete, Path existingSources, Path existingClasses, Path output, Map outputNameMap) implements WorkToDoJob { } /** diff --git a/src/main/java/net/fabricmc/loom/decompilers/cache/ClassEntry.java b/src/main/java/net/fabricmc/loom/decompilers/cache/ClassEntry.java index f01db22f7..666ae2c2f 100644 --- a/src/main/java/net/fabricmc/loom/decompilers/cache/ClassEntry.java +++ b/src/main/java/net/fabricmc/loom/decompilers/cache/ClassEntry.java @@ -28,11 +28,23 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.StringJoiner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import net.fabricmc.loom.util.Checksum; -public record ClassEntry(String parentClass, List innerClasses) { +/** + * @param name The class name + * @param innerClasses A list of inner class names + * @param superClasses A list of parent classes (super and interface) from the class and all inner classes + */ +public record ClassEntry(String name, List innerClasses, List superClasses) { + private static final Logger LOGGER = LoggerFactory.getLogger(ClassEntry.class); + /** * Copy the class and its inner classes to the target root. * @param sourceRoot The root of the source jar @@ -41,9 +53,9 @@ public record ClassEntry(String parentClass, List innerClasses) { * @throws IOException If an error occurs while copying the files */ public void copyTo(Path sourceRoot, Path targetRoot) throws IOException { - Path targetPath = targetRoot.resolve(parentClass); + Path targetPath = targetRoot.resolve(name); Files.createDirectories(targetPath.getParent()); - Files.copy(sourceRoot.resolve(parentClass), targetPath); + Files.copy(sourceRoot.resolve(name), targetPath); for (String innerClass : innerClasses) { Files.copy(sourceRoot.resolve(innerClass), targetRoot.resolve(innerClass)); @@ -60,7 +72,7 @@ public void copyTo(Path sourceRoot, Path targetRoot) throws IOException { public String hash(Path root) throws IOException { StringJoiner joiner = new StringJoiner(","); - joiner.add(Checksum.sha256Hex(Files.readAllBytes(root.resolve(parentClass)))); + joiner.add(Checksum.sha256Hex(Files.readAllBytes(root.resolve(name)))); for (String innerClass : innerClasses) { joiner.add(Checksum.sha256Hex(Files.readAllBytes(root.resolve(innerClass)))); @@ -69,7 +81,34 @@ public String hash(Path root) throws IOException { return Checksum.sha256Hex(joiner.toString().getBytes()); } + /** + * Return a hash of the class and its super classes. + */ + public String hashSuperHierarchy(Map hashes) throws IOException { + final String selfHash = Objects.requireNonNull(hashes.get(name), "Hash for own class not found"); + + if (superClasses.isEmpty()) { + return selfHash; + } + + StringJoiner joiner = new StringJoiner(","); + joiner.add(selfHash); + + for (String superClass : superClasses) { + final String superHash = hashes.get(superClass + ".class"); + + if (superHash != null) { + joiner.add(superHash); + } else if (!superClass.startsWith("java/")) { + // This will happen if the super class is not part of the input jar + LOGGER.debug("Hash for super class {} of {} not found", superClass, name); + } + } + + return Checksum.sha256Hex(joiner.toString().getBytes()); + } + public String sourcesFileName() { - return parentClass.replace(".class", ".java"); + return name.replace(".class", ".java"); } } diff --git a/src/main/java/net/fabricmc/loom/decompilers/cache/JarWalker.java b/src/main/java/net/fabricmc/loom/decompilers/cache/JarWalker.java index ab2f9924d..4c2c35ca7 100644 --- a/src/main/java/net/fabricmc/loom/decompilers/cache/JarWalker.java +++ b/src/main/java/net/fabricmc/loom/decompilers/cache/JarWalker.java @@ -25,6 +25,10 @@ package net.fabricmc.loom.decompilers.cache; import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -33,11 +37,22 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Stream; +import org.gradle.api.JavaVersion; +import org.objectweb.asm.ClassReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import net.fabricmc.loom.util.CompletableFutureCollector; import net.fabricmc.loom.util.FileSystemUtil; public final class JarWalker { @@ -88,7 +103,8 @@ public static List findClasses(FileSystemUtil.Delegate fs) throws IO Collections.sort(outerClasses); - List classEntries = new ArrayList<>(); + final Executor executor = getExecutor(); + List> classEntries = new ArrayList<>(); for (String outerClass : outerClasses) { List innerClasList = innerClasses.get(outerClass); @@ -99,10 +115,71 @@ public static List findClasses(FileSystemUtil.Delegate fs) throws IO Collections.sort(innerClasList); } - ClassEntry classEntry = new ClassEntry(outerClass, Collections.unmodifiableList(innerClasList)); - classEntries.add(classEntry); + classEntries.add(getClassEntry(outerClass, innerClasList, fs, executor)); } - return Collections.unmodifiableList(classEntries); + try { + return classEntries.stream() + .collect(CompletableFutureCollector.allOf()) + .get(10, TimeUnit.MINUTES); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException("Failed to get class entries", e); + } + } + + private static CompletableFuture getClassEntry(String outerClass, List innerClasses, FileSystemUtil.Delegate fs, Executor executor) { + List>> parentClassesFutures = new ArrayList<>(); + + // Get the super classes of the outer class and any inner classes + parentClassesFutures.add(CompletableFuture.supplyAsync(() -> getSuperClasses(outerClass, fs), executor)); + + for (String innerClass : innerClasses) { + parentClassesFutures.add(CompletableFuture.supplyAsync(() -> getSuperClasses(innerClass, fs), executor)); + } + + return parentClassesFutures.stream() + .collect(CompletableFutureCollector.allOf()) + .thenApply(lists -> lists.stream() + .flatMap(List::stream) + .filter(JarWalker::isNotReservedClass) + .distinct() + .toList()) + .thenApply(parentClasses -> new ClassEntry(outerClass, innerClasses, parentClasses)); + } + + private static List getSuperClasses(String classFile, FileSystemUtil.Delegate fs) { + try (InputStream is = Files.newInputStream(fs.getPath(classFile))) { + final ClassReader reader = new ClassReader(is); + + List parentClasses = new ArrayList<>(); + String superName = reader.getSuperName(); + + if (superName != null) { + parentClasses.add(superName); + } + + Collections.addAll(parentClasses, reader.getInterfaces()); + return Collections.unmodifiableList(parentClasses); + } catch (IOException e) { + throw new UncheckedIOException("Failed to read class file: " + classFile, e); + } + } + + private static Executor getExecutor() { + if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) { + try { + Method m = Executors.class.getMethod("newVirtualThreadPerTaskExecutor"); + return (ExecutorService) m.invoke(null); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException("Failed to create virtual thread executor", e); + } + } + + return ForkJoinPool.commonPool(); + } + + // Slight optimization, if we skip over Object + private static boolean isNotReservedClass(String name) { + return !"java/lang/Object".equals(name); } } diff --git a/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java b/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java index 842eeee4b..a0bd80141 100644 --- a/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java +++ b/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java @@ -156,6 +156,11 @@ public abstract class GenerateSourcesTask extends AbstractLoomTask { @ApiStatus.Experimental public abstract Property getUseCache(); + @Input + @Option(option = "reset-cache", description = "When set the cache will be reset") + @ApiStatus.Experimental + public abstract Property getResetCache(); + // Internal outputs @ApiStatus.Internal @Internal @@ -188,6 +193,7 @@ public GenerateSourcesTask(DecompilerOptions decompilerOptions) { getUnpickRuntimeClasspath().from(getProject().getConfigurations().getByName(Constants.Configurations.UNPICK_CLASSPATH)); getUseCache().convention(true); + getResetCache().convention(false); } @TaskAction @@ -214,6 +220,11 @@ public void run() throws IOException { try (var timer = new Timer("Decompiled sources with cache")) { final Path cacheFile = getDecompileCacheFile().getAsFile().get().toPath(); + if (getResetCache().get()) { + LOGGER.warn("Resetting decompile cache"); + Files.deleteIfExists(cacheFile); + } + // TODO ensure we have a lock on this file to prevent multiple tasks from running at the same time // TODO handle being unable to read the cache file Files.createDirectories(cacheFile.getParent()); @@ -250,16 +261,16 @@ private void runWithCache(Path cacheRoot) throws IOException { if (job instanceof CachedJarProcessor.WorkToDoJob workToDoJob) { Path inputJar = workToDoJob.incomplete(); - @Nullable Path existing = (job instanceof CachedJarProcessor.PartialWorkJob partialWorkJob) ? partialWorkJob.existing() : null; + @Nullable Path existingClasses = (job instanceof CachedJarProcessor.PartialWorkJob partialWorkJob) ? partialWorkJob.existingClasses() : null; if (getUnpickDefinitions().isPresent()) { try (var timer = new Timer("Unpick")) { - inputJar = unpickJar(inputJar, existing); + inputJar = unpickJar(inputJar, existingClasses); } } try (var timer = new Timer("Decompile")) { - outputLineNumbers = runDecompileJob(inputJar, workToDoJob.output(), existing); + outputLineNumbers = runDecompileJob(inputJar, workToDoJob.output(), existingClasses); } if (Files.notExists(workToDoJob.output())) { @@ -277,6 +288,8 @@ private void runWithCache(Path cacheRoot) throws IOException { cachedJarProcessor.completeJob(sourcesJar, job, outputLineNumbers); } + LOGGER.info("Decompiled sources written to {}", sourcesJar); + // This is the minecraft jar used at runtime. final Path classesJar = minecraftJar.getPath(); @@ -326,6 +339,8 @@ private void runWithoutCache() throws IOException { throw new RuntimeException("Failed to decompile sources"); } + LOGGER.info("Decompiled sources written to {}", sourcesJar); + if (lineNumbers == null) { LOGGER.info("No line numbers to remap, skipping remapping"); return; @@ -436,9 +451,9 @@ private MinecraftJar rebuildInputJar() { ); } - private Path unpickJar(Path inputJar, @Nullable Path existingJar) { + private Path unpickJar(Path inputJar, @Nullable Path existingClasses) { final Path outputJar = getUnpickOutputJar().get().getAsFile().toPath(); - final List args = getUnpickArgs(inputJar, outputJar, existingJar); + final List args = getUnpickArgs(inputJar, outputJar, existingClasses); ExecResult result = getExecOperations().javaexec(spec -> { spec.getMainClass().set("daomephsta.unpick.cli.Main"); @@ -452,7 +467,7 @@ private Path unpickJar(Path inputJar, @Nullable Path existingJar) { return outputJar; } - private List getUnpickArgs(Path inputJar, Path outputJar, @Nullable Path existingJar) { + private List getUnpickArgs(Path inputJar, Path outputJar, @Nullable Path existingClasses) { var fileArgs = new ArrayList(); fileArgs.add(inputJar.toFile()); @@ -469,8 +484,8 @@ private List getUnpickArgs(Path inputJar, Path outputJar, @Nullable Path fileArgs.add(file); } - if (existingJar != null) { - fileArgs.add(existingJar.toFile()); + if (existingClasses != null) { + fileArgs.add(existingClasses.toFile()); } return fileArgs.stream() @@ -505,15 +520,15 @@ private void remapLineNumbers(ClassLineNumbers lineNumbers, Path inputJar, Path LOGGER.info("Wrote linemap to {}", lineMap); } - private void doWork(@Nullable IPCServer ipcServer, Path inputJar, Path outputJar, Path linemapFile, @Nullable Path existingJar) { + private void doWork(@Nullable IPCServer ipcServer, Path inputJar, Path outputJar, Path linemapFile, @Nullable Path existingClasses) { final String jvmMarkerValue = UUID.randomUUID().toString(); final WorkQueue workQueue = createWorkQueue(jvmMarkerValue); ConfigurableFileCollection classpath = getProject().files(); classpath.from(getProject().getConfigurations().getByName(Constants.Configurations.MINECRAFT_COMPILE_LIBRARIES)); - if (existingJar != null) { - classpath.from(existingJar); + if (existingClasses != null) { + classpath.from(existingClasses); } workQueue.submit(DecompileAction.class, params -> { diff --git a/src/main/java/net/fabricmc/loom/util/CompletableFutureCollector.java b/src/main/java/net/fabricmc/loom/util/CompletableFutureCollector.java new file mode 100644 index 000000000..77ccb20f4 --- /dev/null +++ b/src/main/java/net/fabricmc/loom/util/CompletableFutureCollector.java @@ -0,0 +1,77 @@ +/* + * This file is part of fabric-loom, licensed under the MIT License (MIT). + * + * Copyright (c) 2024 FabricMC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package net.fabricmc.loom.util; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; + +public final class CompletableFutureCollector> implements Collector, CompletableFuture>> { + private CompletableFutureCollector() { + } + + public static > Collector, CompletableFuture>> allOf() { + return new CompletableFutureCollector<>(); + } + + @Override + public Supplier> supplier() { + return ArrayList::new; + } + + @Override + public BiConsumer, T> accumulator() { + return List::add; + } + + @Override + public BinaryOperator> combiner() { + return (left, right) -> { + left.addAll(right); + return left; + }; + } + + @Override + public Function, CompletableFuture>> finisher() { + return ls -> CompletableFuture.allOf(ls.toArray(CompletableFuture[]::new)) + .thenApply(v -> ls + .stream() + .map(CompletableFuture::join) + .toList()); + } + + @Override + public Set characteristics() { + return Collections.emptySet(); + } +} diff --git a/src/test/groovy/net/fabricmc/loom/test/unit/JarWalkerTest.groovy b/src/test/groovy/net/fabricmc/loom/test/unit/JarWalkerTest.groovy deleted file mode 100644 index 7f0d0c57e..000000000 --- a/src/test/groovy/net/fabricmc/loom/test/unit/JarWalkerTest.groovy +++ /dev/null @@ -1,87 +0,0 @@ -/* - * This file is part of fabric-loom, licensed under the MIT License (MIT). - * - * Copyright (c) 2024 FabricMC - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package net.fabricmc.loom.test.unit - -import spock.lang.Specification - -import net.fabricmc.loom.decompilers.cache.JarWalker -import net.fabricmc.loom.test.util.ZipTestUtils -import net.fabricmc.loom.util.FileSystemUtil - -class JarWalkerTest extends Specification { - def "find classes in jar"() { - given: - def jar = ZipTestUtils.createZip([ - "net/fabricmc/Test.class": "", - "net/fabricmc/other/Test.class": "", - "net/fabricmc/other/Test\$Inner.class": "", - "net/fabricmc/other/Test\$1.class": "", - ]) - when: - def entries = JarWalker.findClasses(jar) - then: - entries.size() == 2 - - entries[0].parentClass() == "net/fabricmc/Test.class" - entries[0].sourcesFileName() == "net/fabricmc/Test.java" - entries[0].innerClasses().size() == 0 - - entries[1].parentClass() == "net/fabricmc/other/Test.class" - entries[1].sourcesFileName() == "net/fabricmc/other/Test.java" - entries[1].innerClasses().size() == 2 - entries[1].innerClasses()[0] == "net/fabricmc/other/Test\$1.class" - entries[1].innerClasses()[1] == "net/fabricmc/other/Test\$Inner.class" - } - - def "Hash Classes"() { - given: - def jar = ZipTestUtils.createZip(zipEntries) - when: - def entries = JarWalker.findClasses(jar) - def hash = FileSystemUtil.getJarFileSystem(jar).withCloseable { fs -> - return entries[0].hash(fs.root) - } - then: - entries.size() == 1 - hash == expectedHash - where: - expectedHash | zipEntries - "2339de144d8a4a1198adf8142b6d3421ec0baacea13c9ade42a93071b6d62e43" | [ - "net/fabricmc/Test.class": "abc123", - ] - "1053cfadf4e371ec89ff5b58d9b3bdb80373f3179e804b2e241171223709f4d1" | [ - "net/fabricmc/other/Test.class": "Hello", - "net/fabricmc/other/Test\$Inner.class": "World", - "net/fabricmc/other/Test\$Inner\$2.class": "123", - "net/fabricmc/other/Test\$1.class": "test", - ] - "f30b705f3a921b60103a4ee9951aff59b6db87cc289ba24563743d753acff433" | [ - "net/fabricmc/other/Test.class": "Hello", - "net/fabricmc/other/Test\$Inner.class": "World", - "net/fabricmc/other/Test\$Inner\$2.class": "abc123", - "net/fabricmc/other/Test\$1.class": "test", - ] - } -} diff --git a/src/test/groovy/net/fabricmc/loom/test/unit/cache/CachedJarProcessorTest.groovy b/src/test/groovy/net/fabricmc/loom/test/unit/cache/CachedJarProcessorTest.groovy index 6b157603d..9e2e88565 100644 --- a/src/test/groovy/net/fabricmc/loom/test/unit/cache/CachedJarProcessorTest.groovy +++ b/src/test/groovy/net/fabricmc/loom/test/unit/cache/CachedJarProcessorTest.groovy @@ -25,33 +25,42 @@ package net.fabricmc.loom.test.unit.cache import java.nio.file.Files +import java.nio.file.Path +import java.time.Duration +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.Opcodes import spock.lang.Specification +import spock.lang.TempDir import net.fabricmc.loom.decompilers.ClassLineNumbers import net.fabricmc.loom.decompilers.cache.CachedData import net.fabricmc.loom.decompilers.cache.CachedFileStore +import net.fabricmc.loom.decompilers.cache.CachedFileStoreImpl import net.fabricmc.loom.decompilers.cache.CachedJarProcessor import net.fabricmc.loom.test.util.ZipTestUtils import net.fabricmc.loom.util.ZipUtils class CachedJarProcessorTest extends Specification { - static Map jarEntries = [ - "net/fabricmc/Example.class": "", - "net/fabricmc/other/Test.class": "", - "net/fabricmc/other/Test\$Inner.class": "", - "net/fabricmc/other/Test\$1.class": "", + static Map jarEntries = [ + "net/fabricmc/Example.class": newClass("net/fabricmc/Example"), + "net/fabricmc/other/Test.class": newClass("net/fabricmc/other/Test"), + "net/fabricmc/other/Test\$Inner.class": newClass("net/fabricmc/other/Test\$Inner"), + "net/fabricmc/other/Test\$1.class": newClass("net/fabricmc/other/Test\$1"), ] - static String ExampleHash = "abc123/cd372fb85148700fa88095e3492d3f9f5beb43e555e5ff26d95f5a6adc36f8e6" - static String TestHash = "abc123/ecd40b16ec50b636a390cb8da716a22606965f14e526e3051144dd567f336bc5" + static String ExampleHash = "abc123/db5c3a2d04e0c6ea03aef0d217517aa0233f9b8198753d3c96574fe5825a13c4" + static String TestHash = "abc123/06f9f4c7dbca9baa037fbea007298ee15277d97de594bbf6e4a1ee346c079e65" static CachedData ExampleCachedData = new CachedData("net/fabricmc/Example", "Example sources", lineNumber("net/fabricmc/Example")) static CachedData TestCachedData = new CachedData("net/fabricmc/other/Test", "Test sources", lineNumber("net/fabricmc/other/Test")) + @TempDir + Path testPath + def "prepare full work job"() { given: - def jar = ZipTestUtils.createZip(jarEntries) + def jar = ZipTestUtils.createZipFromBytes(jarEntries) def cache = Mock(CachedFileStore) def processor = new CachedJarProcessor(cache, "abc123") @@ -71,7 +80,7 @@ class CachedJarProcessorTest extends Specification { def "prepare partial work job"() { given: - def jar = ZipTestUtils.createZip(jarEntries) + def jar = ZipTestUtils.createZipFromBytes(jarEntries) def cache = Mock(CachedFileStore) def processor = new CachedJarProcessor(cache, "abc123") @@ -85,7 +94,8 @@ class CachedJarProcessorTest extends Specification { lineMap.get("net/fabricmc/Example") == ExampleCachedData.lineNumbers() workJob.outputNameMap().size() == 1 - ZipUtils.unpackNullable(workJob.existing(), "net/fabricmc/Example.java") == "Example sources".bytes + ZipUtils.unpackNullable(workJob.existingSources(), "net/fabricmc/Example.java") == "Example sources".bytes + ZipUtils.unpackNullable(workJob.existingClasses(), "net/fabricmc/Example.class") == newClass("net/fabricmc/Example") // Provide one cached entry // And then one call not finding the entry in the cache @@ -97,7 +107,7 @@ class CachedJarProcessorTest extends Specification { def "prepare completed work job"() { given: - def jar = ZipTestUtils.createZip(jarEntries) + def jar = ZipTestUtils.createZipFromBytes(jarEntries) def cache = Mock(CachedFileStore) def processor = new CachedJarProcessor(cache, "abc123") @@ -125,7 +135,7 @@ class CachedJarProcessorTest extends Specification { def "complete full work job"() { given: - def jar = ZipTestUtils.createZip(jarEntries) + def jar = ZipTestUtils.createZipFromBytes(jarEntries) def cache = Mock(CachedFileStore) def processor = new CachedJarProcessor(cache, "abc123") @@ -152,7 +162,7 @@ class CachedJarProcessorTest extends Specification { ZipUtils.unpackNullable(outputJar, "net/fabricmc/Example.java") == "Example sources".bytes ZipUtils.unpackNullable(outputJar, "net/fabricmc/other/Test.java") == "Test sources".bytes - // Expect two calls looking for the existing entry in the cache + // Expect two calls looking for the existingSources entry in the cache 1 * cache.getEntry(ExampleHash) >> null 1 * cache.getEntry(TestHash) >> null @@ -165,7 +175,7 @@ class CachedJarProcessorTest extends Specification { def "complete partial work job"() { given: - def jar = ZipTestUtils.createZip(jarEntries) + def jar = ZipTestUtils.createZipFromBytes(jarEntries) def cache = Mock(CachedFileStore) def processor = new CachedJarProcessor(cache, "abc123") @@ -203,7 +213,7 @@ class CachedJarProcessorTest extends Specification { def "complete completed work job"() { given: - def jar = ZipTestUtils.createZip(jarEntries) + def jar = ZipTestUtils.createZipFromBytes(jarEntries) def cache = Mock(CachedFileStore) def processor = new CachedJarProcessor(cache, "abc123") @@ -231,6 +241,55 @@ class CachedJarProcessorTest extends Specification { 0 * _ // Strict mock } + def "hierarchy change invalidates cache"() { + given: + def jar1 = ZipTestUtils.createZipFromBytes( + [ + "net/fabricmc/Example.class": newClass("net/fabricmc/Example"), + "net/fabricmc/other/Test.class": newClass("net/fabricmc/other/Test", ), + "net/fabricmc/other/Test\$Inner.class": newClass("net/fabricmc/other/Test\$Inner", ["net/fabricmc/Example"] as String[]), + "net/fabricmc/other/Test\$1.class": newClass("net/fabricmc/other/Test\$1"), + ] + ) + // The second jar changes Example, so we would expect Test to be invalidated, thus causing a full decompile in this case + def jar2 = ZipTestUtils.createZipFromBytes( + [ + "net/fabricmc/Example.class": newClass("net/fabricmc/Example", ["java/lang/Runnable"] as String[]), + "net/fabricmc/other/Test.class": newClass("net/fabricmc/other/Test", ), + "net/fabricmc/other/Test\$Inner.class": newClass("net/fabricmc/other/Test\$Inner", ["net/fabricmc/Example"] as String[]), + "net/fabricmc/other/Test\$1.class": newClass("net/fabricmc/other/Test\$1"), + ] + ) + + def cache = new CachedFileStoreImpl<>(testPath.resolve("cache"), CachedData.SERIALIZER, new CachedFileStoreImpl.CacheRules(50_000, Duration.ofDays(90))) + def processor = new CachedJarProcessor(cache, "abc123") + + when: + def workRequest = processor.prepareJob(jar1) + def workJob = workRequest.job() as CachedJarProcessor.FullWorkJob + def outputSourcesJar = workJob.output() + + // Do the work, such as decompiling. + ZipUtils.add(outputSourcesJar, "net/fabricmc/Example.java", "Example sources") + ZipUtils.add(outputSourcesJar, "net/fabricmc/other/Test.java", "Test sources") + + // Complete the job + def outputJar = Files.createTempFile("loom-test-output", ".jar") + Files.delete(outputJar) + + ClassLineNumbers lineNumbers = lineNumbers([ + "net/fabricmc/Example", + "net/fabricmc/other/Test" + ]) + processor.completeJob(outputJar, workJob, lineNumbers) + + workRequest = processor.prepareJob(jar2) + def newWorkJob = workRequest.job() + + then: + newWorkJob instanceof CachedJarProcessor.FullWorkJob + } + private static ClassLineNumbers lineNumbers(List names) { return new ClassLineNumbers(names.collectEntries { [it, lineNumber(it)] }) } @@ -238,4 +297,10 @@ class CachedJarProcessorTest extends Specification { private static ClassLineNumbers.Entry lineNumber(String name) { return new ClassLineNumbers.Entry(name, 0, 0, [:]) } + + private static byte[] newClass(String name, String[] interfaces = null, String superName = "java/lang/Object") { + def writer = new ClassWriter(0) + writer.visit(Opcodes.V17, Opcodes.ACC_PUBLIC, name, null, superName, interfaces) + return writer.toByteArray() + } } diff --git a/src/test/groovy/net/fabricmc/loom/test/unit/cache/JarWalkerTest.groovy b/src/test/groovy/net/fabricmc/loom/test/unit/cache/JarWalkerTest.groovy new file mode 100644 index 000000000..3490738c8 --- /dev/null +++ b/src/test/groovy/net/fabricmc/loom/test/unit/cache/JarWalkerTest.groovy @@ -0,0 +1,154 @@ +/* + * This file is part of fabric-loom, licensed under the MIT License (MIT). + * + * Copyright (c) 2024 FabricMC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package net.fabricmc.loom.test.unit.cache + +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.Opcodes +import spock.lang.Specification + +import net.fabricmc.loom.decompilers.cache.JarWalker +import net.fabricmc.loom.test.util.ZipTestUtils +import net.fabricmc.loom.util.FileSystemUtil + +class JarWalkerTest extends Specification { + def "find classes in jar"() { + given: + def jar = ZipTestUtils.createZipFromBytes([ + "net/fabricmc/Test.class": newClass("net/fabricmc/Test"), + "net/fabricmc/other/Test.class": newClass("net/fabricmc/other/Test"), + "net/fabricmc/other/Test\$Inner.class": newClass("net/fabricmc/other/Test\$Inner"), + "net/fabricmc/other/Test\$1.class": newClass("net/fabricmc/other/Test\$1"), + ]) + when: + def entries = JarWalker.findClasses(jar) + then: + entries.size() == 2 + + entries[0].name() == "net/fabricmc/Test.class" + entries[0].sourcesFileName() == "net/fabricmc/Test.java" + entries[0].innerClasses().size() == 0 + + entries[1].name() == "net/fabricmc/other/Test.class" + entries[1].sourcesFileName() == "net/fabricmc/other/Test.java" + entries[1].innerClasses().size() == 2 + entries[1].innerClasses()[0] == "net/fabricmc/other/Test\$1.class" + entries[1].innerClasses()[1] == "net/fabricmc/other/Test\$Inner.class" + } + + def "Hash Classes"() { + given: + def jar = ZipTestUtils.createZipFromBytes(zipEntries) + when: + def entries = JarWalker.findClasses(jar) + def hash = FileSystemUtil.getJarFileSystem(jar).withCloseable { fs -> + return entries[0].hash(fs.root) + } + then: + entries.size() == 1 + hash == expectedHash + where: + expectedHash | zipEntries + "b055df8d9503b60050f6d0db387c84c47fedb4d9ed82c4f8174b4e465a9c479b" | [ + "net/fabricmc/Test.class": newClass("net/fabricmc/Test"), + ] + "3ba069bc20db1ee1b4bb69450dba3fd57a91059bd85e788d5af712aee3191792" | [ + "net/fabricmc/other/Test.class": newClass("net/fabricmc/other/Test"), + "net/fabricmc/other/Test\$Inner.class": newClass("net/fabricmc/other/Test\$Inner"), + "net/fabricmc/other/Test\$Inner\$2.class": newClass("net/fabricmc/other/Test\$Inner\$2"), + "net/fabricmc/other/Test\$1.class": newClass("net/fabricmc/other/Test\$1"), + ] + "3ba069bc20db1ee1b4bb69450dba3fd57a91059bd85e788d5af712aee3191792" | [ + "net/fabricmc/other/Test.class": newClass("net/fabricmc/other/Test"), + "net/fabricmc/other/Test\$Inner.class": newClass("net/fabricmc/other/Test\$Inner"), + "net/fabricmc/other/Test\$Inner\$2.class": newClass("net/fabricmc/other/Test\$Inner\$2"), + "net/fabricmc/other/Test\$1.class": newClass("net/fabricmc/other/Test\$1"), + ] + } + + def "simple class"() { + given: + def jarEntries = [ + "net/fabricmc/Example.class": newClass("net/fabricmc/Example") + ] + def jar = ZipTestUtils.createZipFromBytes(jarEntries) + + when: + def classes = JarWalker.findClasses(jar) + + then: + classes.size() == 1 + classes[0].name() == "net/fabricmc/Example.class" + classes[0].innerClasses() == [] + classes[0].superClasses() == [] + } + + def "class with interfaces"() { + given: + def jarEntries = [ + "net/fabricmc/Example.class": newClass("net/fabricmc/Example", ["java/lang/Runnable"] as String[]) + ] + def jar = ZipTestUtils.createZipFromBytes(jarEntries) + + when: + def classes = JarWalker.findClasses(jar) + + then: + classes.size() == 1 + classes[0].name() == "net/fabricmc/Example.class" + classes[0].innerClasses() == [] + classes[0].superClasses() == ["java/lang/Runnable"] + } + + def "inner classes"() { + given: + def jarEntries = [ + "net/fabricmc/other/Test.class": newClass("net/fabricmc/other/Test"), + "net/fabricmc/other/Test\$Inner.class": newClass("net/fabricmc/other/Test\$Inner", null, "net/fabricmc/other/Super"), + "net/fabricmc/other/Test\$1.class": newClass("net/fabricmc/other/Test\$1", ["java/lang/Runnable"] as String[]), + ] + def jar = ZipTestUtils.createZipFromBytes(jarEntries) + + when: + def classes = JarWalker.findClasses(jar) + + then: + classes.size() == 1 + classes[0].name() == "net/fabricmc/other/Test.class" + classes[0].innerClasses() == [ + "net/fabricmc/other/Test\$1.class", + "net/fabricmc/other/Test\$Inner.class" + ] + classes[0].superClasses() == [ + "java/lang/Runnable", + "net/fabricmc/other/Super" + ] + } + + private static byte[] newClass(String name, String[] interfaces = null, String superName = "java/lang/Object") { + def writer = new ClassWriter(0) + writer.visit(Opcodes.V17, Opcodes.ACC_PUBLIC, name, null, superName, interfaces) + return writer.toByteArray() + } +}