From e985f2c7431e1c87421a0443e943ec8e80fa05e8 Mon Sep 17 00:00:00 2001 From: Irina Batinic <117161143+irina-batinic-sonarsource@users.noreply.github.com> Date: Wed, 1 Mar 2023 18:06:24 +0100 Subject: [PATCH] Irina/sonarjava 4396 (#4317) --- .../internal/InternalCheckVerifierTest.java | 11 +- .../checks/MissingPackageInfoCheckTest.java | 55 ++-- .../checks/UselessPackageInfoCheckTest.java | 50 ++-- .../checks/helpers/HashCacheTestHelper.java | 73 ++++++ .../ExcessiveContentRequestCheckTest.java | 92 +++++-- ...pringBeansShouldBeAccessibleCheckTest.java | 47 ++-- .../java/org/sonar/java/SonarComponents.java | 56 ++-- .../sonar/java/caching/ContentHashCache.java | 117 +++++++++ .../sonar/java/caching/FileHashingUtils.java | 53 ++++ .../org/sonar/java/SonarComponentsTest.java | 38 ++- .../java/caching/ContentHashCacheTest.java | 240 ++++++++++++++++++ .../sonar/java/model/VisitorsBridgeTest.java | 4 + 12 files changed, 706 insertions(+), 130 deletions(-) create mode 100644 java-checks/src/test/java/org/sonar/java/checks/helpers/HashCacheTestHelper.java create mode 100644 java-frontend/src/main/java/org/sonar/java/caching/ContentHashCache.java create mode 100644 java-frontend/src/main/java/org/sonar/java/caching/FileHashingUtils.java create mode 100644 java-frontend/src/test/java/org/sonar/java/caching/ContentHashCacheTest.java diff --git a/java-checks-testkit/src/test/java/org/sonar/java/checks/verifier/internal/InternalCheckVerifierTest.java b/java-checks-testkit/src/test/java/org/sonar/java/checks/verifier/internal/InternalCheckVerifierTest.java index b872d716803..1cefb239d2c 100644 --- a/java-checks-testkit/src/test/java/org/sonar/java/checks/verifier/internal/InternalCheckVerifierTest.java +++ b/java-checks-testkit/src/test/java/org/sonar/java/checks/verifier/internal/InternalCheckVerifierTest.java @@ -19,7 +19,10 @@ */ package org.sonar.java.checks.verifier.internal; +import java.io.File; +import java.io.IOException; import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -33,6 +36,7 @@ import org.sonar.check.Rule; import org.sonar.java.AnalysisException; import org.sonar.java.caching.DummyCache; +import org.sonar.java.caching.FileHashingUtils; import org.sonar.java.caching.JavaReadCacheImpl; import org.sonar.java.caching.JavaWriteCacheImpl; import org.sonar.java.reporting.AnalyzerMessage; @@ -950,9 +954,10 @@ void addFiles_throws_an_IllegalArgumentException_if_file_added_before() { } @Test - void withCache_effectively_sets_the_caches_for_scanWithoutParsing() { - ReadCache readCache = new InternalReadCache(); - WriteCache writeCache = new InternalWriteCache(); + void withCache_effectively_sets_the_caches_for_scanWithoutParsing() throws IOException, NoSuchAlgorithmException { + InputFile inputFile = InternalInputFile.inputFile("", new File(TEST_FILE), InputFile.Status.SAME); + ReadCache readCache = new InternalReadCache().put("java:contentHash:MD5::" + TEST_FILE, FileHashingUtils.inputFileContentHash(inputFile)); + WriteCache writeCache = new InternalWriteCache().bind(readCache); CacheContext cacheContext = new InternalCacheContext( true, new JavaReadCacheImpl(readCache), diff --git a/java-checks/src/test/java/org/sonar/java/checks/MissingPackageInfoCheckTest.java b/java-checks/src/test/java/org/sonar/java/checks/MissingPackageInfoCheckTest.java index 3fa4044cc57..71eaec55c9d 100644 --- a/java-checks/src/test/java/org/sonar/java/checks/MissingPackageInfoCheckTest.java +++ b/java-checks/src/test/java/org/sonar/java/checks/MissingPackageInfoCheckTest.java @@ -19,9 +19,6 @@ */ package org.sonar.java.checks; -import java.io.IOException; -import java.io.InputStream; -import java.util.Set; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -30,15 +27,20 @@ import org.sonar.api.utils.log.LogTesterJUnit5; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.java.AnalysisException; +import org.sonar.java.caching.FileHashingUtils; +import org.sonar.java.checks.helpers.HashCacheTestHelper; import org.sonar.java.checks.verifier.CheckVerifier; import org.sonar.java.checks.verifier.internal.InternalReadCache; import org.sonar.java.checks.verifier.internal.InternalWriteCache; -import org.sonar.plugins.java.api.InputFileScannerContext; -import org.sonar.plugins.java.api.JavaFileScannerContext; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.NoSuchAlgorithmException; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Assertions.in; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -98,6 +100,7 @@ void caching() { mainCodeSourcesPath("checks/packageInfo/nopackageinfo/nopackageinfo.java") ) .withCheck(new MissingPackageInfoCheck()) + .withCache(readCache, writeCache) .verifyIssueOnProject(EXPECTED_MESSAGE); var check = spy(new MissingPackageInfoCheck()); @@ -119,26 +122,30 @@ void caching() { verify(check, times(0)).scanFile(any()); verify(check, times(5)).scanWithoutParsing(any()); assertThat(writeCache2.getData()) - .hasSize(5) + .hasSizeGreaterThanOrEqualTo(5) .containsExactlyInAnyOrderEntriesOf(writeCache.getData()); } @Test - void cache_deserialization_throws_IOException() throws IOException { + void cache_deserialization_throws_IOException() throws IOException, NoSuchAlgorithmException { + String filePath = mainCodeSourcesPath("checks/packageInfo/HelloWorld.java"); + InputFile cachedFile = HashCacheTestHelper.inputFileFromPath(filePath); + byte[] cachedHash = FileHashingUtils.inputFileContentHash(cachedFile); var inputStream = mock(InputStream.class); doThrow(new IOException()).when(inputStream).readAllBytes(); - var readCache = mock(ReadCache.class); - doReturn(inputStream).when(readCache).read(any()); - doReturn(true).when(readCache).contains(any()); - - var verifier = CheckVerifier.newVerifier() - .withCache(readCache, writeCache) - .addFiles(InputFile.Status.SAME, - mainCodeSourcesPath("checks/packageInfo/HelloWorld.java") - ) + var localReadCache = mock(ReadCache.class); + InternalWriteCache localWriteCache = new InternalWriteCache().bind(localReadCache); + doReturn(inputStream).when(localReadCache).read("java:S1228;S4032:package:" + cachedFile.key()); + doReturn(true).when(localReadCache).contains(any()); + doReturn(new ByteArrayInputStream(cachedHash)) + .when(localReadCache).read("java:contentHash:MD5:" + cachedFile.key()); + + var localVerifier = CheckVerifier.newVerifier() + .withCache(localReadCache, localWriteCache) + .addFiles(InputFile.Status.SAME, filePath) .withCheck(new MissingPackageInfoCheck()); - assertThatThrownBy(verifier::verifyNoIssues) + assertThatThrownBy(localVerifier::verifyNoIssues) .isInstanceOf(AnalysisException.class) .hasRootCauseInstanceOf(IOException.class); } @@ -159,12 +166,13 @@ void write_cache_multiple_writes() { } @Test - void emptyCache() { + void emptyCache() throws NoSuchAlgorithmException, IOException { logTester.setLevel(LoggerLevel.TRACE); - verifier - .addFiles(InputFile.Status.SAME, - mainCodeSourcesPath("checks/packageInfo/HelloWorld.java") - ) + String filePath = mainCodeSourcesPath("checks/packageInfo/HelloWorld.java"); + ReadCache populatedReadCache = HashCacheTestHelper.internalReadCacheFromFile(filePath); + CheckVerifier.newVerifier() + .addFiles(InputFile.Status.SAME, filePath) + .withCache(populatedReadCache, new InternalWriteCache().bind(populatedReadCache)) .withCheck(new MissingPackageInfoCheck()) .verifyNoIssues(); @@ -172,4 +180,5 @@ void emptyCache() { .filter(msg -> msg.matches("Cache miss for key '[^']+'"))) .hasSize(1); } + } diff --git a/java-checks/src/test/java/org/sonar/java/checks/UselessPackageInfoCheckTest.java b/java-checks/src/test/java/org/sonar/java/checks/UselessPackageInfoCheckTest.java index 54d72ad8c3b..050ea1a58c8 100644 --- a/java-checks/src/test/java/org/sonar/java/checks/UselessPackageInfoCheckTest.java +++ b/java-checks/src/test/java/org/sonar/java/checks/UselessPackageInfoCheckTest.java @@ -19,8 +19,10 @@ */ package org.sonar.java.checks; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.security.NoSuchAlgorithmException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -29,7 +31,8 @@ import org.sonar.api.utils.log.LogTesterJUnit5; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.java.AnalysisException; -import org.sonar.java.caching.CacheReadException; +import org.sonar.java.caching.FileHashingUtils; +import org.sonar.java.checks.helpers.HashCacheTestHelper; import org.sonar.java.checks.verifier.CheckVerifier; import org.sonar.java.checks.verifier.internal.InternalReadCache; import org.sonar.java.checks.verifier.internal.InternalWriteCache; @@ -106,19 +109,24 @@ void defaultPackage() { @Test void caching() throws IOException, ClassNotFoundException { + String changedFilePath1 = mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/package-info.java"); + String changedFilePath2 = mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFiles/package-info.java"); verifier .onFiles( mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/HelloWorld1.java"), mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/HelloWorld2.java"), - mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/package-info.java"), - mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFiles/package-info.java") + changedFilePath1, + changedFilePath2 ) .withCheck(new UselessPackageInfoCheck()) + .withCache(readCache, writeCache) .verifyIssueOnFile("Remove this package."); var check = spy(new UselessPackageInfoCheck()); var populatedReadCache = new InternalReadCache().putAll(writeCache); + populatedReadCache.put(HashCacheTestHelper.contentHashKey(changedFilePath1), new byte[0]); + populatedReadCache.put(HashCacheTestHelper.contentHashKey(changedFilePath2), new byte[0]); var writeCache2 = new InternalWriteCache().bind(populatedReadCache); CheckVerifier.newVerifier() .withCache(populatedReadCache, writeCache2) @@ -127,8 +135,8 @@ void caching() throws IOException, ClassNotFoundException { mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/HelloWorld2.java") ) .addFiles(InputFile.Status.CHANGED, - mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/package-info.java"), - mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFiles/package-info.java") + changedFilePath1, + changedFilePath2 ) .withCheck(check) .verifyIssueOnFile("Remove this package."); @@ -136,23 +144,28 @@ void caching() throws IOException, ClassNotFoundException { verify(check, times(2)).scanFile(any()); verify(check, times(2)).scanWithoutParsing(any()); assertThat(writeCache2.getData()) - .hasSize(4) + .hasSizeGreaterThanOrEqualTo(4) .containsExactlyInAnyOrderEntriesOf(writeCache.getData()); } @Test - void cache_deserialization_throws_IOException() throws IOException { + void cache_deserialization_throws_IOException() throws IOException, NoSuchAlgorithmException { var inputStream = mock(InputStream.class); doThrow(new IOException()).when(inputStream).readAllBytes(); - var readCache = mock(ReadCache.class); - doReturn(inputStream).when(readCache).read(any()); - doReturn(true).when(readCache).contains(any()); + var localReadCache = mock(ReadCache.class); + + String filePath = mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/HelloWorld1.java"); + InputFile cachedFile = HashCacheTestHelper.inputFileFromPath(filePath); + byte[] cachedHash = FileHashingUtils.inputFileContentHash(cachedFile); + + doReturn(inputStream).when(localReadCache).read("java:S1228;S4032:package:"+cachedFile.key()); + doReturn(true).when(localReadCache).contains(any()); + doReturn(new ByteArrayInputStream(cachedHash)) + .when(localReadCache).read("java:contentHash:MD5:"+cachedFile.key()); var verifier = CheckVerifier.newVerifier() - .withCache(readCache, writeCache) - .addFiles(InputFile.Status.SAME, - mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/HelloWorld1.java") - ) + .withCache(localReadCache, new InternalWriteCache().bind(localReadCache)) + .addFiles(InputFile.Status.SAME, filePath) .withCheck(new UselessPackageInfoCheck()); assertThatThrownBy(verifier::verifyNoIssues) @@ -176,13 +189,14 @@ void write_cache_multiple_writes() { } @Test - void emptyCache() { + void emptyCache() throws NoSuchAlgorithmException, IOException { logTester.setLevel(LoggerLevel.TRACE); + String filePath = mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/HelloWorld1.java"); + ReadCache populatedReadCache = HashCacheTestHelper.internalReadCacheFromFile(filePath); verifier - .addFiles(InputFile.Status.SAME, - mainCodeSourcesPath("checks/UselessPackageInfoCheck/packageWithNoOtherFilesButNotPackageInfo/HelloWorld1.java") - ) + .addFiles(InputFile.Status.SAME, filePath) .withCheck(new UselessPackageInfoCheck()) + .withCache(populatedReadCache, new InternalWriteCache().bind(populatedReadCache)) .verifyNoIssues(); assertThat(logTester.logs(LoggerLevel.TRACE).stream() diff --git a/java-checks/src/test/java/org/sonar/java/checks/helpers/HashCacheTestHelper.java b/java-checks/src/test/java/org/sonar/java/checks/helpers/HashCacheTestHelper.java new file mode 100644 index 00000000000..b25ca59f64f --- /dev/null +++ b/java-checks/src/test/java/org/sonar/java/checks/helpers/HashCacheTestHelper.java @@ -0,0 +1,73 @@ +/* + * SonarQube Java + * Copyright (C) 2012-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.java.checks.helpers; + +import java.io.File; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Collection; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.sensor.cache.ReadCache; +import org.sonar.java.caching.FileHashingUtils; +import org.sonar.java.checks.verifier.internal.InternalInputFile; +import org.sonar.java.checks.verifier.internal.InternalReadCache; + +public class HashCacheTestHelper { + + public static InputFile inputFileFromPath(String path) { + return InternalInputFile + .inputFile("", new File(path), InputFile.Status.SAME); + } + + public static String contentHashKey(String path) { + return contentHashKey(inputFileFromPath(path)); + } + + public static String contentHashKey(InputFile inputFile) { + return "java:contentHash:MD5:" + inputFile.key(); + } + + public static ReadCache internalReadCacheFromFile(String path) throws NoSuchAlgorithmException, IOException { + InputFile cachedFile = inputFileFromPath(path); + byte[] cachedHash = FileHashingUtils.inputFileContentHash(cachedFile); + InternalReadCache localReadCache = new InternalReadCache().put(contentHashKey(cachedFile), cachedHash); + return localReadCache; + } + + public static ReadCache internalReadCacheFromFiles(Collection paths) throws NoSuchAlgorithmException, IOException { + InternalReadCache localReadCache = new InternalReadCache(); + for (String path : paths) { + InputFile cachedFile = inputFileFromPath(path); + byte[] cachedHash = FileHashingUtils.inputFileContentHash(cachedFile); + localReadCache.put(contentHashKey(cachedFile), cachedHash); + } + return localReadCache; + } + + public static byte[] getSlightlyDifferentContentHash(String path) throws NoSuchAlgorithmException, IOException { + InputFile cachedFile = inputFileFromPath(path); + byte[] cachedHash = FileHashingUtils.inputFileContentHash(cachedFile); + byte[] copy = Arrays.copyOf(cachedHash, cachedHash.length+1); + copy[cachedHash.length] = 10; + return copy; + } + +} diff --git a/java-checks/src/test/java/org/sonar/java/checks/security/ExcessiveContentRequestCheckTest.java b/java-checks/src/test/java/org/sonar/java/checks/security/ExcessiveContentRequestCheckTest.java index 334262ef808..0c3421f2510 100644 --- a/java-checks/src/test/java/org/sonar/java/checks/security/ExcessiveContentRequestCheckTest.java +++ b/java-checks/src/test/java/org/sonar/java/checks/security/ExcessiveContentRequestCheckTest.java @@ -19,11 +19,6 @@ */ package org.sonar.java.checks.security; -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -33,12 +28,21 @@ import org.sonar.api.utils.log.LogTesterJUnit5; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.java.AnalysisException; +import org.sonar.java.caching.FileHashingUtils; +import org.sonar.java.checks.helpers.HashCacheTestHelper; import org.sonar.java.checks.security.ExcessiveContentRequestCheck.CachedResult; import org.sonar.java.checks.verifier.CheckVerifier; import org.sonar.java.checks.verifier.internal.InternalInputFile; import org.sonar.java.checks.verifier.internal.InternalReadCache; import org.sonar.java.checks.verifier.internal.InternalWriteCache; +import java.io.File; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.any; @@ -89,19 +93,25 @@ String computeCacheKey(String path) { } @Test - void no_issue_raised_on_unchanged_files_with_empty_cache() { + void no_issue_raised_on_unchanged_files_with_empty_cache() throws IOException, NoSuchAlgorithmException { logTester.setLevel(LoggerLevel.TRACE); var check = spy(new ExcessiveContentRequestCheck()); verifier .addFiles(InputFile.Status.SAME, safeSourceFile, unsafeSourceFile, sanitizerSourceFile) - .withCheck(check) - .verifyNoIssues(); + .withCheck(check); + + // Add expected file hashes to the cache to match their status + readCache.put(HashCacheTestHelper.contentHashKey(safeSourceFile), FileHashingUtils.inputFileContentHash(safeSourceFile)); + readCache.put(HashCacheTestHelper.contentHashKey(unsafeSourceFile), FileHashingUtils.inputFileContentHash(unsafeSourceFile)); + readCache.put(HashCacheTestHelper.contentHashKey(sanitizerSourceFile), FileHashingUtils.inputFileContentHash(sanitizerSourceFile)); + + verifier.verifyNoIssues(); verify(check, times(3)).scanWithoutParsing(any()); verify(check, times(3)).leaveFile(any()); - assertThat(writeCache.getData()).containsExactlyInAnyOrderEntriesOf(expectedFinalCacheState); + assertThat(writeCache.getData()).containsAllEntriesOf(expectedFinalCacheState); List logs = logTester.getLogs(LoggerLevel.TRACE).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); assertThat(logs). contains( @@ -112,7 +122,7 @@ void no_issue_raised_on_unchanged_files_with_empty_cache() { } @Test - void no_issue_raised_when_changed_unsafe_file_is_covered_by_unchanged_cached_safe_files() { + void no_issue_raised_when_changed_unsafe_file_is_covered_by_unchanged_cached_safe_files() throws IOException, NoSuchAlgorithmException { readCache.put(computeCacheKey(safeSourceFile), toBytes(new CachedResult(true, true))); readCache.put(computeCacheKey(sanitizerSourceFile), toBytes(new CachedResult(false, true))); @@ -121,17 +131,23 @@ void no_issue_raised_when_changed_unsafe_file_is_covered_by_unchanged_cached_saf verifier .addFiles(InputFile.Status.SAME, safeSourceFile, sanitizerSourceFile) .addFiles(InputFile.Status.CHANGED, unsafeSourceFile) - .withCheck(check) - .verifyNoIssues(); + .withCheck(check); + + // Add expected file hashes to the cache to match their status + readCache.put(HashCacheTestHelper.contentHashKey(safeSourceFile), FileHashingUtils.inputFileContentHash(safeSourceFile)); + readCache.put(HashCacheTestHelper.contentHashKey(unsafeSourceFile), new byte[]{}); + readCache.put(HashCacheTestHelper.contentHashKey(sanitizerSourceFile), FileHashingUtils.inputFileContentHash(sanitizerSourceFile)); + + verifier.verifyNoIssues(); verify(check, times(2)).scanWithoutParsing(any()); verify(check, times(1)).leaveFile(any()); - assertThat(writeCache.getData()).containsExactlyInAnyOrderEntriesOf(expectedFinalCacheState); + assertThat(writeCache.getData()).containsAllEntriesOf(expectedFinalCacheState); } @Test - void no_issue_raised_when_cached_unsafe_file_is_covered_by_changed_safe_files() { + void no_issue_raised_when_cached_unsafe_file_is_covered_by_changed_safe_files() throws IOException, NoSuchAlgorithmException { //readCache.put(computeCacheKey(unsafeSourceFile), new byte[]{1, 0}); readCache.put(computeCacheKey(unsafeSourceFile), toBytes(new CachedResult(true, false))); @@ -139,29 +155,41 @@ void no_issue_raised_when_cached_unsafe_file_is_covered_by_changed_safe_files() verifier .addFiles(InputFile.Status.SAME, unsafeSourceFile) .addFiles(InputFile.Status.CHANGED, safeSourceFile, sanitizerSourceFile) - .withCheck(check) - .verifyNoIssues(); + .withCheck(check); + + // Add expected file hashes to the cache to match their status + readCache.put(HashCacheTestHelper.contentHashKey(safeSourceFile), new byte[]{}); + readCache.put(HashCacheTestHelper.contentHashKey(unsafeSourceFile), FileHashingUtils.inputFileContentHash(unsafeSourceFile)); + readCache.put(HashCacheTestHelper.contentHashKey(safeSourceFile), new byte[]{}); + + verifier.verifyNoIssues(); verify(check, times(1)).scanWithoutParsing(any()); verify(check, times(2)).leaveFile(any()); - assertThat(writeCache.getData()).containsExactlyInAnyOrderEntriesOf(expectedFinalCacheState); + assertThat(writeCache.getData()).containsAllEntriesOf(expectedFinalCacheState); } @Test - void no_issue_raised_when_all_results_are_cached() { + void no_issue_raised_when_all_results_are_cached() throws IOException, NoSuchAlgorithmException { readCache.putAll(expectedFinalCacheState); var check = spy(new ExcessiveContentRequestCheck()); verifier .addFiles(InputFile.Status.SAME, unsafeSourceFile, safeSourceFile, sanitizerSourceFile) - .withCheck(check) - .verifyNoIssues(); + .withCheck(check); + + // Add expected file hashes to the cache to match their status + readCache.put(HashCacheTestHelper.contentHashKey(safeSourceFile), FileHashingUtils.inputFileContentHash(safeSourceFile)); + readCache.put(HashCacheTestHelper.contentHashKey(unsafeSourceFile), FileHashingUtils.inputFileContentHash(unsafeSourceFile)); + readCache.put(HashCacheTestHelper.contentHashKey(sanitizerSourceFile), FileHashingUtils.inputFileContentHash(sanitizerSourceFile)); + + verifier.verifyNoIssues(); verify(check, times(3)).scanWithoutParsing(any()); verify(check, never()).leaveFile(any()); - assertThat(writeCache.getData()).containsExactlyInAnyOrderEntriesOf(expectedFinalCacheState); + assertThat(writeCache.getData()).containsAllEntriesOf(expectedFinalCacheState); } @Test @@ -191,12 +219,12 @@ void log_when_failing_to_write_to_cache() throws IOException { } @Test - void log_when_copying_from_previous_cache() throws IOException { + void log_when_copying_from_previous_cache() throws IOException, NoSuchAlgorithmException { readCache.putAll(expectedFinalCacheState); var spyOnWriteCache = spy(writeCache); IllegalArgumentException expectedException = new IllegalArgumentException("boom"); - doThrow(expectedException).when(spyOnWriteCache).copyFromPrevious(any()); + doThrow(expectedException).when(spyOnWriteCache).copyFromPrevious(computeCacheKey(safeSourceFile)); logTester.setLevel(LoggerLevel.TRACE); @@ -206,6 +234,11 @@ void log_when_copying_from_previous_cache() throws IOException { .withCheck(new ExcessiveContentRequestCheck()) .withCache(readCache, spyOnWriteCache); + // Add expected file hashes to the cache to match their status + readCache.put(HashCacheTestHelper.contentHashKey(safeSourceFile), FileHashingUtils.inputFileContentHash(safeSourceFile)); + readCache.put(HashCacheTestHelper.contentHashKey(unsafeSourceFile), new byte[]{}); + readCache.put(HashCacheTestHelper.contentHashKey(sanitizerSourceFile), new byte[]{}); + assertThatThrownBy(verifier::verifyNoIssues) .isInstanceOf(AnalysisException.class) .hasRootCause(expectedException); @@ -218,7 +251,7 @@ void log_when_copying_from_previous_cache() throws IOException { } @Test - void scanWithoutParsing_returns_false_when_cached_data_is_corrupted() { + void scanWithoutParsing_returns_false_when_cached_data_is_corrupted() throws IOException, NoSuchAlgorithmException { var check = spy(new ExcessiveContentRequestCheck()); readCache.put(computeCacheKey(unsafeSourceFile), null); readCache.put(computeCacheKey(safeSourceFile), new byte[0]); @@ -228,8 +261,13 @@ void scanWithoutParsing_returns_false_when_cached_data_is_corrupted() { verifier .addFiles(InputFile.Status.SAME, unsafeSourceFile, safeSourceFile, sanitizerSourceFile) - .withCheck(check) - .verifyNoIssues(); + .withCheck(check); + + readCache.put(HashCacheTestHelper.contentHashKey(safeSourceFile), FileHashingUtils.inputFileContentHash(safeSourceFile)); + readCache.put(HashCacheTestHelper.contentHashKey(unsafeSourceFile), FileHashingUtils.inputFileContentHash(unsafeSourceFile)); + readCache.put(HashCacheTestHelper.contentHashKey(sanitizerSourceFile), FileHashingUtils.inputFileContentHash(sanitizerSourceFile)); + + verifier.verifyNoIssues(); verify(check, times(3)).scanWithoutParsing(any()); verify(check, times(3)).leaveFile(any()); @@ -241,7 +279,7 @@ void scanWithoutParsing_returns_false_when_cached_data_is_corrupted() { "Cached entry is unreadable for rule java:S5693 on file " + safeSourceFile ); - assertThat(writeCache.getData()).containsExactlyInAnyOrderEntriesOf(expectedFinalCacheState); + assertThat(writeCache.getData()).containsAllEntriesOf(expectedFinalCacheState); } @Test diff --git a/java-checks/src/test/java/org/sonar/java/checks/spring/SpringBeansShouldBeAccessibleCheckTest.java b/java-checks/src/test/java/org/sonar/java/checks/spring/SpringBeansShouldBeAccessibleCheckTest.java index 0e30aab48da..e14198d51c2 100644 --- a/java-checks/src/test/java/org/sonar/java/checks/spring/SpringBeansShouldBeAccessibleCheckTest.java +++ b/java-checks/src/test/java/org/sonar/java/checks/spring/SpringBeansShouldBeAccessibleCheckTest.java @@ -19,8 +19,10 @@ */ package org.sonar.java.checks.spring; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -33,6 +35,8 @@ import org.sonar.api.utils.log.LogTesterJUnit5; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.java.AnalysisException; +import org.sonar.java.caching.FileHashingUtils; +import org.sonar.java.checks.helpers.HashCacheTestHelper; import org.sonar.java.checks.verifier.CheckVerifier; import org.sonar.java.checks.verifier.internal.InternalReadCache; import org.sonar.java.checks.verifier.internal.InternalWriteCache; @@ -142,7 +146,7 @@ void testSpringBootApplicationWithAnnotation() { } @Test - void caching() { + void caching() throws NoSuchAlgorithmException, IOException { var unchangedFiles = Stream.of( "app/SpringBootApp1.java", "fourthApp/SpringBootApp4.java" @@ -155,22 +159,29 @@ void caching() { "Ko/Ko.java" ).map(path -> mainCodeSourcesPath(BASE_PATH + "springBootApplication/" + path)).collect(Collectors.toList()); + ReadCache existingReadCache = HashCacheTestHelper.internalReadCacheFromFiles(unchangedFiles); + writeCache.bind(existingReadCache); var check = spy(new SpringBeansShouldBeAccessibleCheck()); verifier .addFiles(InputFile.Status.SAME, unchangedFiles) .addFiles(InputFile.Status.CHANGED, changedFiles) .withCheck(check) + .withCache(existingReadCache, writeCache) .verifyIssues(); verify(check, times(15)).visitNode(any()); verify(check, times(2)).scanWithoutParsing(any()); assertThat(writeCache.getData()) - .hasSize(7); + .hasSizeGreaterThanOrEqualTo(7); check = spy(new SpringBeansShouldBeAccessibleCheck()); var populatedReadCache = new InternalReadCache().putAll(writeCache); + for(String changedFile : changedFiles) { + populatedReadCache.put(HashCacheTestHelper.contentHashKey(changedFile), + HashCacheTestHelper.getSlightlyDifferentContentHash(changedFile)); + } var finalWriteCache = new InternalWriteCache().bind(populatedReadCache); CheckVerifier.newVerifier() .withCache(populatedReadCache, finalWriteCache) @@ -182,23 +193,28 @@ void caching() { verify(check, times(12)).visitNode(any()); verify(check, times(2)).scanWithoutParsing(any()); assertThat(finalWriteCache.getData()) - .hasSize(7) + .hasSizeGreaterThanOrEqualTo(7) .containsExactlyEntriesOf(writeCache.getData()); } @Test - void cache_deserialization_throws_IOException() throws IOException { + void cache_deserialization_throws_IOException() throws IOException, NoSuchAlgorithmException { var inputStream = mock(InputStream.class); doThrow(new IOException()).when(inputStream).readAllBytes(); - var readCache = mock(ReadCache.class); - doReturn(inputStream).when(readCache).read(any()); - doReturn(true).when(readCache).contains(any()); + var localReadCache = mock(ReadCache.class); + + String filePath = mainCodeSourcesPath(BASE_PATH + "springBootApplication/app/SpringBootApp1.java"); + InputFile cachedFile = HashCacheTestHelper.inputFileFromPath(filePath); + byte[] cachedHash = FileHashingUtils.inputFileContentHash(cachedFile); + + doReturn(inputStream).when(localReadCache).read("java:S4605:targeted:" + cachedFile.key()); + doReturn(true).when(localReadCache).contains(any()); + doReturn(new ByteArrayInputStream(cachedHash)) + .when(localReadCache).read("java:contentHash:MD5:" + cachedFile.key()); var verifier = CheckVerifier.newVerifier() - .withCache(readCache, writeCache) - .addFiles(InputFile.Status.SAME, - mainCodeSourcesPath(BASE_PATH + "springBootApplication/app/SpringBootApp1.java") - ) + .withCache(localReadCache, new InternalWriteCache().bind(localReadCache)) + .addFiles(InputFile.Status.SAME, filePath) .withCheck(new SpringBeansShouldBeAccessibleCheck()); assertThatThrownBy(verifier::verifyNoIssues) @@ -224,13 +240,14 @@ void write_cache_multiple_writes() { } @Test - void emptyCache() { + void emptyCache() throws NoSuchAlgorithmException, IOException { logTester.setLevel(LoggerLevel.TRACE); + String filePath = mainCodeSourcesPath(BASE_PATH + "springBootApplication/app/SpringBootApp1.java"); + ReadCache populatedReadCache = HashCacheTestHelper.internalReadCacheFromFile(filePath); verifier - .addFiles(InputFile.Status.SAME, - mainCodeSourcesPath(BASE_PATH + "springBootApplication/app/SpringBootApp1.java") - ) + .addFiles(InputFile.Status.SAME, filePath) .withCheck(new SpringBeansShouldBeAccessibleCheck()) + .withCache(populatedReadCache, new InternalWriteCache().bind(populatedReadCache)) .verifyNoIssues(); assertThat(logTester.logs(LoggerLevel.TRACE).stream().filter( diff --git a/java-frontend/src/main/java/org/sonar/java/SonarComponents.java b/java-frontend/src/main/java/org/sonar/java/SonarComponents.java index cf06f9a2462..81ce21ab184 100644 --- a/java-frontend/src/main/java/org/sonar/java/SonarComponents.java +++ b/java-frontend/src/main/java/org/sonar/java/SonarComponents.java @@ -20,28 +20,6 @@ package org.sonar.java; import com.sonar.sslr.api.RecognitionException; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.LongSupplier; -import java.util.function.UnaryOperator; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; import org.sonar.api.SonarProduct; import org.sonar.api.batch.ScannerSide; import org.sonar.api.batch.bootstrap.ProjectDefinition; @@ -61,6 +39,7 @@ import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.java.annotations.VisibleForTesting; +import org.sonar.java.caching.ContentHashCache; import org.sonar.java.classpath.ClasspathForMain; import org.sonar.java.classpath.ClasspathForTest; import org.sonar.java.exceptions.ApiMismatchException; @@ -75,6 +54,29 @@ import org.sonarsource.api.sonarlint.SonarLintSide; import org.sonarsource.sonarlint.plugin.api.SonarLintRuntime; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.LongSupplier; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + @ScannerSide @SonarLintSide public class SonarComponents { @@ -374,6 +376,7 @@ public boolean isAutoScanCheckFiltering() { /** * Returns the batch mode size as read from configuration, in Kilo Bytes. If not value can be found, compute dynamically an ideal value. + * * @return the batch mode size or a default value of -1L. */ public long getBatchModeSizeInKB() { @@ -403,6 +406,7 @@ public File projectLevelWorkDir() { /** * Returns an OS-independent key that should identify the module within the project + * * @return A key representing the module */ public String getModuleKey() { @@ -452,6 +456,7 @@ public boolean canSkipUnchangedFiles() throws ApiMismatchException { public boolean fileCanBeSkipped(InputFile inputFile) { + var contentHashCache = new ContentHashCache(context); if (inputFile instanceof GeneratedFile) { // Generated files should not be skipped as we cannot assess the change status of the source file return false; @@ -476,9 +481,14 @@ public boolean fileCanBeSkipped(InputFile inputFile) { ); alreadyLoggedSkipStatus = true; } + contentHashCache.writeToCache(inputFile); + return false; + } + if (!canSkipInContext) { + contentHashCache.writeToCache(inputFile); return false; } - return canSkipInContext && inputFile.status() == InputFile.Status.SAME; + return contentHashCache.hasSameHashCached(inputFile); } public InputComponent project() { diff --git a/java-frontend/src/main/java/org/sonar/java/caching/ContentHashCache.java b/java-frontend/src/main/java/org/sonar/java/caching/ContentHashCache.java new file mode 100644 index 00000000000..874adcf61a8 --- /dev/null +++ b/java-frontend/src/main/java/org/sonar/java/caching/ContentHashCache.java @@ -0,0 +1,117 @@ +/* + * SonarQube Java + * Copyright (C) 2012-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.java.caching; + +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.sensor.SensorContext; +import org.sonar.api.batch.sensor.cache.ReadCache; +import org.sonar.api.batch.sensor.cache.WriteCache; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class ContentHashCache { + + private static final Logger LOG = Loggers.get(ContentHashCache.class); + private static final String CONTENT_HASH_KEY = String.format("java:contentHash:%s:", FileHashingUtils.HASH_ALGORITHM); + private static final String HASH_COMPUTE_FAIL_MSG = "Failed to compute content hash for file %s"; + + private ReadCache readCache; + private WriteCache writeCache; + private final boolean enabled; + + public ContentHashCache(SensorContext context) { + CacheContextImpl cacheContext = CacheContextImpl.of(context); + enabled = cacheContext.isCacheEnabled(); + + if (enabled) { + readCache = context.previousCache(); + writeCache = context.nextCache(); + } + } + + public boolean hasSameHashCached(InputFile inputFile) { + if (!enabled) { + if (inputFile.status() == InputFile.Status.SAME) { + LOG.trace(() -> "Cache is disabled. File status is: " + inputFile.status() + ". File can be skipped."); + return true; + } + LOG.trace(() -> "Cache is disabled. File status is: " + inputFile.status() + ". File can't be skipped."); + return false; + } + String cacheKey = getCacheKey(inputFile); + try { + LOG.trace(() -> "Reading cache for the file " + inputFile.key()); + byte[] cachedHash = readCache.read(cacheKey).readAllBytes(); + byte[] fileHash = FileHashingUtils.inputFileContentHash(inputFile); + boolean isHashEqual = MessageDigest.isEqual(fileHash, cachedHash); + if (isHashEqual) { + copyFromPrevious(inputFile); + } else { + writeToCache(inputFile); + } + return isHashEqual; + } catch (IllegalArgumentException e) { + LOG.warn(String.format("Could not find key %s in the cache", cacheKey)); + writeToCache(inputFile); + } catch (IOException | NoSuchAlgorithmException e) { + LOG.warn(String.format(HASH_COMPUTE_FAIL_MSG, inputFile.key())); + } + return false; + } + + public boolean contains(InputFile inputFile) { + if (!enabled) { + LOG.trace(() -> "Cannot lookup cached hashes when the cache is disabled (" + inputFile.key() + ")."); + return false; + } + return readCache.contains(getCacheKey(inputFile)); + } + + public boolean writeToCache(InputFile inputFile) { + if (!enabled) { + LOG.trace(() -> "Cannot write hashes to the cache when the cache is disabled (" + inputFile.key() + ")."); + return false; + } + LOG.trace(() -> "Writing to the cache for file " + inputFile.key()); + String cacheKey = getCacheKey(inputFile); + try { + writeCache.write(cacheKey, FileHashingUtils.inputFileContentHash(inputFile)); + return true; + } catch (IllegalArgumentException e) { + LOG.warn(String.format("Tried to write multiple times to cache key %s. Ignoring writes after the first.", cacheKey)); + } catch (IOException | NoSuchAlgorithmException e) { + LOG.warn(String.format(HASH_COMPUTE_FAIL_MSG, inputFile.key())); + } + return false; + } + + private void copyFromPrevious(InputFile inputFile) { + LOG.trace(() -> "Copying cache from previous for file " + inputFile.key()); + writeCache.copyFromPrevious(getCacheKey(inputFile)); + } + + private static String getCacheKey(InputFile inputFile) { + return CONTENT_HASH_KEY + inputFile.key(); + } +} diff --git a/java-frontend/src/main/java/org/sonar/java/caching/FileHashingUtils.java b/java-frontend/src/main/java/org/sonar/java/caching/FileHashingUtils.java new file mode 100644 index 00000000000..8dbbe8c2c77 --- /dev/null +++ b/java-frontend/src/main/java/org/sonar/java/caching/FileHashingUtils.java @@ -0,0 +1,53 @@ +/* + * SonarQube Java + * Copyright (C) 2012-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.java.caching; + +import org.sonar.api.batch.fs.InputFile; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class FileHashingUtils { + + private FileHashingUtils() { + } + + public static final String HASH_ALGORITHM = "MD5"; + + public static byte[] inputFileContentHash(InputFile inputFile) throws IOException, NoSuchAlgorithmException { + byte[] contentBytes = inputFile.contents().getBytes(StandardCharsets.UTF_8); + MessageDigest messageDigest = MessageDigest.getInstance(HASH_ALGORITHM); + return messageDigest.digest(contentBytes); + } + + public static byte[] inputFileContentHash(String filepath) throws IOException, NoSuchAlgorithmException { + File file = new File(filepath); + String contents = new String(Files.readAllBytes(file.toPath()), UTF_8); + MessageDigest messageDigest = MessageDigest.getInstance(HASH_ALGORITHM); + return messageDigest.digest(contents.getBytes(UTF_8)); + } + +} diff --git a/java-frontend/src/test/java/org/sonar/java/SonarComponentsTest.java b/java-frontend/src/test/java/org/sonar/java/SonarComponentsTest.java index f47c9d372c8..5c450e538a5 100644 --- a/java-frontend/src/test/java/org/sonar/java/SonarComponentsTest.java +++ b/java-frontend/src/test/java/org/sonar/java/SonarComponentsTest.java @@ -21,6 +21,7 @@ import com.sonar.sslr.api.RecognitionException; import java.io.File; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -728,52 +729,44 @@ void fileCanBeSkipped_returns_false_when_the_file_is_a_generated_file() throws A checkFactory ) ); + SensorContext contextMock = mock(SensorContext.class); + sonarComponents.setSensorContext(contextMock); + InputFile inputFile = new GeneratedFile(Path.of("non-existing-generated-file.java")); assertThat(sonarComponents.fileCanBeSkipped(inputFile)).isFalse(); } @Test - void fileCanBeSkipped_always_returns_false_when_skipUnchangedFiles_is_false() throws ApiMismatchException { + void fileCanBeSkipped_always_returns_false_when_skipUnchangedFiles_is_false() throws ApiMismatchException, IOException { SonarComponents sonarComponents = mock(SonarComponents.class, CALLS_REAL_METHODS); + SensorContext contextMock = mock(SensorContext.class); + sonarComponents.setSensorContext(contextMock); + when(sonarComponents.canSkipUnchangedFiles()).thenReturn(false); InputFile inputFile = mock(InputFile.class); assertThat(sonarComponents.fileCanBeSkipped(inputFile)).isFalse(); - verify(inputFile, times(0)).status(); } @Test - void fileCanBeSkipped_always_returns_true_when_skipUnchangedFiles_is_true_and_file_status_is_same() throws ApiMismatchException { + void fileCanBeSkipped_returns_false_when_inputFileStatusIsDifferentFromSame() throws ApiMismatchException { SonarComponents sonarComponents = mock(SonarComponents.class, CALLS_REAL_METHODS); - when(sonarComponents.canSkipUnchangedFiles()).thenReturn(true); - - InputFile inputFile = mock(InputFile.class); - when(inputFile.status()).thenReturn(InputFile.Status.SAME); - - assertThat(sonarComponents.fileCanBeSkipped(inputFile)).isTrue(); - } + SensorContext contextMock = mock(SensorContext.class); + sonarComponents.setSensorContext(contextMock); - @Test - void fileCanBeSkipped_always_returns_false_when_skipUnchangedFiles_is_true_and_file_status_is_not_same() throws ApiMismatchException { - SonarComponents sonarComponents = mock(SonarComponents.class, CALLS_REAL_METHODS); when(sonarComponents.canSkipUnchangedFiles()).thenReturn(true); - InputFile inputFile = mock(InputFile.class); - when(inputFile.status()).thenReturn(InputFile.Status.ADDED); - - assertThat(sonarComponents.fileCanBeSkipped(inputFile)).isFalse(); - when(inputFile.status()).thenReturn(InputFile.Status.CHANGED); - assertThat(sonarComponents.fileCanBeSkipped(inputFile)).isFalse(); - } @Test void fileCanBeSkipped_returns_false_when_canSkipUnchangedFile_isFalse() throws ApiMismatchException { SonarComponents sonarComponents = mock(SonarComponents.class, CALLS_REAL_METHODS); + SensorContext contextMock = mock(SensorContext.class); + sonarComponents.setSensorContext(contextMock); ApiMismatchException apiMismatchException = new ApiMismatchException(new NoSuchMethodError("API version mismatch :-(")); doThrow(apiMismatchException).when(sonarComponents).canSkipUnchangedFiles(); @@ -803,9 +796,12 @@ private static Stream fileCanBeSkipped_only_logs_on_first_call_input( @ParameterizedTest @MethodSource("fileCanBeSkipped_only_logs_on_first_call_input") - void fileCanBeSkipped_only_logs_on_the_first_call(SonarComponents sonarComponents, InputFile inputFile, String logMessage) throws ApiMismatchException { + void fileCanBeSkipped_only_logs_on_the_first_call(SonarComponents sonarComponents, InputFile inputFile, String logMessage) throws IOException { assertThat(logTester.getLogs(LoggerLevel.INFO)).isNull(); + SensorContext contextMock = mock(SensorContext.class); + sonarComponents.setSensorContext(contextMock); + when(inputFile.contents()).thenReturn(""); sonarComponents.fileCanBeSkipped(inputFile); List logs = logTester.getLogs(LoggerLevel.INFO); assertThat(logs).hasSize(1); diff --git a/java-frontend/src/test/java/org/sonar/java/caching/ContentHashCacheTest.java b/java-frontend/src/test/java/org/sonar/java/caching/ContentHashCacheTest.java new file mode 100644 index 00000000000..6a020c5c356 --- /dev/null +++ b/java-frontend/src/test/java/org/sonar/java/caching/ContentHashCacheTest.java @@ -0,0 +1,240 @@ +/* + * SonarQube Java + * Copyright (C) 2012-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.java.caching; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.sensor.cache.ReadCache; +import org.sonar.api.batch.sensor.cache.WriteCache; +import org.sonar.api.batch.sensor.internal.SensorContextTester; +import org.sonar.api.utils.log.LogAndArguments; +import org.sonar.api.utils.log.LogTesterJUnit5; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.java.TestUtils; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class ContentHashCacheTest { + + @RegisterExtension + LogTesterJUnit5 logTester = new LogTesterJUnit5(); + + private final File file = new File("src/test/files/api/JavaFileScannerContext.java"); + private final InputFile inputFile = TestUtils.inputFile(file.getAbsoluteFile().getAbsolutePath(), file, InputFile.Type.TEST); + + @Test + void hasSameHashCached_returns_true_when_content_hash_file_is_in_read_cache() throws IOException, NoSuchAlgorithmException { + logTester.setLevel(LoggerLevel.TRACE); + ContentHashCache contentHashCache = new ContentHashCache(getSensorContextTester()); + Assertions.assertTrue(contentHashCache.hasSameHashCached(inputFile)); + + List logs = logTester.getLogs(LoggerLevel.TRACE).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); + assertThat(logs). + contains("Reading cache for the file " + inputFile.key(), + "Copying cache from previous for file " + inputFile.key()); + } + + @Test + void hasSameHashCached_returns_false_when_content_hash_file_is_not_in_read_cache() { + logTester.setLevel(LoggerLevel.TRACE); + ContentHashCache contentHashCache = new ContentHashCache(getSensorContextTesterWithEmptyCache(true)); + Assertions.assertFalse(contentHashCache.hasSameHashCached(inputFile)); + + List logs = logTester.getLogs(LoggerLevel.TRACE).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); + assertThat(logs). + contains("Reading cache for the file " + inputFile.key(), + "Writing to the cache for file " + inputFile.key()); + } + + @Test + void hasSameHashCached_returns_false_when_cache_is_disabled_and_input_file_status_is_same() { + logTester.setLevel(LoggerLevel.TRACE); + InputFile inputFile1 = mock(InputFile.class); + when(inputFile1.status()).thenReturn(InputFile.Status.SAME); + when(inputFile1.key()).thenReturn("key"); + ContentHashCache contentHashCache = new ContentHashCache(getSensorContextTesterWithEmptyCache(false)); + Assertions.assertTrue(contentHashCache.hasSameHashCached(inputFile1)); + + List logs = logTester.getLogs(LoggerLevel.TRACE).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); + assertThat(logs). + contains("Cache is disabled. File status is: " + inputFile1.status() + ". File can be skipped."); + } + + @Test + void hasSameHashCached_returns_false_cache_is_disabled_and_input_file_status_is_changed() { + logTester.setLevel(LoggerLevel.TRACE); + InputFile inputFile1 = mock(InputFile.class); + when(inputFile1.status()).thenReturn(InputFile.Status.CHANGED); + ContentHashCache contentHashCache = new ContentHashCache(getSensorContextTesterWithEmptyCache(false)); + Assertions.assertFalse(contentHashCache.hasSameHashCached(inputFile1)); + + List logs = logTester.getLogs(LoggerLevel.TRACE).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); + assertThat(logs). + contains("Cache is disabled. File status is: " + inputFile1.status() + ". File can't be skipped."); + } + + @Test + void hasSameHashCached_writesToCache_when_key_is_not_present() { + logTester.setLevel(LoggerLevel.TRACE); + ContentHashCache contentHashCache = new ContentHashCache(getSensorContextTesterWithEmptyCache(true)); + contentHashCache.hasSameHashCached(inputFile); + Assertions.assertTrue(contentHashCache.writeToCache(inputFile)); + + List logs = logTester.getLogs(LoggerLevel.TRACE).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); + assertThat(logs). + contains("Writing to the cache for file " + inputFile.key()); + } + + @Test + void hasSameHashCached_returns_false_when_content_hash_file_is_not_same_as_one_in_cache() { + logTester.setLevel(LoggerLevel.TRACE); + SensorContextTester sensorContext = SensorContextTester.create(file.getAbsoluteFile()); + sensorContext.setCacheEnabled(true); + ReadCache readCache = mock(ReadCache.class); + when(readCache.read("java:contentHash:MD5:" + inputFile.key())).thenReturn(new ByteArrayInputStream("Dummy content hash".getBytes())); + when(readCache.contains("java:contentHash:MD5:" + inputFile.key())).thenReturn(true); + WriteCache writeCache = mock(WriteCache.class); + sensorContext.setPreviousCache(readCache); + sensorContext.setNextCache(writeCache); + ContentHashCache contentHashCache = new ContentHashCache(sensorContext); + Assertions.assertFalse(contentHashCache.hasSameHashCached(inputFile)); + + List logs = logTester.getLogs(LoggerLevel.TRACE).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); + assertThat(logs). + contains("Reading cache for the file " + inputFile.key(), + "Writing to the cache for file " + inputFile.key()); + } + + @Test + void hasSameHashCached_returns_false_when_FileHashingUtils_throws_exception() throws IOException { + logTester.setLevel(LoggerLevel.WARN); + SensorContextTester sensorContext = SensorContextTester.create(file.getAbsoluteFile()); + sensorContext.setCacheEnabled(true); + ReadCache readCache = mock(ReadCache.class); + // Mocking input file to throw IOException because + // mocking static method requires mockito-inline, which currently breaks the tests. + InputFile inputFile1 = mock(InputFile.class); + when(inputFile1.key()).thenReturn("key"); + when(readCache.read("java:contentHash:MD5:" + inputFile1.key())).thenReturn(new ByteArrayInputStream("string".getBytes())); + when(readCache.contains("java:contentHash:MD5:" + inputFile1.key())).thenReturn(true); + WriteCache writeCache = mock(WriteCache.class); + sensorContext.setPreviousCache(readCache); + sensorContext.setNextCache(writeCache); + when(inputFile1.contents()).thenThrow(new IOException()); + ContentHashCache contentHashCache = new ContentHashCache(sensorContext); + Assertions.assertFalse(contentHashCache.hasSameHashCached(inputFile1)); + + List logs = logTester.getLogs(LoggerLevel.WARN).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); + assertThat(logs). + contains("Failed to compute content hash for file " + inputFile1.key()); + } + + @Test + void contains_returns_true_when_file_is_in_cache() throws IOException, NoSuchAlgorithmException { + ContentHashCache contentHashCache = new ContentHashCache(getSensorContextTester()); + Assertions.assertTrue(contentHashCache.contains(inputFile)); + } + + @Test + void contains_returns_false_when_file_is_not_in_cache() { + ContentHashCache contentHashCache = new ContentHashCache(getSensorContextTesterWithEmptyCache(true)); + Assertions.assertFalse(contentHashCache.contains(inputFile)); + } + + @Test + void contains_returns_false_when_cache_is_disabled() { + ContentHashCache contentHashCache = new ContentHashCache(getSensorContextTesterWithEmptyCache(false)); + Assertions.assertFalse(contentHashCache.contains(inputFile)); + } + + @Test + void writeToCache_returns_false_when_writing_to_cache_throws_exception() throws IOException, NoSuchAlgorithmException { + logTester.setLevel(LoggerLevel.WARN); + SensorContextTester sensorContext = SensorContextTester.create(file.getAbsoluteFile()); + sensorContext.setCacheEnabled(true); + WriteCache writeCache = mock(WriteCache.class); + sensorContext.setNextCache(writeCache); + doThrow(new IllegalArgumentException()).when(writeCache).write("java:contentHash:MD5:" + inputFile.key(), FileHashingUtils.inputFileContentHash(file.getPath())); + ContentHashCache contentHashCache = new ContentHashCache(sensorContext); + Assertions.assertFalse(contentHashCache.writeToCache(inputFile)); + + List logs = logTester.getLogs(LoggerLevel.WARN).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); + assertThat(logs). + contains("Tried to write multiple times to cache key java:contentHash:MD5:" + inputFile.key() + ". Ignoring writes after the first."); + } + + @Test + void writeToCache_returns_false_when_FileHashingUtils_throws_exception() throws IOException { + logTester.setLevel(LoggerLevel.WARN); + SensorContextTester sensorContext = SensorContextTester.create(file.getAbsoluteFile()); + sensorContext.setCacheEnabled(true); + WriteCache writeCache = mock(WriteCache.class); + sensorContext.setNextCache(writeCache); + // Mocking input file to throw IOException because + // mocking static method requires mockito-inline, which currently breaks the tests. + InputFile inputFile1 = mock(InputFile.class); + when(inputFile1.key()).thenReturn("key"); + when(inputFile1.contents()).thenThrow(new IOException()); + ContentHashCache contentHashCache = new ContentHashCache(sensorContext); + Assertions.assertFalse(contentHashCache.writeToCache(inputFile1)); + + List logs = logTester.getLogs(LoggerLevel.WARN).stream().map(LogAndArguments::getFormattedMsg).collect(Collectors.toList()); + assertThat(logs). + contains("Failed to compute content hash for file " + inputFile1.key()); + } + + private SensorContextTester getSensorContextTesterWithEmptyCache(boolean isCacheEnabled) { + SensorContextTester sensorContext = SensorContextTester.create(file.getAbsoluteFile()); + sensorContext.setCacheEnabled(isCacheEnabled); + ReadCache readCache = mock(ReadCache.class); + when(readCache.read("java:contentHash:MD5:" + inputFile.key())).thenThrow(new IllegalArgumentException()); + WriteCache writeCache = mock(WriteCache.class); + sensorContext.setPreviousCache(readCache); + sensorContext.setNextCache(writeCache); + + return sensorContext; + } + + private SensorContextTester getSensorContextTester() throws IOException, NoSuchAlgorithmException { + SensorContextTester sensorContext = SensorContextTester.create(file.getAbsoluteFile()); + sensorContext.setCacheEnabled(true); + ReadCache readCache = mock(ReadCache.class); + when(readCache.read("java:contentHash:MD5:" + inputFile.key())).thenReturn(new ByteArrayInputStream(FileHashingUtils.inputFileContentHash(inputFile))); + when(readCache.contains("java:contentHash:MD5:" + inputFile.key())).thenReturn(true); + WriteCache writeCache = mock(WriteCache.class); + sensorContext.setPreviousCache(readCache); + sensorContext.setNextCache(writeCache); + + return sensorContext; + } + +} diff --git a/java-frontend/src/test/java/org/sonar/java/model/VisitorsBridgeTest.java b/java-frontend/src/test/java/org/sonar/java/model/VisitorsBridgeTest.java index 74b4cff5264..72c1a1c45de 100644 --- a/java-frontend/src/test/java/org/sonar/java/model/VisitorsBridgeTest.java +++ b/java-frontend/src/test/java/org/sonar/java/model/VisitorsBridgeTest.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.internal.SensorContextTester; import org.sonar.api.config.internal.MapSettings; import org.sonar.api.utils.log.LogAndArguments; @@ -497,6 +498,9 @@ void scanWithoutParsing_returns_false_when_the_file_is_a_generated_file() throws InputFile inputFile = new GeneratedFile(Path.of("non-existing-generated-file.java")); SonarComponents sonarComponents = spy(new SonarComponents(null, null, null, null, null)); + SensorContext contextMock = mock(SensorContext.class); + sonarComponents.setSensorContext(contextMock); + doReturn(true).when(sonarComponents).canSkipUnchangedFiles(); VisitorsBridge visitorsBridge = new VisitorsBridge( Collections.singletonList(new EndOfAnalysisVisitor()),