[No-jira] Extract autoscan test to its own module (#4277)

The goal is to shorten the feedback for developers loop by: * Decoupling autoscan from the other ruling tests both in the sources and at runtime * Skipping the compilation of the test sources from the test itself * Only running the test on Linux as running it on WIndows in CI has shown little value so far
SonarSource · Nov 17, 2023 · 1e6fff7 · 1e6fff7
1 parent e1bafe1
commit 1e6fff7
Show file tree

Hide file tree

Showing 501 changed files with 3,234 additions and 2 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -186,9 +186,28 @@ ruling_win_task:
     - cd its/ruling
     - mvn package -Pit-ruling -Dsonar.runtimeVersion=LATEST_RELEASE[9.9] -Dmaven.test.redirectTestOutputToFile=false -B -e -V -Dparallel=methods -DuseUnlimitedThreads=true
   cleanup_before_cache_script: cleanup_maven_repository
+
+autoscan_task:
+  depends_on:
+    - build
+  <<: *ONLY_SONARSOURCE_QA
+  eks_container:
+    <<: *CONTAINER_DEFINITION
+    image: ${CIRRUS_AWS_ACCOUNT}.dkr.ecr.eu-central-1.amazonaws.com/base:j17-latest
+    cpu: 14
+    memory: 6G
+  maven_cache:
+    folder: ${CIRRUS_WORKING_DIR}/.m2/repository
+  autoscan_script:
+    - source cirrus-env QA
+    - source set_maven_build_version $BUILD_NUMBER
+    - mvn clean compile --projects java-checks-test-sources --also-make-dependents
+    - cd its/autoscan
+    - mvn clean package --batch-mode --errors --show-version --activate-profiles it-autoscan -Dsonar.runtimeVersion=LATEST_RELEASE[9.9] -Dmaven.test.redirectTestOutputToFile=false -Dparallel=methods -DuseUnlimitedThreads=true
+  cleanup_before_cache_script: cleanup_maven_repository
   on_failure:
     actual_artifacts:
-      path: "${CIRRUS_WORKING_DIR}/its/ruling/target/actual/**/*"
+      path: "${CIRRUS_WORKING_DIR}/its/autoscan/target/actual/**/*"
 
 promote_task:
   depends_on:
@@ -198,6 +217,7 @@ promote_task:
     - ruling_win
     - plugin_qa
     - ws_scan
+    - autoscan
   <<: *ONLY_SONARSOURCE_QA
   eks_container:
     <<: *CONTAINER_DEFINITION

diff --git a/its/autoscan/README.md b/its/autoscan/README.md
@@ -0,0 +1,30 @@
+# Autoscan
+
+The tests in this module are designed to detect differences between the issues the Java analyzer can find with and without compiled code.
+The goal here is to spot and fix the potential FPs, and verify the expected FNs between that would show up in [SonarCloud's automatic analysis](https://docs.sonarcloud.io/advanced-setup/automatic-analysis/).
+
+## Testing
+
+### Compiling the sources
+
+Make sure that the `java-checks-tests-sources` module has been compiled (ie: the .class files in `java-checks-tests-sources/target/` are up to date).
+
+In doubt, go to the top-level of the project and run:
+```shell
+# cd ../../ or back to the top level of sonar-java
+mvn clean compile --projects java-checks-test-sources --also-make-dependents
+```
+
+## Running the tests
+
+To run the tests from this folder, run:
+```shell
+mvn clean package --batch-mode --errors --show-version \
+   --activate-profiles it-autoscan \
+  -Dsonar.runtimeVersion=LATEST_RELEASE[9.9]
+```
+
+
+## Updating the expected results
+
+The expected results are listed in [autoscan-diff-by-rules.json](src%2Ftest%2Fresources%2Fautoscan%2Fautoscan-diff-by-rules.json). 
diff --git a/its/autoscan/pom.xml b/its/autoscan/pom.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.sonarsource.java</groupId>
+    <artifactId>java-its</artifactId>
+    <version>7.29.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>it-java-autoscan</artifactId>
+
+  <name>SonarQube Java :: ITs :: Autoscan</name>
+  <inceptionYear>2013</inceptionYear>
+
+  <properties>
+    <surefire.argLine>-server</surefire.argLine>
+    <maven.test.redirectTestOutputToFile>false</maven.test.redirectTestOutputToFile>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.code.gson</groupId>
+      <artifactId>gson</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.sonarsource.orchestrator</groupId>
+      <artifactId>sonar-orchestrator</artifactId>
+      <version>${orchestrator.version}</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.oracle</groupId>
+          <artifactId>ojdbc6</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.sonarsource.sonarqube</groupId>
+      <artifactId>sonar-ws</artifactId>
+      <version>${sonar.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <profiles>
+    <profile>
+      <id>it-autoscan</id>
+      <properties>
+        <skipTests>false</skipTests>
+      </properties>
+    </profile>
+  </profiles>
+</project>
diff --git a/.../java/org/sonar/java/it/AutoScanTest.java → .../java/org/sonar/java/it/AutoScanTest.java b/.../java/org/sonar/java/it/AutoScanTest.java → .../java/org/sonar/java/it/AutoScanTest.java
@@ -97,7 +97,7 @@ public void javaCheckTestSources() throws Exception {
 
     MavenBuild mavenBuild = MavenBuild.create()
       .setPom(FileLocation.of(PROJECT_LOCATION + "pom.xml").getFile().getCanonicalFile())
-      .setCleanPackageSonarGoals()
+      .addSonarGoal()
       .addArgument("-DskipTests")
       .addArgument("-Panalyze-tests")
       .setProperty("sonar.projectKey", PROJECT_KEY)

diff --git a/its/autoscan/src/test/java/org/sonar/java/it/ProfileGenerator.java b/its/autoscan/src/test/java/org/sonar/java/it/ProfileGenerator.java
@@ -0,0 +1,158 @@
+/*
+ * SonarQube Java
+ * Copyright (C) 2013-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.java.it;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Files;
+import com.sonar.orchestrator.Orchestrator;
+import com.sonar.orchestrator.container.Server;
+import com.sonar.orchestrator.locator.FileLocation;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonarqube.ws.Rules;
+import org.sonarqube.ws.client.HttpConnector;
+import org.sonarqube.ws.client.WsClient;
+import org.sonarqube.ws.client.WsClientFactories;
+import org.sonarqube.ws.client.rules.SearchRequest;
+
+public class ProfileGenerator {
+  private static final String LANGUAGE = "java";
+  private static final String REPOSITORY_KEY = "java";
+  // note that 500 is the maximum allowed by SQ rules/api/search
+  private static final int NUMBER_RULES_BY_PAGE = 500;
+
+  private static final Logger LOG = LoggerFactory.getLogger(ProfileGenerator.class);
+
+  static void generate(Orchestrator orchestrator, ImmutableMap<String, ImmutableMap<String, String>> rulesParameters,
+    Set<String> excluded, Set<String> subsetOfEnabledRules, Set<String> activatedRuleKeys) {
+    generate(orchestrator, null, rulesParameters, excluded, subsetOfEnabledRules, activatedRuleKeys);
+  }
+
+  /**
+   * @return the list of enabled rule keys for the given profile
+   */
+  static void generate(Orchestrator orchestrator, @Nullable String qualityProfile, ImmutableMap<String, ImmutableMap<String, String>> rulesParameters,
+    Set<String> excluded, Set<String> subsetOfEnabledRules, Set<String> activatedRuleKeys) {
+    try {
+      LOG.info("Generating profile containing all the rules");
+      StringBuilder sb = new StringBuilder()
+        .append("<profile>")
+        .append("<name>rules</name>")
+        .append("<language>").append(LANGUAGE).append("</language>")
+        .append("<rules>");
+
+      for (String key : getRuleKeys(orchestrator, qualityProfile)) {
+        if (excluded.contains(key) || (!subsetOfEnabledRules.isEmpty() && !subsetOfEnabledRules.contains(key))) {
+          continue;
+        }
+        activatedRuleKeys.add(key);
+        sb.append("<rule>")
+          .append("<repositoryKey>").append(REPOSITORY_KEY).append("</repositoryKey>")
+          .append("<key>").append(key).append("</key>")
+          .append("<priority>INFO</priority>");
+        if (rulesParameters.containsKey(key)) {
+          sb.append("<parameters>");
+          for (Map.Entry<String, String> parameter : rulesParameters.get(key).entrySet()) {
+            sb.append("<parameter>")
+              .append("<key>").append(parameter.getKey()).append("</key>")
+              .append("<value>").append(parameter.getValue()).append("</value>")
+              .append("</parameter>");
+          }
+          sb.append("</parameters>");
+        }
+        sb.append("</rule>");
+      }
+
+      sb.append("</rules>")
+        .append("</profile>");
+
+      File file = File.createTempFile("profile", ".xml");
+      Files.asCharSink(file, StandardCharsets.UTF_8).write(sb);
+      LOG.info("Restoring profile to SonarQube");
+      orchestrator.getServer().restoreProfile(FileLocation.of(file));
+      file.delete();
+    } catch (IOException e) {
+      throw new IllegalStateException(e);
+    }
+  }
+
+  private static List<String> getRuleKeys(Orchestrator orchestrator, @Nullable String qualityProfile) {
+    List<String> ruleKeys = new ArrayList<>();
+    // pages are 1-based
+    int currentPage = 1;
+
+    long totalNumberRules;
+    long collectedRulesNumber;
+    Optional<String> qualityProfileName = getQualityProfileName(orchestrator, qualityProfile);
+    do {
+      SearchRequest searchRequest = new SearchRequest()
+        .setLanguages(Collections.singletonList(LANGUAGE))
+        .setRepositories(Collections.singletonList(REPOSITORY_KEY))
+        .setP(Integer.toString(currentPage))
+        .setPs(Integer.toString(NUMBER_RULES_BY_PAGE));
+      qualityProfileName.ifPresent(qProfile -> searchRequest.setActivation("true").setQprofile(qProfile));
+
+      Rules.SearchResponse searchResponse = newAdminWsClient(orchestrator).rules().search(searchRequest);
+
+      searchResponse.getRulesList().stream()
+        .map(Rules.Rule::getKey)
+        .map(key -> key.split(":")[1])
+        .forEach(ruleKeys::add);
+
+      // update number of rules
+      collectedRulesNumber = ruleKeys.size();
+      totalNumberRules = searchResponse.getTotal();
+      LOG.info("Collected rule keys: {} / {}", collectedRulesNumber, totalNumberRules);
+      // prepare for next page
+      currentPage++;
+    } while (collectedRulesNumber != totalNumberRules);
+
+    return ruleKeys;
+  }
+
+  private static Optional<String> getQualityProfileName(Orchestrator orchestrator, @Nullable String qualityProfile) {
+    if (qualityProfile == null || qualityProfile.isEmpty()) {
+      return Optional.empty();
+    }
+    org.sonarqube.ws.client.qualityprofiles.SearchRequest request = new org.sonarqube.ws.client.qualityprofiles.SearchRequest().setLanguage(LANGUAGE);
+
+    return newAdminWsClient(orchestrator).qualityprofiles().search(request).getProfilesList().stream()
+      .filter(p -> qualityProfile.equalsIgnoreCase(p.getName()))
+      .map(p -> p.getKey())
+      .findFirst();
+  }
+
+  static WsClient newAdminWsClient(Orchestrator orchestrator) {
+    return WsClientFactories.getDefault().newClient(HttpConnector.newBuilder()
+      .credentials(Server.ADMIN_LOGIN, Server.ADMIN_PASSWORD)
+      .url(orchestrator.getServer().getUrl())
+      .build());
+  }
+}