License detection now shows 3 most probable licenses with match perct…

…ng (#37)

.gitignore

@@ -16,3 +16,4 @@ jshell.history
 nbactions.xml
 nb-configuration.xml
 /plugin.cargo/nbproject/
+/plugin.npm/nbproject/

LicenseDetector/src/main/java/com/phsyberdome/licensedetector/LicenseDetector.java

@@ -6,6 +6,7 @@
 import com.phsyberdome.common.utils.CLIHelper;
 import com.phsyberdome.common.utils.models.Pair;
 import com.phsyberdome.common.utils.FileUtil;
+import com.phsyberdome.common.utils.models.LicenseDetectionResult;
 import com.phsyberdome.common.utils.NetworkHelper;
 import java.io.File;
 import java.nio.file.Path;
@@ -96,13 +97,13 @@ private Map<String,Double> analyze(String licenseContent) {
         return queryResult;
     }
 
-    public Pair<String,String> detect(String path){
+    public LicenseDetectionResult detect(String path){
         // Check if it is valid url
 
         if(NetworkHelper.isValidURL(path)){
 
             Path rootPath = FileUtil.getFilePathFromURL(path,cloneLocation.toString());
-            Pair<String,String> result = filePathDetection(rootPath.toString());
+            LicenseDetectionResult result = filePathDetection(rootPath.toString());
             if(rootPath.toFile().exists()){
                 FileUtil.deleteDirectory(rootPath.toFile());
             }
@@ -114,8 +115,9 @@ public Pair<String,String> detect(String path){
     }
 
 
-    private Pair<String,String> filePathDetection(String path) {
+    private LicenseDetectionResult filePathDetection(String path) {
         Path pathToFile = searchLicenseFile(path);
+        var results = new LicenseDetectionResult();
         totalScanned++;
         if(pathToFile == null) {
             CLIHelper.updateCurrentLine("Couldn't get license file at "+path,Ansi.Color.RED);
@@ -124,19 +126,29 @@ private Pair<String,String> filePathDetection(String path) {
             // Maybe handover this responsiblity to the cloud service.
             pathToFile = searchReadmeFile(path);
             if(pathToFile == null){
-                return new Pair<>("null","null");
+                results.addProbableLicense(new Pair<>("null",0.0));
+                results.setAnalyzedContent("");
+                return results;
             }
             String content = FileUtil.readFile(pathToFile);
-            return new Pair<>("null",content);
+            results.addProbableLicense(new Pair<>("null",0.0));
+            results.setAnalyzedContent("");
+            return results;
         }
 
         String content = readLicense(pathToFile);
 
         Map<String,Double> res = analyze(content);
         if(res.isEmpty()) {
-            return new Pair<>("null","null");
+            results.addProbableLicense(new Pair<>("null",0.0));
+            results.setAnalyzedContent(content);
+            return results;
         }else{
-            return new Pair<>(res.entrySet().iterator().next().getKey(),content);
+            res.entrySet().forEach(possibility ->  {
+                results.addProbableLicense(new Pair<>(possibility.getKey(), possibility.getValue()));
+            });
+            results.setAnalyzedContent(content);
+            return results;
         }
     }
 

common-utils/src/main/java/com/phsyberdome/common/interfaces/LicenseDetectorInterface.java

@@ -2,6 +2,7 @@
 package com.phsyberdome.common.interfaces;
 
 import com.phsyberdome.common.utils.CLIHelper;
+import com.phsyberdome.common.utils.models.LicenseDetectionResult;
 import com.phsyberdome.common.utils.models.Pair;
 import org.fusesource.jansi.Ansi;
 
@@ -12,8 +13,10 @@
  */
 public interface LicenseDetectorInterface {
 
-    default public Pair<String,String> detect(String content) {
-        return new Pair<>("Detector not initialized!","1.0");
+    default public LicenseDetectionResult detect(String content) {
+        var result = new LicenseDetectionResult();
+        result.addProbableLicense(new Pair<>("No dependencies scanned!",1.0));
+        return result;
     }
 
     default public void printScanStats() {

common-utils/src/main/java/com/phsyberdome/common/utils/models/LicenseDetectionResult.java

@@ -0,0 +1,85 @@
+
+package com.phsyberdome.common.utils.models;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ * @author Pratham Gahlout
+ */
+public class LicenseDetectionResult {
+
+    private static final double DIFFERENTIATING_TOLERANCE = 0.05; // 5% 
+
+    private List<Pair<String,Double>> licenses;
+    private String content;
+
+    public LicenseDetectionResult(List<Pair<String,Double>> licenses, String content) {
+        this.licenses = licenses;
+        this.content = content;
+    }
+
+    public LicenseDetectionResult(String content) {
+        this.content = content;
+        this.licenses = new ArrayList<>();
+    }
+
+    public LicenseDetectionResult() {
+        this.licenses = new ArrayList<>();
+        this.content = "";
+    }
+
+    public void addProbableLicense(Pair<String,Double> license) {
+        if(licenses==null) {
+            this.licenses = new ArrayList<>();
+        }
+        this.licenses.add(license);
+    }
+
+    private void sortLicensesByProbability() {
+        List<Pair<String,Double>> modifiable = new ArrayList<>(this.licenses);
+        modifiable.sort((Pair<String, Double> o1, Pair<String, Double> o2) -> {
+            if(o1.second > o2.second){
+                return -1;
+            }else if(o1.second < o2.second) {
+                return 1;
+            }else{
+                return 0;
+            }
+        });
+        this.setLicenses(modifiable);
+    }
+
+    public List<Pair<String,Double>> getLicenses(){
+        return this.licenses;
+    }
+
+    public String getAnalyzedContent() {
+        return this.content;
+    }
+
+    public void setAnalyzedContent(String content){
+        this.content = content;
+    }
+
+    public void setLicenses(List<Pair<String,Double>> licenses) {
+        this.licenses = licenses;
+    }
+
+    public LicenseDetectionResult getResultWithMostProbableLicenses() {
+        var newResult = new LicenseDetectionResult();
+        newResult.setAnalyzedContent(this.content);
+        var highest = this.licenses.getFirst().second;
+        newResult.setLicenses(this.licenses.stream().filter(item -> (highest - item.second) < DIFFERENTIATING_TOLERANCE)
+                .map(item -> {
+                     return new Pair<>(item.first,Math.floor(item.second * 1000)/1000);
+                }).limit(3).toList());
+        newResult.sortLicensesByProbability();
+        return newResult;
+    }
+
+    public String getLicensesAsString() {
+        return String.join(" | ", licenses.stream().map(item -> item.first + " " + item.second).toList());
+    }
+}

plugin.cargo/src/main/java/com/phsyberdome/plugin/cargo/PluginCargo.java

@@ -84,6 +84,8 @@ public ArrayList<Module> readModules() {
         String version = toml.getString("version","null");
 
         Toml dependencies = toml.getTable("dependencies");
+        if(dependencies==null)
+            dependencies = toml.getTable("workspace.dependencies");
 
         Module root = new Module(moduleTitle, version);
         if(dependencies != null) {
@@ -109,17 +111,21 @@ public ArrayList<Module> readModules() {
 
     private void resolveDependencyTree(Module module) {
         String license = RegistryHelper.getLicenseFromRegistry(module.getName(), module.getVersion());
+        String analyzedContent = "{Registry-Metadata}";
         if(license.isEmpty()) {
             // Try to see if there is an open source url? If yes then analyze that
             String repoUrl = RegistryHelper.getSourceCodeRepoLink(module.getName());
             if(!repoUrl.isEmpty()) {
                 Path path = FileUtil.getFilePathFromURL(repoUrl,this.cloneLocation.toString());
                 if(path!=null){
-                    license = licenseDetector.detect(path.toString()).first;
+                    var detectionResults = licenseDetector.detect(path.toString());
+                    license = detectionResults.getResultWithMostProbableLicenses().getLicensesAsString();
+                    analyzedContent = detectionResults.getAnalyzedContent();
                 }
             }
         }
         module.setLicense(license);
+        module.setAnalyzedContent(analyzedContent);
         // Get dependencies declared in the registry against this version
         List<Pair<String,String>> depData = RegistryHelper.getDependenciesOfCrate(module.getName(),module.getVersion());
         for(var dep: depData) {

plugin.maven/src/main/java/com/phsyberdome/plugin/maven/PluginMaven.java

@@ -178,9 +178,9 @@ private void getLicenseAndTransitiveDependenciesForModule(Module root) {
             return;
         }
         buildDependencyTree(root,path);
-        Pair<String,String> detectionResult = licenseDetector.detect(path.toString());
-        root.setLicense(detectionResult.first);
-        root.setAnalyzedContent(detectionResult.second);
+        var detectionResult = licenseDetector.detect(path.toString());
+        root.setLicense(detectionResult.getResultWithMostProbableLicenses().getLicensesAsString());
+        root.setAnalyzedContent(detectionResult.getAnalyzedContent());
     }
 
     private void buildDependencyTree(Module root, Path pathToModule) {

plugin.npm/pom.xml

@@ -11,6 +11,24 @@
             <artifactId>common-utils</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>5.6.0</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <version>5.6.0</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>5.6.0</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelper.java

@@ -354,7 +354,7 @@ public static List<String> fetchAllVersions(String name){
                 temp.append(line);
             }
             List<Pair<String,String>> versionObjs =  JSONHelper.getValues("/versions", temp.toString());
-            return (List<String>) versionObjs.stream().map(pair -> pair.second);
+            return versionObjs.stream().map(pair -> pair.first).toList();
 
         } catch (MalformedURLException ex) {
             Logger.getLogger(NPMVersionHelper.class.getName()).log(Level.SEVERE, null, ex);

plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelperV2.java

@@ -4,6 +4,7 @@
 
 import static com.phsyberdome.plugin.npm.NPMVersionHelper.fetchAllVersions;
 import com.vdurmont.semver4j.Semver;
+import com.vdurmont.semver4j.SemverException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -27,9 +28,12 @@ public static String pinpointPackageVersion(String name,String version){
         List<String> allVersions = fetchAllVersions(name);
         List<Semver> candidates = new ArrayList<>();
         for(String ver:allVersions){
-            Semver semver = new Semver(ver,Semver.SemverType.NPM);
-            if(semver.satisfies(version)){
-                candidates.add(semver);
+            try {
+                Semver semver = new Semver(ver,Semver.SemverType.NPM);
+                if(semver.satisfies(version)){
+                    candidates.add(semver);
+                }
+            } catch(SemverException ex) {
             }
         }
 

plugin.npm/src/main/java/com/phsyberdome/plugin/npm/PluginNpm.java

@@ -174,10 +174,9 @@ private void getRootModuleWithDependenciesFromLockFile(Module root,JsonNode node
                 modulePath = FileUtil.getFilePathFromURL(registryUrl, this.cloneLocation.toString());
             }
 
-            Pair<String,String> detectionResult = licenseDetector.detect(modulePath.toString());
-            String license = detectionResult.first;
-            m.setLicense(license);
-            m.setAnalyzedContent(detectionResult.second);
+            var detectionResult = licenseDetector.detect(modulePath.toString());
+            m.setLicense(detectionResult.getResultWithMostProbableLicenses().getLicensesAsString());
+            m.setAnalyzedContent(detectionResult.getAnalyzedContent());
             JsonNode dependencies = body.get("dependencies");
             if(dependencies!=null){
                 resolveTransitiveDependencies(m,dependencies);
@@ -235,10 +234,9 @@ private void getRootModuleWithDependencies(Module root,JsonNode node) {
                 if(registryUrl == null || registryUrl.isBlank()) continue;
                 Path modulePath = FileUtil.getFilePathFromURL(registryUrl, this.cloneLocation.toString());
             //}
-            Pair<String,String> detectionResult = licenseDetector.detect(modulePath.toString());
-            String license = detectionResult.first;
-            m.setLicense(license);
-            m.setAnalyzedContent(detectionResult.second);
+            var detectionResult = licenseDetector.detect(modulePath.toString());
+            m.setLicense(detectionResult.getResultWithMostProbableLicenses().getLicensesAsString());
+            m.setAnalyzedContent(detectionResult.getAnalyzedContent());
             resolveTransitiveDependencies(m,modulePath);
             scannedDependencies.add(m);
 //            if(body.get("dependencies") != null) {
@@ -270,10 +268,9 @@ private void resolveTransitiveDependencies(Module root,JsonNode node) {
                 if(registryUrl == null || registryUrl.isBlank()) continue;
                 modulePath = FileUtil.getFilePathFromURL(registryUrl, this.cloneLocation.toString());
             }
-            Pair<String,String> detectionResult = licenseDetector.detect(modulePath.toString());
-            String license = detectionResult.first;
-            m.setLicense(license);
-            m.setAnalyzedContent(detectionResult.second);
+            var detectionResult = licenseDetector.detect(modulePath.toString());
+            m.setLicense(detectionResult.getResultWithMostProbableLicenses().getLicensesAsString());
+            m.setAnalyzedContent(detectionResult.getAnalyzedContent());
             JsonNode dependencies = body.get("dependencies");
             if(dependencies != null) {
                 resolveTransitiveDependencies(m,dependencies);