AECORE-115 Assessment Merger

libraries/ae-inventory-processor/src/main/java/org/metaeffekt/core/inventory/processor/model/AssetMetaData.java

@@ -35,7 +35,11 @@ public AssetMetaData() {
      * Core attributes to support license data.
      */
     public enum Attribute implements AbstractModelBase.Attribute {
-        ASSET_ID("Asset Id");
+        ASSET_ID("Asset Id"),
+        NAME("Name"),
+        VERSION("Version"),
+        ASSESSMENT("Assessment"),
+        ;
 
         private String key;
 

libraries/ae-inventory-processor/src/main/java/org/metaeffekt/core/inventory/processor/report/AssessmentInventoryMerger.java

@@ -0,0 +1,166 @@
+/*
+ * Copyright 2009-2022 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.metaeffekt.core.inventory.processor.report;
+
+import org.apache.commons.lang3.StringUtils;
+import org.metaeffekt.core.inventory.processor.model.AssetMetaData;
+import org.metaeffekt.core.inventory.processor.model.Inventory;
+import org.metaeffekt.core.inventory.processor.reader.InventoryReader;
+import org.metaeffekt.core.inventory.processor.writer.InventoryWriter;
+import org.metaeffekt.core.util.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+public class AssessmentInventoryMerger {
+
+    private final static Logger LOG = LoggerFactory.getLogger(AssessmentInventoryMerger.class);
+
+    private List<File> inputInventoryFiles;
+    private List<Inventory> inputInventories;
+
+    public AssessmentInventoryMerger(List<File> inputInventoryFiles, List<Inventory> inputInventories) {
+        this.inputInventoryFiles = inputInventoryFiles;
+        this.inputInventories = inputInventories;
+    }
+
+    public AssessmentInventoryMerger() {
+        this.inputInventoryFiles = new ArrayList<>();
+        this.inputInventories = new ArrayList<>();
+    }
+
+    public Inventory mergeInventories() throws IOException {
+        final Inventory outputInventory = new Inventory();
+
+        final Map<Inventory, File> collectedInventories = new LinkedHashMap<>();
+        inputInventories.forEach(i -> collectedInventories.put(i, null));
+
+        final List<File> inventoryFiles = collectInventoryFiles();
+
+        LOG.info("Processing [{}] inventories", collectedInventories.size() + inventoryFiles.size());
+
+        {
+            final InventoryReader reader = new InventoryReader();
+            for (File inventoryFile : inventoryFiles) {
+                collectedInventories.put(reader.readInventory(inventoryFile), inventoryFile);
+            }
+        }
+
+        final Map<String, List<Inventory>> assessmentContextInventoryMap = new HashMap<>();
+        final Map<String, Inventory> assessmentInventoryMap = new HashMap<>();
+
+        for (Map.Entry<Inventory, File> inputInventoryWithFile : collectedInventories.entrySet()) {
+            final Inventory inputInventory = inputInventoryWithFile.getKey();
+            final File inputInventoryFile = inputInventoryWithFile.getValue();
+
+            if (inputInventory.getAssetMetaData().size() != 1) {
+                throw new IllegalStateException("Inventory must contain exactly one asset meta data entry" + (inputInventoryFile != null ? " in file [" + inputInventoryFile.getAbsolutePath() + "]" : ""));
+            }
+
+            final AssetMetaData assetMetaData = inputInventory.getAssetMetaData().get(0);
+
+            // in this case the assessment context is the asset (assessment by asset case)
+            final String assetName = assetMetaData.get(AssetMetaData.Attribute.NAME);
+            if (StringUtils.isEmpty(assetName)) {
+                throw new IllegalStateException("Asset name must not be empty on asset meta data" + (inputInventoryFile != null ? " in file [" + inputInventoryFile.getAbsolutePath() + "]" : ""));
+            }
+            final String assessmentContext = formatNormalizedAssessmentContextName(assetName);
+            LOG.info("Processing inventory with asset [{}] and assessment context [{}]", assetName, assessmentContext);
+
+            final List<Inventory> commonInventories = assessmentContextInventoryMap.computeIfAbsent(assessmentContext, a -> new ArrayList<>());
+            final int inventoryDisplayIndex = commonInventories.size() + 1;
+            final String localUniqueAssessmentId = String.format("%s-%03d", assessmentContext, inventoryDisplayIndex);
+
+            assetMetaData.set(AssetMetaData.Attribute.ASSESSMENT, localUniqueAssessmentId);
+            assetMetaData.set(AssetMetaData.Attribute.NAME, assetName.toUpperCase());
+
+            commonInventories.add(inputInventory);
+            assessmentInventoryMap.put(localUniqueAssessmentId, inputInventory);
+
+            // contribute asset metadata to output inventory
+            outputInventory.getAssetMetaData().add(assetMetaData);
+        }
+
+        // iterate over each asset in the output inventory and add its respective vulnerabilities
+        for (AssetMetaData assetMetaData : outputInventory.getAssetMetaData()) {
+            final String localUniqueAssessmentId = assetMetaData.get(AssetMetaData.Attribute.ASSESSMENT);
+            if (StringUtils.isNotEmpty(localUniqueAssessmentId)) {
+                final Inventory singleInventory = assessmentInventoryMap.get(localUniqueAssessmentId);
+                if (singleInventory != null) {
+                    outputInventory.getVulnerabilityMetaData(localUniqueAssessmentId).addAll(singleInventory.getVulnerabilityMetaData());
+                }
+            }
+        }
+
+        return outputInventory;
+    }
+
+    protected String formatNormalizedAssessmentContextName(String assetName) {
+        String assessmentContext = assetName.toUpperCase().replace(" ", "_");
+        final int maxAllowedChars = 25;
+        assessmentContext = assessmentContext.substring(0, Math.min(assessmentContext.length(), maxAllowedChars - InventoryWriter.VULNERABILITY_ASSESSMENT_WORKSHEET_PREFIX.length()));
+
+        // remove trailing "_-"
+        while (assessmentContext.endsWith("_") || assessmentContext.endsWith("-")) {
+            assessmentContext = assessmentContext.substring(0, assessmentContext.length() - 1);
+        }
+        return assessmentContext;
+    }
+
+    private List<File> collectInventoryFiles() {
+        final List<File> inventoryFiles = new ArrayList<>();
+
+        for (File inputInventory : inputInventoryFiles) {
+            if (inputInventory.isDirectory()) {
+                final String[] files = FileUtils.scanDirectoryForFiles(inputInventory, "*.xls");
+                for (String file : files) {
+                    inventoryFiles.add(new File(inputInventory, file));
+                }
+            } else {
+                inventoryFiles.add(inputInventory);
+            }
+        }
+
+        return inventoryFiles;
+    }
+
+    public void addInputInventoryFile(File inputInventory) {
+        this.inputInventoryFiles.add(inputInventory);
+    }
+
+    public void setInputInventoryFiles(List<File> inputInventoryFiles) {
+        this.inputInventoryFiles = inputInventoryFiles;
+    }
+
+    public List<File> getInputInventoryFiles() {
+        return inputInventoryFiles;
+    }
+
+    public void addInputInventory(Inventory inputInventory) {
+        this.inputInventories.add(inputInventory);
+    }
+
+    public void setInputInventories(List<Inventory> inputInventories) {
+        this.inputInventories = inputInventories;
+    }
+
+    public List<Inventory> getInputInventories() {
+        return inputInventories;
+    }
+}

libraries/ae-inventory-processor/src/main/java/org/metaeffekt/core/inventory/processor/report/AssessmentReportAdapter.java

@@ -69,7 +69,7 @@ public VulnerabilityCounts countVulnerabilities(AssetMetaData assetMetaData, boo
                         break;
                 }
 
-                boolean isAssessed = vulnerabilityMetaData.isStatus(STATUS_VALUE_APPLICABLE) ||
+                final boolean isAssessed = vulnerabilityMetaData.isStatus(STATUS_VALUE_APPLICABLE) ||
                         vulnerabilityMetaData.isStatus(STATUS_VALUE_NOTAPPLICABLE) ||
                         vulnerabilityMetaData.isStatus(STATUS_VALUE_VOID);
 

libraries/ae-inventory-processor/src/main/java/org/metaeffekt/core/inventory/processor/writer/InventoryWriter.java

@@ -24,7 +24,7 @@
 
 public class InventoryWriter extends AbstractXlsInventoryWriter {
 
-    private final Logger LOG = LoggerFactory.getLogger(getClass());
+    private final static Logger LOG = LoggerFactory.getLogger(InventoryWriter.class);
 
     public static final String VULNERABILITY_ASSESSMENT_WORKSHEET_PREFIX = "Assessment-";
 

libraries/ae-inventory-processor/src/main/resources/META-INF/templates/en/assessment-report/macros/assessment-report.vm

@@ -29,7 +29,6 @@
         <tbody>
 #foreach ($asset in $assets)
 #set($counts = $assessmentReportAdapter.countVulnerabilities($asset, $useModifiedSeverity))
-$counts
             <row  align="center">
                 <entry align="center">$report.xmlEscapeString($asset.get("Name"))</entry>
                 <entry>$counts.criticalCounter</entry>

libraries/ae-inventory-processor/src/test/java/org/metaeffekt/core/inventory/processor/report/AssessmentInventoryMergerTest.java

@@ -0,0 +1,146 @@
+/*
+ * Copyright 2009-2022 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.metaeffekt.core.inventory.processor.report;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.metaeffekt.core.inventory.processor.model.AssetMetaData;
+import org.metaeffekt.core.inventory.processor.model.Inventory;
+import org.metaeffekt.core.inventory.processor.model.VulnerabilityMetaData;
+
+import java.io.IOException;
+
+public class AssessmentInventoryMergerTest {
+
+    @Test
+    public void mergeTwoInventoriesSuccessfullyTest() throws IOException {
+        final AssessmentInventoryMerger merger = new AssessmentInventoryMerger();
+
+        {
+            final Inventory inputInventory = new Inventory();
+            merger.addInputInventory(inputInventory);
+
+            final AssetMetaData amd = new AssetMetaData();
+            inputInventory.getAssetMetaData().add(amd);
+            amd.set(AssetMetaData.Attribute.ASSET_ID, "some id");
+            amd.set(AssetMetaData.Attribute.NAME, "some id");
+
+            final VulnerabilityMetaData vmd = new VulnerabilityMetaData();
+            inputInventory.getVulnerabilityMetaData().add(vmd);
+            vmd.set(VulnerabilityMetaData.Attribute.NAME, "CVE-2018-1234");
+        }
+
+        {
+            final Inventory inputInventory = new Inventory();
+            merger.addInputInventory(inputInventory);
+
+            final AssetMetaData amd = new AssetMetaData();
+            inputInventory.getAssetMetaData().add(amd);
+            amd.set(AssetMetaData.Attribute.ASSET_ID, "some other id");
+            amd.set(AssetMetaData.Attribute.NAME, "some other id");
+
+            final VulnerabilityMetaData vmd = new VulnerabilityMetaData();
+            inputInventory.getVulnerabilityMetaData().add(vmd);
+            vmd.set(VulnerabilityMetaData.Attribute.NAME, "CVE-2020-4321");
+        }
+
+        final Inventory merged = merger.mergeInventories();
+
+        Assertions.assertThat(merged.getAssetMetaData()).hasSize(2);
+        Assertions.assertThat(merged.getVulnerabilityMetaData("SOME_ID-001")).hasSize(1);
+        Assertions.assertThat(merged.getVulnerabilityMetaData("SOME_OTHER_ID-001")).hasSize(1);
+        Assertions.assertThat(merged.getVulnerabilityMetaData("SOME_ID-001").iterator().next().get(VulnerabilityMetaData.Attribute.NAME)).isEqualTo("CVE-2018-1234");
+        Assertions.assertThat(merged.getVulnerabilityMetaData("SOME_OTHER_ID-001").iterator().next().get(VulnerabilityMetaData.Attribute.NAME)).isEqualTo("CVE-2020-4321");
+    }
+
+    @Test
+    public void mergeTwoInventoriesFailReasonsTest() {
+        final AssessmentInventoryMerger merger = new AssessmentInventoryMerger();
+
+        final Inventory inputInventory = new Inventory();
+        merger.addInputInventory(inputInventory);
+
+        final VulnerabilityMetaData vmd = new VulnerabilityMetaData();
+        inputInventory.getVulnerabilityMetaData().add(vmd);
+        vmd.set(VulnerabilityMetaData.Attribute.NAME, "CVE-2018-1234");
+
+        Assertions.assertThatThrownBy(merger::mergeInventories)
+                .isInstanceOf(IllegalStateException.class)
+                .hasMessageContaining("exactly one asset");
+
+        final AssetMetaData amd = new AssetMetaData();
+        inputInventory.getAssetMetaData().add(amd);
+
+        Assertions.assertThatThrownBy(merger::mergeInventories)
+                .isInstanceOf(IllegalStateException.class)
+                .hasMessageContaining("name must not be empty");
+
+        inputInventory.getAssetMetaData().add(amd);
+        amd.set(AssetMetaData.Attribute.ASSET_ID, "some id");
+        amd.set(AssetMetaData.Attribute.NAME, "some id");
+
+        Assertions.assertThatThrownBy(merger::mergeInventories)
+                .isInstanceOf(IllegalStateException.class)
+                .hasMessageContaining("exactly one asset");
+    }
+
+    @Test
+    public void mergeInventoriesWithDuplicateAssetsTest() throws IOException {
+        final AssessmentInventoryMerger merger = new AssessmentInventoryMerger();
+
+        {
+            final Inventory inputInventory = new Inventory();
+            merger.addInputInventory(inputInventory);
+
+            final AssetMetaData amd = new AssetMetaData();
+            inputInventory.getAssetMetaData().add(amd);
+            amd.set(AssetMetaData.Attribute.ASSET_ID, "duplicate id");
+            amd.set(AssetMetaData.Attribute.NAME, "duplicate id");
+
+            final VulnerabilityMetaData vmd = new VulnerabilityMetaData();
+            inputInventory.getVulnerabilityMetaData().add(vmd);
+            vmd.set(VulnerabilityMetaData.Attribute.NAME, "CVE-2018-1234");
+        }
+
+        {
+            final Inventory inputInventory = new Inventory();
+            merger.addInputInventory(inputInventory);
+
+            final AssetMetaData amd = new AssetMetaData();
+            inputInventory.getAssetMetaData().add(amd);
+            amd.set(AssetMetaData.Attribute.ASSET_ID, "duplicate id");
+            amd.set(AssetMetaData.Attribute.NAME, "duplicate id");
+
+            final VulnerabilityMetaData vmd = new VulnerabilityMetaData();
+            inputInventory.getVulnerabilityMetaData().add(vmd);
+            vmd.set(VulnerabilityMetaData.Attribute.NAME, "CVE-2020-4321");
+        }
+
+        final Inventory merged = merger.mergeInventories();
+
+        Assertions.assertThat(merged.getAssetMetaData()).hasSize(2);
+        Assertions.assertThat(merged.getVulnerabilityMetaData("DUPLICATE_ID-001")).hasSize(1);
+        Assertions.assertThat(merged.getVulnerabilityMetaData("DUPLICATE_ID-002")).hasSize(1);
+        Assertions.assertThat(merged.getVulnerabilityMetaData("DUPLICATE_ID-001").iterator().next().get(VulnerabilityMetaData.Attribute.NAME)).isEqualTo("CVE-2018-1234");
+        Assertions.assertThat(merged.getVulnerabilityMetaData("DUPLICATE_ID-002").iterator().next().get(VulnerabilityMetaData.Attribute.NAME)).isEqualTo("CVE-2020-4321");
+    }
+
+    @Test
+    public void normalizeNameTest() {
+        final AssessmentInventoryMerger merger = new AssessmentInventoryMerger();
+        Assertions.assertThat(merger.formatNormalizedAssessmentContextName("Test-Name-longer-than-25-chars")).isEqualTo("TEST-NAME-LONG");
+    }
+}