MET-6199 optimisation and improvements (#168)

* MET-6199 address sonar tech debt part 1 * MET-6199 address sonar tech debt part 2 * MET-6199 add coverage unit tests part 1 * MET-6199 add coverage unit tests part 2 * MET-6199 sonar recommendations * MET-6199 fix failing unit test * MET-6199 add coverage unit tests part 3 * MET-6199 add new transaction every time status is saved and clean up queries update * MET-6199 use database queries to get counters * MET-6199 remove unused imports * MET-6199 update unit tests * MET-6199 update unit tests part 2 * MET-6199 sonar recommendations * MET-6199 add lock mechanism for status update * MET-6199 enable ready condition
europeana · Nov 11, 2024 · bc50bb4 · bc50bb4
1 parent 64b6e05
commit bc50bb4
Show file tree

Hide file tree

Showing 81 changed files with 1,488 additions and 623 deletions.
diff --git a/src/main/java/eu/europeana/metis/sandbox/config/DeBiasConfig.java b/src/main/java/eu/europeana/metis/sandbox/config/DeBiasConfig.java
@@ -36,6 +36,8 @@ public class DeBiasConfig {
    * @param datasetRepository the dataset repository
    * @param recordLogRepository the record log repository
    * @param recordDeBiasPublishable the record publishable
+   * @param recordDeBiasMainRepository the record de bias main repository
+   * @param recordDeBiasDetailRepository the record de bias detail repository
    * @return the detect service
    */
   @Bean

diff --git a/src/main/java/eu/europeana/metis/sandbox/config/SandboxConfig.java b/src/main/java/eu/europeana/metis/sandbox/config/SandboxConfig.java
@@ -264,17 +264,17 @@ private Properties schemaProperties() {
     }
 
     private void addSchemaProperties(String key, Object value, Properties props) {
-        if (value instanceof String string) {
-            props.setProperty(key, string);
-        } else if (value instanceof Map<?, ?> map) {
-            for (Map.Entry<?, ?> entry : map.entrySet()) {
-                String nestedKey = key + "." + entry.getKey().toString();
-                Object nestedValue = entry.getValue();
-                addSchemaProperties(nestedKey, nestedValue, props);
-            }
-        } else {
-            throw new IllegalArgumentException("Property value: " + value);
+      switch (value) {
+        case String string -> props.setProperty(key, string);
+        case Map<?, ?> map -> {
+          for (Map.Entry<?, ?> entry : map.entrySet()) {
+            String nestedKey = key + "." + entry.getKey().toString();
+            Object nestedValue = entry.getValue();
+            addSchemaProperties(nestedKey, nestedValue, props);
+          }
         }
+        case null, default -> throw new IllegalArgumentException("Property value: " + value);
+      }
     }
 
     private static class Schema {

diff --git a/src/main/java/eu/europeana/metis/sandbox/controller/DatasetController.java b/src/main/java/eu/europeana/metis/sandbox/controller/DatasetController.java
@@ -426,10 +426,10 @@ public List<LanguageView> getAllLanguages() {
     @ResponseStatus(HttpStatus.OK)
     public boolean processDeBias(@PathVariable("id") Integer datasetId) {
         ProgressInfoDto progressInfoDto = reportService.getReport(datasetId.toString());
-        if (Optional.ofNullable(debiasStateService.getDeBiasStatus(datasetId))
-                    .map(DeBiasStatusDto::getState)
-                    .orElse("").equals("READY")
-            && progressInfoDto.getStatus().equals(Status.COMPLETED)) {
+        if (progressInfoDto.getStatus().equals(Status.COMPLETED) &&
+            "READY".equals(Optional.ofNullable(debiasStateService.getDeBiasStatus(datasetId))
+                                   .map(DeBiasStatusDto::getState)
+                                   .orElse(""))) {
             debiasStateService.cleanDeBiasReport(datasetId);
             return debiasStateService.process(datasetId);
         } else {

diff --git a/src/main/java/eu/europeana/metis/sandbox/controller/PatternAnalysisController.java b/src/main/java/eu/europeana/metis/sandbox/controller/PatternAnalysisController.java
@@ -30,11 +30,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.Lock;
-import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
@@ -45,7 +43,6 @@
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestController;
 
@@ -172,7 +169,7 @@ public ResponseEntity<List<ProblemPattern>> getRecordPatternAnalysis(
                                 .map(DatasetProblemPatternAnalysisFilter::cleanMessageReportForP7TitleIsEnough)
                                 .map(DatasetProblemPatternAnalysisFilter::sortRecordAnalysisByRecordId)
                                 .sorted(Comparator.comparing(problemPattern -> problemPattern.getProblemPatternDescription().getProblemPatternId()))
-                                .collect(Collectors.toList()),
+                                .toList(),
                         HttpStatus.OK);
     }
 
@@ -187,7 +184,6 @@ public ResponseEntity<List<ProblemPattern>> getRecordPatternAnalysis(
     @ApiResponse(responseCode = "404", description = "Not able to retrieve all timestamps values")
     @GetMapping(value = "execution-timestamps", produces = APPLICATION_JSON_VALUE)
     @ResponseStatus(HttpStatus.OK)
-    @ResponseBody
     public Set<LocalDateTime> getAllExecutionTimestamps() {
         return executionPointService.getAllExecutionTimestamps();
     }
@@ -250,7 +246,7 @@ private List<ProblemPattern> getSortedProblemPatternList(DatasetProblemPatternAn
                     .map(DatasetProblemPatternAnalysisFilter::cleanMessageReportForP7TitleIsEnough)
                     .map(DatasetProblemPatternAnalysisFilter::sortRecordAnalysisByRecordId)
                     .sorted(Comparator.comparing(problemPattern -> problemPattern.getProblemPatternDescription().getProblemPatternId()))
-                    .collect(Collectors.toList());
+                    .toList();
         }
     }
 
@@ -267,7 +263,7 @@ public static ProblemPattern sortRecordAnalysisByRecordId(ProblemPattern problem
                     problemPattern.getRecordAnalysisList()
                             .stream()
                             .sorted(Comparator.comparing(RecordAnalysis::getRecordId))
-                            .collect(Collectors.toList()));
+                            .toList());
         }
 
         /**
@@ -286,9 +282,9 @@ public static ProblemPattern cleanMessageReportForP7TitleIsEnough(ProblemPattern
                                         recordAnalysis.getProblemOccurrenceList()
                                                 .stream()
                                                 .map(problemOccurrence -> new ProblemOccurrence("", problemOccurrence.getAffectedRecordIds()))
-                                                .collect(Collectors.toList())
+                                                .toList()
                                 ))
-                                .collect(Collectors.toList()));
+                                .toList());
             } else {
                 return problemPattern;
             }

diff --git a/src/main/java/eu/europeana/metis/sandbox/domain/Record.java b/src/main/java/eu/europeana/metis/sandbox/domain/Record.java
@@ -44,20 +44,20 @@ private Record(Long recordId, String europeanaId, String providerId, String data
   /**
    * Creates a record based on the provided record but replacing the content with the one provided
    *
-   * @param record must not be null
+   * @param recordValue must not be null
    * @param content must not be null. Xml representation of the record
    * @return record object
    */
-  public static Record from(Record record, byte[] content) {
+  public static Record from(Record recordValue, byte[] content) {
 
     return Record.builder()
-        .recordId(record.getRecordId())
-        .europeanaId(record.getEuropeanaId())
-        .datasetId(record.getDatasetId())
-        .datasetName(record.getDatasetName())
+        .recordId(recordValue.getRecordId())
+        .europeanaId(recordValue.getEuropeanaId())
+        .datasetId(recordValue.getDatasetId())
+        .datasetName(recordValue.getDatasetName())
         .content(content)
-        .country(record.getCountry())
-        .language(record.getLanguage())
+        .country(recordValue.getCountry())
+        .language(recordValue.getLanguage())
         .build();
   }
 
@@ -130,10 +130,10 @@ public boolean equals(Object o) {
     if (o == null || getClass() != o.getClass()) {
       return false;
     }
-    Record record = (Record) o;
+    Record recordToCompare = (Record) o;
     return
-        recordId.equals(record.recordId) &&
-            datasetId.equals(record.datasetId);
+        recordId.equals(recordToCompare.recordId) &&
+            datasetId.equals(recordToCompare.datasetId);
   }
 
   @Override

diff --git a/src/main/java/eu/europeana/metis/sandbox/domain/RecordInfo.java b/src/main/java/eu/europeana/metis/sandbox/domain/RecordInfo.java
@@ -9,31 +9,31 @@
  */
 public class RecordInfo {
 
-  private final Record record;
+  private final Record recordValue;
   private final List<RecordError> errors;
 
   /**
    * Constructor, defaults errors to an empty non-modifiable list
    *
-   * @param record must not be null
+   * @param recordValue must not be null
    */
-  public RecordInfo(Record record) {
-    this(record, List.of());
+  public RecordInfo(Record recordValue) {
+    this(recordValue, List.of());
   }
 
   /**
    * Constructor, store errors as a list
    *
-   * @param record must not be null
+   * @param recordValue must not be null
    * @param errors must not be null
    */
-  public RecordInfo(Record record, List<RecordError> errors) {
-    this.record = record;
+  public RecordInfo(Record recordValue, List<RecordError> errors) {
+    this.recordValue = recordValue;
     this.errors = Collections.unmodifiableList(errors);
   }
 
-  public Record getRecord() {
-    return record;
+  public Record getRecordValue() {
+    return recordValue;
   }
 
   public List<RecordError> getErrors() {
@@ -49,12 +49,12 @@ public boolean equals(Object o) {
       return false;
     }
     RecordInfo that = (RecordInfo) o;
-    return record.equals(that.record) && errors.equals(that.errors);
+    return recordValue.equals(that.recordValue) && errors.equals(that.errors);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(record, errors);
+    return Objects.hash(recordValue, errors);
   }
 
 }
diff --git a/src/main/java/eu/europeana/metis/sandbox/domain/RecordProcessEvent.java b/src/main/java/eu/europeana/metis/sandbox/domain/RecordProcessEvent.java
@@ -31,7 +31,7 @@ public RecordInfo getRecordInfo() {
   }
 
   public Record getRecord() {
-    return recordInfo.getRecord();
+    return recordInfo.getRecordValue();
   }
 
   public List<RecordError> getRecordErrors() {

diff --git a/src/main/java/eu/europeana/metis/sandbox/dto/ExceptionModelDto.java b/src/main/java/eu/europeana/metis/sandbox/dto/ExceptionModelDto.java
@@ -38,4 +38,4 @@ public HttpStatus getStatus() {
   public String getMessage() {
     return this.message;
   }
-}
+}
diff --git a/src/main/java/eu/europeana/metis/sandbox/dto/debias/DeBiasReportDto.java b/src/main/java/eu/europeana/metis/sandbox/dto/debias/DeBiasReportDto.java
@@ -27,38 +27,13 @@ public class DeBiasReportDto extends DeBiasStatusDto {
    * @param datasetId the dataset id
    * @param state the state
    * @param creationDate the creation date
+   * @param total the total records
+   * @param processed the processed records
    * @param deBiasReportRowList the de bias report row list
    */
-  public DeBiasReportDto(Integer datasetId, String state, ZonedDateTime creationDate, List<DeBiasReportRow> deBiasReportRowList) {
-    super(datasetId, state, creationDate);
-    this.deBiasReportRowList = deBiasReportRowList;
-  }
-
-  /**
-   * Gets dataset id.
-   *
-   * @return the dataset id
-   */
-  public Integer getDatasetId() {
-    return super.getDatasetId();
-  }
-
-  /**
-   * Gets state.
-   *
-   * @return the state
-   */
-  public String getState() {
-    return super.getState();
-  }
-
-  /**
-   * Gets creation date.
-   *
-   * @return the creation date
-   */
-  public ZonedDateTime getCreationDate() {
-    return super.getCreationDate();
+  public DeBiasReportDto(Integer datasetId, String state, ZonedDateTime creationDate, int total, int processed, List<DeBiasReportRow> deBiasReportRowList) {
+    super(datasetId, state, creationDate, total, processed);
+    this.deBiasReportRowList = Collections.unmodifiableList(deBiasReportRowList);
   }
 
   /**

diff --git a/src/main/java/eu/europeana/metis/sandbox/dto/debias/DeBiasStatusDto.java b/src/main/java/eu/europeana/metis/sandbox/dto/debias/DeBiasStatusDto.java
@@ -26,17 +26,28 @@ public class DeBiasStatusDto {
   @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss")
   private final ZonedDateTime creationDate;
 
+  @JsonProperty("total-records")
+  private final Integer total;
+
+  @JsonProperty("processed-records")
+  private final Integer processed;
+
   /**
    * Instantiates a new Detection info dto.
    *
    * @param datasetId the dataset id
    * @param state the state
    * @param creationDate the creation date
+   * @param total the total
+   * @param processed the proccessed
    */
-  public DeBiasStatusDto(Integer datasetId, String state, ZonedDateTime creationDate) {
+  public DeBiasStatusDto(Integer datasetId, String state, ZonedDateTime creationDate,
+      Integer total, Integer processed) {
     this.datasetId = datasetId;
     this.state = state;
     this.creationDate = creationDate;
+    this.total = total;
+    this.processed = processed;
   }
 
   /**
@@ -66,4 +77,21 @@ public ZonedDateTime getCreationDate() {
     return creationDate;
   }
 
+  /**
+   * Gets total.
+   *
+   * @return the total
+   */
+  public Integer getTotal() {
+    return total;
+  }
+
+  /**
+   * Gets success.
+   *
+   * @return the success
+   */
+  public Integer getProcessed() {
+    return processed;
+  }
 }
diff --git a/src/main/java/eu/europeana/metis/sandbox/dto/report/DatasetLogDto.java b/src/main/java/eu/europeana/metis/sandbox/dto/report/DatasetLogDto.java
@@ -3,7 +3,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import eu.europeana.metis.sandbox.common.Status;
 import io.swagger.annotations.ApiModel;
-import java.util.Objects;
 
 /**
  * Represent errors or warnings in the dataset report

diff --git a/src/main/java/eu/europeana/metis/sandbox/dto/report/ProgressInfoDto.java b/src/main/java/eu/europeana/metis/sandbox/dto/report/ProgressInfoDto.java
@@ -5,6 +5,7 @@
 import eu.europeana.metis.sandbox.common.Step;
 import io.swagger.annotations.ApiModel;
 
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -51,7 +52,7 @@ public ProgressInfoDto(String portalPublishUrl, Long totalRecords, Long processe
                            List<DatasetLogDto> datasetLogs, TiersZeroInfo tiersZeroInfo) {
         this.processedRecords = processedRecords;
         this.tiersZeroInfo = tiersZeroInfo;
-        if (!errorType.equals("")) {
+        if (!errorType.isEmpty()) {
             this.status = Status.FAILED;
             this.totalRecords = totalRecords != null ? totalRecords : 0L;
         } else if (totalRecords == null) {
@@ -91,7 +92,7 @@ public long getProcessedRecords() {
     }
 
     public List<ProgressByStepDto> getProgressByStep() {
-        return progressByStep;
+        return Collections.unmodifiableList(progressByStep);
     }
 
     public boolean getRecordLimitExceeded() {

diff --git a/src/main/java/eu/europeana/metis/sandbox/entity/RecordEntity.java b/src/main/java/eu/europeana/metis/sandbox/entity/RecordEntity.java
@@ -227,7 +227,8 @@ public static class RecordEntityBuilder{
     /**
      * Constructor
      */
-    public RecordEntityBuilder(){
+    public RecordEntityBuilder() {
+      // no op
     }
 
     /**
-Original file line number
+Diff line change
@@ Expand Up / @@ -38,4 +38,4 @@ public HttpStatus getStatus() { @@
       public String getMessage() {
         return this.message;
       }
-    }
+    }