MET-6199 get latest changes from branch 'develop' into feat/MET-6199_…

…backend_optimization_debias

.github/workflows/ci.yml

@@ -14,6 +14,7 @@ jobs:
           sonar_organization: europeana
           sonar_project_key: europeana_metis-sandbox
           sonar_token: ${{ secrets.METIS_SONAR_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
       - name: Upload all jars to target-jars
         uses: europeana/metis-actions/actions/upload-target-jars@main
 

src/main/java/eu/europeana/metis/sandbox/repository/HarvestingParameterRepository.java

@@ -2,6 +2,8 @@
 
 import eu.europeana.metis.sandbox.entity.HarvestingParameterEntity;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 
 /**
  * Repository connected to Harvesting Parameters table
@@ -19,5 +21,8 @@ public interface HarvestingParameterRepository extends JpaRepository<HarvestingP
      * Removes the entity from the table based on the dataset id
      * @param datasetId The dataset id associated to the entity
      */
-    void deleteByDatasetId_DatasetId(Integer datasetId);
+    @Modifying
+    @Query("DELETE FROM HarvestingParameterEntity hpe WHERE EXISTS (SELECT 1 FROM DatasetEntity dte "
+        + "WHERE dte.datasetId = hpe.datasetId.datasetId AND hpe.datasetId.datasetId = ?1)")
+    void deleteByDatasetIdDatasetId(Integer datasetId);
 }

src/main/java/eu/europeana/metis/sandbox/repository/RecordErrorLogRepository.java

@@ -46,5 +46,7 @@ public interface RecordErrorLogRepository extends JpaRepository<RecordErrorLogEn
    * @param datasetId must not be null
    */
   @Modifying
+  @Query("DELETE FROM RecordErrorLogEntity ele WHERE EXISTS (SELECT 1 FROM RecordEntity rec "
+      + "WHERE rec.id = ele.recordId.id AND rec.datasetId = ?1)")
   void deleteByRecordIdDatasetId(String datasetId);
 }

src/main/java/eu/europeana/metis/sandbox/repository/RecordLogRepository.java

@@ -107,6 +107,8 @@ public interface RecordLogRepository extends JpaRepository<RecordLogEntity, Long
    * @param datasetId must not be null
    */
   @Modifying
+  @Query("DELETE FROM RecordLogEntity rle WHERE EXISTS (SELECT 1 FROM RecordEntity rec "
+      + "WHERE rec.id = rle.recordId.id AND rec.datasetId = ?1)")
   void deleteByRecordIdDatasetId(String datasetId);
 
   /**
@@ -116,6 +118,9 @@ public interface RecordLogRepository extends JpaRepository<RecordLogEntity, Long
    * @param step the step
    */
   @Modifying
+  @Query("DELETE FROM RecordLogEntity rle WHERE "+
+      " EXISTS (SELECT 1 FROM RecordEntity rec WHERE rec.id = rle.recordId.id AND rec.datasetId = ?1)"+
+      " AND rle.step = ?2")
   void deleteByRecordIdDatasetIdAndStep(String datasetId, Step step);
 
   /**

src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/DatasetProblemPatternRepository.java

@@ -18,15 +18,18 @@ public interface DatasetProblemPatternRepository extends JpaRepository<DatasetPr
    * @param datasetId the dataset id
    */
   @Modifying
+  @Query("DELETE FROM DatasetProblemPattern dpp WHERE EXISTS ("
+      + " SELECT 1 FROM ExecutionPoint ep WHERE dpp.executionPoint.executionPointId = ep.executionPointId AND ep.datasetId = ?1)")
   void deleteByExecutionPointDatasetId(String datasetId);
 
   /**
-   * Get metrics by problem pattern ocurrences for a given time using custom query
+   * Get metrics by problem pattern occurrences for a given time using custom query
    *
    * @return metrics Dataset Problem Pattern Statistics
    * @see DatasetProblemPatternStatistic
    */
-  @Query(value = "SELECT new eu.europeana.metis.sandbox.common.aggregation.DatasetProblemPatternStatistic(dpp.datasetProblemPatternCompositeKey.patternId, SUM(dpp.recordOccurrences)) "
+  @Query(value = "SELECT new eu.europeana.metis.sandbox.common.aggregation.DatasetProblemPatternStatistic("
+      + "dpp.datasetProblemPatternCompositeKey.patternId, SUM(dpp.recordOccurrences)) "
       + "FROM DatasetProblemPattern dpp "
       + "GROUP BY dpp.datasetProblemPatternCompositeKey.patternId")
   List<DatasetProblemPatternStatistic> getMetricProblemPatternStatistics();

src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/ExecutionPointRepository.java

@@ -7,12 +7,35 @@
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
 
+/**
+ * The interface Execution point repository.
+ */
 public interface ExecutionPointRepository extends JpaRepository<ExecutionPoint, Integer> {
 
+  /**
+   * Find by dataset id and execution step and execution timestamp execution point.
+   *
+   * @param datasetId the dataset id
+   * @param executionStep the execution step
+   * @param executionTimestamp the execution timestamp
+   * @return the execution point
+   */
   ExecutionPoint findByDatasetIdAndExecutionStepAndExecutionTimestamp(String datasetId, String executionStep, LocalDateTime executionTimestamp);
 
+  /**
+   * Find first by dataset id and execution step order by execution timestamp desc optional.
+   *
+   * @param datasetId the dataset id
+   * @param executionStep the execution step
+   * @return the optional
+   */
   Optional<ExecutionPoint> findFirstByDatasetIdAndExecutionStepOrderByExecutionTimestampDesc(String datasetId, String executionStep);
 
+  /**
+   * Delete by dataset id.
+   *
+   * @param datasetId the dataset id
+   */
   @Modifying
   void deleteByDatasetId(String datasetId);
 

src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternOccurrenceRepository.java

@@ -3,10 +3,25 @@
 import eu.europeana.metis.sandbox.entity.problempatterns.RecordProblemPatternOccurrence;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 
+/**
+ * The interface Record problem pattern occurrence repository.
+ */
 public interface RecordProblemPatternOccurrenceRepository extends JpaRepository<RecordProblemPatternOccurrence, Integer> {
 
+    /**
+     * Delete by record problem pattern execution point dataset id.
+     *
+     * @param datasetId the dataset id
+     */
     @Modifying
+    @Query("DELETE FROM RecordProblemPatternOccurrence rppo WHERE "+
+        " EXISTS (SELECT 1 FROM RecordProblemPattern rpp "
+        + " WHERE rppo.recordProblemPattern.recordProblemPatternId = rpp.recordProblemPatternId "
+        + " AND EXISTS (SELECT 1 FROM ExecutionPoint ep "
+        + " WHERE ep.executionPointId = rpp.executionPoint.executionPointId"
+        + " AND ep.datasetId= ?1))")
     void deleteByRecordProblemPatternExecutionPointDatasetId(String datasetId);
 
 }

src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternRepository.java

@@ -4,12 +4,29 @@
 import eu.europeana.metis.sandbox.entity.problempatterns.RecordProblemPattern;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 
+/**
+ * The interface Record problem pattern repository.
+ */
 public interface RecordProblemPatternRepository extends JpaRepository<RecordProblemPattern, Integer> {
 
+  /**
+   * Find by execution point record problem pattern.
+   *
+   * @param executionPoint the execution point
+   * @return the record problem pattern
+   */
   RecordProblemPattern findByExecutionPoint(ExecutionPoint executionPoint);
 
+  /**
+   * Delete by execution point dataset id.
+   *
+   * @param datasetId the dataset id
+   */
   @Modifying
+  @Query("DELETE FROM RecordProblemPattern rpp WHERE EXISTS (SELECT 1 FROM ExecutionPoint ep "
+      + "WHERE rpp.executionPoint.executionPointId = ep.executionPointId AND ep.datasetId = ?1)")
   void deleteByExecutionPointDatasetId(String datasetId);
 
 }

src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordTitleRepository.java

@@ -6,13 +6,36 @@
 import java.util.List;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 
+/**
+ * The interface Record title repository.
+ */
 public interface RecordTitleRepository extends JpaRepository<RecordTitle, RecordTitleCompositeKey> {
 
+  /**
+   * Find all by execution point list.
+   *
+   * @param executionPoint the execution point
+   * @return the list
+   */
   List<RecordTitle> findAllByExecutionPoint(ExecutionPoint executionPoint);
 
+  /**
+   * Delete by execution point.
+   *
+   * @param executionPoint the execution point
+   */
   @Modifying
   void deleteByExecutionPoint(ExecutionPoint executionPoint);
+
+  /**
+   * Delete by execution point dataset id.
+   *
+   * @param datasetId the dataset id
+   */
   @Modifying
+  @Query("DELETE FROM RecordTitle rti WHERE EXISTS (SELECT 1 FROM ExecutionPoint ep "
+      + "WHERE rti.executionPoint.executionPointId = ep.executionPointId AND ep.datasetId = ?1)")
   void deleteByExecutionPointDatasetId(String datasetId);
 }

src/main/java/eu/europeana/metis/sandbox/service/dataset/DatasetRemoverServiceImpl.java

@@ -58,25 +58,25 @@ public void remove(int days) {
       datasets.forEach(dataset -> {
         try {
           // remove thumbnails (s3)
-          LOGGER.debug("Remove thumbnails for dataset id: [{}]", dataset);
+          LOGGER.info("Remove thumbnails for dataset id: [{}]", dataset);
           thumbnailStoreService.remove(dataset);
           // remove from mongo and solr
-          LOGGER.debug("Remove index for dataset id: [{}]", dataset);
+          LOGGER.info("Remove index for dataset id: [{}]", dataset);
           indexingService.remove(dataset);
           // remove from sandbox
-          LOGGER.debug("Remove record logs for dataset id: [{}]", dataset);
+          LOGGER.info("Remove record logs for dataset id: [{}]", dataset);
           recordLogService.remove(dataset);
-          LOGGER.debug("Remove records for dataset id: [{}]", dataset);
+          LOGGER.info("Remove records for dataset id: [{}]", dataset);
           recordService.remove(dataset);
-          LOGGER.debug("Remove logs for dataset id: [{}]", dataset);
+          LOGGER.info("Remove logs for dataset id: [{}]", dataset);
           datasetLogService.remove(dataset);
-          LOGGER.debug("Remove harvesting parameters for dataset id: [{}]", dataset);
+          LOGGER.info("Remove harvesting parameters for dataset id: [{}]", dataset);
           harvestingParameterService.remove(dataset);
-          LOGGER.debug("Remove problem pattern data associated with dataset id: [{}]", dataset);
+          LOGGER.info("Remove problem pattern data associated with dataset id: [{}]", dataset);
           problemPatternDataRemover.removeProblemPatternDataFromDatasetId(dataset);
-          LOGGER.debug("Remove dataset with id: [{}]", dataset);
+          LOGGER.info("Remove dataset with id: [{}]", dataset);
           datasetService.remove(dataset);
-          LOGGER.debug("Remove debias report with id: [{}]", dataset);
+          LOGGER.info("Remove debias report with id: [{}]", dataset);
           debiasStateService.cleanDeBiasReport(Integer.valueOf(dataset));
         } catch (ServiceException e) {
           LOGGER.error("Failed to remove dataset [{}] ", dataset, e);

src/main/java/eu/europeana/metis/sandbox/service/dataset/HarvestingParameterServiceImpl.java

@@ -62,7 +62,7 @@ public HarvestingParameterEntity getDatasetHarvestingParameters(String datasetId
     public void remove(String datasetId) {
         requireNonNull(datasetId, "Dataset id must not be null");
         try {
-            harvestingParameterRepository.deleteByDatasetId_DatasetId(Integer.parseInt(datasetId));
+            harvestingParameterRepository.deleteByDatasetIdDatasetId(Integer.parseInt(datasetId));
         } catch (RuntimeException e) {
             throw new ServiceException(
                     format("Error removing records for dataset id: [%s]. ", datasetId), e);

src/main/java/eu/europeana/metis/sandbox/service/debias/DeBiasStateServiceImpl.java

@@ -114,6 +114,7 @@ public void cleanDeBiasReport(Integer datasetId) {
     Objects.requireNonNull(datasetId, "Dataset id must not be null");
     this.recordDeBiasDetailRepository.deleteByDebiasIdRecordIdDatasetId(datasetId.toString());
     this.recordDeBiasMainRepository.deleteByRecordIdDatasetId(datasetId.toString());
+    this.datasetDeBiasRepository.deleteByDatasetId(datasetId.toString());
   }
 
   /**

src/test/java/eu/europeana/metis/sandbox/service/dataset/HarvestParameterServiceImplTest.java

@@ -141,12 +141,12 @@ void getDatasetHarvestingParameters_expectSuccess(){
     @Test
     void remove_expectSuccess(){
         harvestingParameterService.remove("1");
-        verify(harvestingParameterRepository).deleteByDatasetId_DatasetId(1);
+        verify(harvestingParameterRepository).deleteByDatasetIdDatasetId(1);
     }
 
     @Test
     void remove_expectFail(){
-        doThrow(new RuntimeException("error")).when(harvestingParameterRepository).deleteByDatasetId_DatasetId(1);
+        doThrow(new RuntimeException("error")).when(harvestingParameterRepository).deleteByDatasetIdDatasetId(1);
         assertThrows(ServiceException.class, () -> harvestingParameterService.remove("1"));
     }