diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83799951..3601f415 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,7 @@ jobs: sonar_organization: europeana sonar_project_key: europeana_metis-sandbox sonar_token: ${{ secrets.METIS_SONAR_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} - name: Upload all jars to target-jars uses: europeana/metis-actions/actions/upload-target-jars@main diff --git a/src/main/java/eu/europeana/metis/sandbox/repository/HarvestingParameterRepository.java b/src/main/java/eu/europeana/metis/sandbox/repository/HarvestingParameterRepository.java index 0eea0540..b0366a81 100644 --- a/src/main/java/eu/europeana/metis/sandbox/repository/HarvestingParameterRepository.java +++ b/src/main/java/eu/europeana/metis/sandbox/repository/HarvestingParameterRepository.java @@ -2,6 +2,8 @@ import eu.europeana.metis.sandbox.entity.HarvestingParameterEntity; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; /** * Repository connected to Harvesting Parameters table @@ -19,5 +21,8 @@ public interface HarvestingParameterRepository extends JpaRepository getMetricProblemPatternStatistics(); diff --git a/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/ExecutionPointRepository.java b/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/ExecutionPointRepository.java index e936e348..1152a5b1 100644 --- a/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/ExecutionPointRepository.java +++ b/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/ExecutionPointRepository.java @@ -7,12 +7,35 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; +/** + * The interface Execution point repository. + */ public interface ExecutionPointRepository extends JpaRepository { + /** + * Find by dataset id and execution step and execution timestamp execution point. + * + * @param datasetId the dataset id + * @param executionStep the execution step + * @param executionTimestamp the execution timestamp + * @return the execution point + */ ExecutionPoint findByDatasetIdAndExecutionStepAndExecutionTimestamp(String datasetId, String executionStep, LocalDateTime executionTimestamp); + /** + * Find first by dataset id and execution step order by execution timestamp desc optional. + * + * @param datasetId the dataset id + * @param executionStep the execution step + * @return the optional + */ Optional findFirstByDatasetIdAndExecutionStepOrderByExecutionTimestampDesc(String datasetId, String executionStep); + /** + * Delete by dataset id. + * + * @param datasetId the dataset id + */ @Modifying void deleteByDatasetId(String datasetId); diff --git a/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternOccurrenceRepository.java b/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternOccurrenceRepository.java index 532283af..96429c14 100644 --- a/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternOccurrenceRepository.java +++ b/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternOccurrenceRepository.java @@ -3,10 +3,25 @@ import eu.europeana.metis.sandbox.entity.problempatterns.RecordProblemPatternOccurrence; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +/** + * The interface Record problem pattern occurrence repository. + */ public interface RecordProblemPatternOccurrenceRepository extends JpaRepository { + /** + * Delete by record problem pattern execution point dataset id. + * + * @param datasetId the dataset id + */ @Modifying + @Query("DELETE FROM RecordProblemPatternOccurrence rppo WHERE "+ + " EXISTS (SELECT 1 FROM RecordProblemPattern rpp " + + " WHERE rppo.recordProblemPattern.recordProblemPatternId = rpp.recordProblemPatternId " + + " AND EXISTS (SELECT 1 FROM ExecutionPoint ep " + + " WHERE ep.executionPointId = rpp.executionPoint.executionPointId" + + " AND ep.datasetId= ?1))") void deleteByRecordProblemPatternExecutionPointDatasetId(String datasetId); } diff --git a/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternRepository.java b/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternRepository.java index 63fa9462..6c358d30 100644 --- a/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternRepository.java +++ b/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordProblemPatternRepository.java @@ -4,12 +4,29 @@ import eu.europeana.metis.sandbox.entity.problempatterns.RecordProblemPattern; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +/** + * The interface Record problem pattern repository. + */ public interface RecordProblemPatternRepository extends JpaRepository { + /** + * Find by execution point record problem pattern. + * + * @param executionPoint the execution point + * @return the record problem pattern + */ RecordProblemPattern findByExecutionPoint(ExecutionPoint executionPoint); + /** + * Delete by execution point dataset id. + * + * @param datasetId the dataset id + */ @Modifying + @Query("DELETE FROM RecordProblemPattern rpp WHERE EXISTS (SELECT 1 FROM ExecutionPoint ep " + + "WHERE rpp.executionPoint.executionPointId = ep.executionPointId AND ep.datasetId = ?1)") void deleteByExecutionPointDatasetId(String datasetId); } diff --git a/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordTitleRepository.java b/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordTitleRepository.java index 262f986e..70129744 100644 --- a/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordTitleRepository.java +++ b/src/main/java/eu/europeana/metis/sandbox/repository/problempatterns/RecordTitleRepository.java @@ -6,13 +6,36 @@ import java.util.List; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +/** + * The interface Record title repository. + */ public interface RecordTitleRepository extends JpaRepository { + /** + * Find all by execution point list. + * + * @param executionPoint the execution point + * @return the list + */ List findAllByExecutionPoint(ExecutionPoint executionPoint); + /** + * Delete by execution point. + * + * @param executionPoint the execution point + */ @Modifying void deleteByExecutionPoint(ExecutionPoint executionPoint); + + /** + * Delete by execution point dataset id. + * + * @param datasetId the dataset id + */ @Modifying + @Query("DELETE FROM RecordTitle rti WHERE EXISTS (SELECT 1 FROM ExecutionPoint ep " + + "WHERE rti.executionPoint.executionPointId = ep.executionPointId AND ep.datasetId = ?1)") void deleteByExecutionPointDatasetId(String datasetId); } diff --git a/src/main/java/eu/europeana/metis/sandbox/service/dataset/DatasetRemoverServiceImpl.java b/src/main/java/eu/europeana/metis/sandbox/service/dataset/DatasetRemoverServiceImpl.java index fb86d1d7..492ae03a 100644 --- a/src/main/java/eu/europeana/metis/sandbox/service/dataset/DatasetRemoverServiceImpl.java +++ b/src/main/java/eu/europeana/metis/sandbox/service/dataset/DatasetRemoverServiceImpl.java @@ -58,25 +58,25 @@ public void remove(int days) { datasets.forEach(dataset -> { try { // remove thumbnails (s3) - LOGGER.debug("Remove thumbnails for dataset id: [{}]", dataset); + LOGGER.info("Remove thumbnails for dataset id: [{}]", dataset); thumbnailStoreService.remove(dataset); // remove from mongo and solr - LOGGER.debug("Remove index for dataset id: [{}]", dataset); + LOGGER.info("Remove index for dataset id: [{}]", dataset); indexingService.remove(dataset); // remove from sandbox - LOGGER.debug("Remove record logs for dataset id: [{}]", dataset); + LOGGER.info("Remove record logs for dataset id: [{}]", dataset); recordLogService.remove(dataset); - LOGGER.debug("Remove records for dataset id: [{}]", dataset); + LOGGER.info("Remove records for dataset id: [{}]", dataset); recordService.remove(dataset); - LOGGER.debug("Remove logs for dataset id: [{}]", dataset); + LOGGER.info("Remove logs for dataset id: [{}]", dataset); datasetLogService.remove(dataset); - LOGGER.debug("Remove harvesting parameters for dataset id: [{}]", dataset); + LOGGER.info("Remove harvesting parameters for dataset id: [{}]", dataset); harvestingParameterService.remove(dataset); - LOGGER.debug("Remove problem pattern data associated with dataset id: [{}]", dataset); + LOGGER.info("Remove problem pattern data associated with dataset id: [{}]", dataset); problemPatternDataRemover.removeProblemPatternDataFromDatasetId(dataset); - LOGGER.debug("Remove dataset with id: [{}]", dataset); + LOGGER.info("Remove dataset with id: [{}]", dataset); datasetService.remove(dataset); - LOGGER.debug("Remove debias report with id: [{}]", dataset); + LOGGER.info("Remove debias report with id: [{}]", dataset); debiasStateService.cleanDeBiasReport(Integer.valueOf(dataset)); } catch (ServiceException e) { LOGGER.error("Failed to remove dataset [{}] ", dataset, e); diff --git a/src/main/java/eu/europeana/metis/sandbox/service/dataset/HarvestingParameterServiceImpl.java b/src/main/java/eu/europeana/metis/sandbox/service/dataset/HarvestingParameterServiceImpl.java index 0a619ca0..4c942a97 100644 --- a/src/main/java/eu/europeana/metis/sandbox/service/dataset/HarvestingParameterServiceImpl.java +++ b/src/main/java/eu/europeana/metis/sandbox/service/dataset/HarvestingParameterServiceImpl.java @@ -62,7 +62,7 @@ public HarvestingParameterEntity getDatasetHarvestingParameters(String datasetId public void remove(String datasetId) { requireNonNull(datasetId, "Dataset id must not be null"); try { - harvestingParameterRepository.deleteByDatasetId_DatasetId(Integer.parseInt(datasetId)); + harvestingParameterRepository.deleteByDatasetIdDatasetId(Integer.parseInt(datasetId)); } catch (RuntimeException e) { throw new ServiceException( format("Error removing records for dataset id: [%s]. ", datasetId), e); diff --git a/src/main/java/eu/europeana/metis/sandbox/service/debias/DeBiasStateServiceImpl.java b/src/main/java/eu/europeana/metis/sandbox/service/debias/DeBiasStateServiceImpl.java index 0e01e3f0..79675ef2 100644 --- a/src/main/java/eu/europeana/metis/sandbox/service/debias/DeBiasStateServiceImpl.java +++ b/src/main/java/eu/europeana/metis/sandbox/service/debias/DeBiasStateServiceImpl.java @@ -114,6 +114,7 @@ public void cleanDeBiasReport(Integer datasetId) { Objects.requireNonNull(datasetId, "Dataset id must not be null"); this.recordDeBiasDetailRepository.deleteByDebiasIdRecordIdDatasetId(datasetId.toString()); this.recordDeBiasMainRepository.deleteByRecordIdDatasetId(datasetId.toString()); + this.datasetDeBiasRepository.deleteByDatasetId(datasetId.toString()); } /** diff --git a/src/test/java/eu/europeana/metis/sandbox/service/dataset/HarvestParameterServiceImplTest.java b/src/test/java/eu/europeana/metis/sandbox/service/dataset/HarvestParameterServiceImplTest.java index e3418463..45acd9c6 100644 --- a/src/test/java/eu/europeana/metis/sandbox/service/dataset/HarvestParameterServiceImplTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/service/dataset/HarvestParameterServiceImplTest.java @@ -141,12 +141,12 @@ void getDatasetHarvestingParameters_expectSuccess(){ @Test void remove_expectSuccess(){ harvestingParameterService.remove("1"); - verify(harvestingParameterRepository).deleteByDatasetId_DatasetId(1); + verify(harvestingParameterRepository).deleteByDatasetIdDatasetId(1); } @Test void remove_expectFail(){ - doThrow(new RuntimeException("error")).when(harvestingParameterRepository).deleteByDatasetId_DatasetId(1); + doThrow(new RuntimeException("error")).when(harvestingParameterRepository).deleteByDatasetIdDatasetId(1); assertThrows(ServiceException.class, () -> harvestingParameterService.remove("1")); }