From df7c68788926f1f322e51ca83ab9a28386e15ea9 Mon Sep 17 00:00:00 2001
From: Jinil Sung <infstar@gmail.com>
Date: Tue, 19 Dec 2023 08:42:53 -0800
Subject: [PATCH] GRAD2-2430: task is complete.

GRAD2-2430: task is complete.
---
 .../config/BatchJobLauncher.java              | 13 +++++++
 ...atchGradAlgorithmJobHistoryRepository.java |  8 +++++
 .../BatchGradAlgorithmStudentRepository.java  |  9 ++++-
 .../BatchJobExecutionRepository.java          | 31 ++++++++++++++++
 .../BatchStepExecutionRepository.java         | 20 +++++++++++
 ...udentCredentialDistributionRepository.java | 10 ++++--
 .../service/GradDashboardService.java         | 36 ++++++++++++++++++-
 .../EducGradBatchGraduationApiConstants.java  |  3 ++
 api/src/main/resources/application.yaml       |  3 ++
 .../service/GradDashboardServiceTest.java     | 22 ++++++++++++
 api/src/test/resources/application.yaml       |  3 ++
 11 files changed, 153 insertions(+), 5 deletions(-)

diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobLauncher.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobLauncher.java
index c2789099..8c7ec12f 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobLauncher.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobLauncher.java
@@ -146,4 +146,17 @@ public void refreshUserScheduledQueue() {
 
         LOGGER.info(BATCH_ENDED);
     }
+
+    @Scheduled(cron = "${batch.purge-old-records.cron}")
+    @SchedulerLock(name = "PurgeOldRecordsLock",
+            lockAtLeastFor = "PT1H", lockAtMostFor = "PT1H") //midnight job so lock for an hour
+    public void purgeOldRecords() {
+        LockAssert.assertLocked();
+        try {
+            this.gradDashboardService.purgeOldBatchHistoryRecords();
+            this.gradDashboardService.purgeOldSpringMetaDataRecords();
+        } catch (Exception e) {
+            LOGGER.error(ERROR_MSG, e.getLocalizedMessage());
+        }
+    }
 }
diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmJobHistoryRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmJobHistoryRepository.java
index 6a8588b7..dbab9180 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmJobHistoryRepository.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmJobHistoryRepository.java
@@ -1,13 +1,17 @@
 package ca.bc.gov.educ.api.batchgraduation.repository;
 
+import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
 
 import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmJobHistoryEntity;
+import org.springframework.transaction.annotation.Transactional;
 
 @Repository
 public interface BatchGradAlgorithmJobHistoryRepository extends JpaRepository<BatchGradAlgorithmJobHistoryEntity, UUID> {
@@ -16,4 +20,8 @@ public interface BatchGradAlgorithmJobHistoryRepository extends JpaRepository<Ba
 
     Optional<BatchGradAlgorithmJobHistoryEntity> findByJobExecutionId(Long batchId);
 
+    @Transactional
+    @Modifying
+    @Query("delete from BatchGradAlgorithmJobHistoryEntity where createDate <= :createDate")
+    void deleteByCreateDateBefore(LocalDateTime createDate);
 }
diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmStudentRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmStudentRepository.java
index 17603024..f2e9ee3e 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmStudentRepository.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmStudentRepository.java
@@ -8,7 +8,9 @@
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
 
+import java.time.LocalDateTime;
 import java.util.*;
 
 @Repository
@@ -58,5 +60,10 @@ void copyAllGradAlgorithmStudents(
             "and status = 'COMPLETED'\n" +
             "group by graduation_program_code\n" +
             "order by graduation_program_code desc", nativeQuery = true)
-    List<Object[]> getGraduationProgramCounts( @Param("batchId") Long batchId);
+    List<Object[]> getGraduationProgramCounts(@Param("batchId") Long batchId);
+
+    @Transactional
+    @Modifying
+    @Query("delete from BatchGradAlgorithmStudentEntity where createDate <= :createDate")
+    void deleteByCreateDateBefore(LocalDateTime createDate);
 }
diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java
index 81a3f1f5..6f92679e 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java
@@ -4,11 +4,42 @@
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.time.LocalDateTime;
 
 @Repository
 public interface BatchJobExecutionRepository extends JpaRepository<BatchJobExecutionEntity, Long> {
 
     Page<BatchJobExecutionEntity> findAllByOrderByCreateTimeDesc(Pageable page);
 
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BATCH_JOB_EXECUTION_PARAMS WHERE JOB_EXECUTION_ID IN (\n" +
+            "SELECT JOB_EXECUTION_ID FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate);",
+            nativeQuery = true)
+    void deleteBatchParamsByCreateTimeBefore(LocalDateTime createDate);
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BATCH_JOB_EXECUTION_CONTEXT WHERE JOB_EXECUTION_ID IN (\n" +
+            "SELECT JOB_EXECUTION_ID FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate);",
+            nativeQuery = true)
+    void deleteBatchContextsByCreateTimeBefore(LocalDateTime createDate);
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BATCH_JOB_INSTANCE WHERE JOB_INSTANCE_ID NOT IN (\n" +
+            "SELECT JOB_INSTANCE_ID FROM BATCH_JOB_EXECUTION);",
+            nativeQuery = true)
+    void deleteBatchInstancesNotInBatchJobs();
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate;",
+            nativeQuery = true)
+    void deleteBatchJobsByCreateTimeBefore(LocalDateTime createDate);
 }
diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchStepExecutionRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchStepExecutionRepository.java
index 0d4adcfa..2194ce72 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchStepExecutionRepository.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchStepExecutionRepository.java
@@ -2,8 +2,12 @@
 
 import ca.bc.gov.educ.api.batchgraduation.entity.BatchStepExecutionEntity;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
 
+import java.time.LocalDateTime;
 import java.util.List;
 
 @Repository
@@ -11,4 +15,20 @@ public interface BatchStepExecutionRepository extends JpaRepository<BatchStepExe
 
     List<BatchStepExecutionEntity> findByJobExecutionIdOrderByEndTimeDesc(Long jobExecutionId);
 
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BATCH_STEP_EXECUTION_CONTEXT WHERE STEP_EXECUTION_ID IN (\n" +
+            "SELECT BATCH_STEP_EXECUTION.STEP_EXECUTION_ID FROM BATCH_STEP_EXECUTION WHERE JOB_EXECUTION_ID IN (\n" +
+            "    SELECT JOB_EXECUTION_ID FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate\n" +
+            "));\n",
+            nativeQuery = true)
+    void deleteBatchStepContextsByCreateTimeBefore(LocalDateTime createDate);
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BATCH_STEP_EXECUTION WHERE JOB_EXECUTION_ID IN (\n" +
+            "    SELECT JOB_EXECUTION_ID FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate);",
+            nativeQuery = true)
+    void deleteBatchStepsByCreateTimeBefore(LocalDateTime createDate);
+
 }
diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/StudentCredentialDistributionRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/StudentCredentialDistributionRepository.java
index 3b5a152d..6ab5fc13 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/StudentCredentialDistributionRepository.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/StudentCredentialDistributionRepository.java
@@ -1,16 +1,15 @@
 package ca.bc.gov.educ.api.batchgraduation.repository;
 
-import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmStudentEntity;
 import ca.bc.gov.educ.api.batchgraduation.entity.StudentCredentialDistributionEntity;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
-import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
 
-import java.util.Date;
+import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
@@ -37,4 +36,9 @@ public interface StudentCredentialDistributionRepository extends JpaRepository<S
 
     long countAllByJobExecutionId(Long batchId);
 
+    @Transactional
+    @Modifying
+    @Query("delete from StudentCredentialDistributionEntity where createDate <= :createDate")
+    void deleteByCreateDateBefore(LocalDateTime createDate);
+
 }
diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardService.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardService.java
index 30f55394..3aaacb28 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardService.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardService.java
@@ -6,6 +6,7 @@
 import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils;
 import ca.bc.gov.educ.api.batchgraduation.transformer.BatchGradAlgorithmJobHistoryTransformer;
 import ca.bc.gov.educ.api.batchgraduation.transformer.BatchProcessingTransformer;
+import ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
@@ -29,15 +30,20 @@ public class GradDashboardService extends GradService {
 	private final BatchStepExecutionRepository batchStepExecutionRepository;
 	private final BatchProcessingTransformer batchProcessingTransformer;
 	private final BatchProcessingRepository batchProcessingRepository;
+	private final StudentCredentialDistributionRepository studentCredentialDistributionRepository;
 	private final RestUtils restUtils;
 
+	private final EducGradBatchGraduationApiConstants constants;
+
     public GradDashboardService(BatchGradAlgorithmJobHistoryRepository batchGradAlgorithmJobHistoryRepository,
 								BatchGradAlgorithmJobHistoryTransformer batchGradAlgorithmJobHistoryTransformer,
 								RestUtils restUtils,
 								BatchJobExecutionRepository batchJobExecutionRepository,
 								BatchStepExecutionRepository batchStepExecutionRepository,
 								BatchProcessingRepository batchProcessingRepository,BatchProcessingTransformer batchProcessingTransformer,
-								BatchGradAlgorithmStudentRepository batchGradAlgorithmStudentRepository) {
+								BatchGradAlgorithmStudentRepository batchGradAlgorithmStudentRepository,
+								StudentCredentialDistributionRepository studentCredentialDistributionRepository,
+								EducGradBatchGraduationApiConstants constants) {
 		this.batchGradAlgorithmJobHistoryRepository = batchGradAlgorithmJobHistoryRepository;
 		this.batchGradAlgorithmJobHistoryTransformer = batchGradAlgorithmJobHistoryTransformer;
 		this.batchProcessingTransformer = batchProcessingTransformer;
@@ -45,7 +51,9 @@ public GradDashboardService(BatchGradAlgorithmJobHistoryRepository batchGradAlgo
 		this.batchJobExecutionRepository = batchJobExecutionRepository;
 		this.batchStepExecutionRepository = batchStepExecutionRepository;
 		this.batchProcessingRepository = batchProcessingRepository;
+		this.studentCredentialDistributionRepository = studentCredentialDistributionRepository;
 		this.restUtils = restUtils;
+		this.constants = constants;
 	}
 
     @Transactional(readOnly = true)
@@ -198,4 +206,30 @@ private void updateBatchJobStatus(BatchGradAlgorithmJobHistory batchJobHistory,
 		batchGradAlgorithmJobHistoryRepository.save(entity);
 		batchJobHistory.setStatus(BatchStatusEnum.FAILED.toString());
 	}
+
+	@Transactional
+	public void purgeOldBatchHistoryRecords() {
+		final LocalDateTime createDateToCompare = this.calculateCreateDateBasedOnStaleEventInDays();
+		this.batchGradAlgorithmStudentRepository.deleteByCreateDateBefore(createDateToCompare);
+		this.studentCredentialDistributionRepository.deleteByCreateDateBefore(createDateToCompare);
+		this.batchGradAlgorithmJobHistoryRepository.deleteByCreateDateBefore(createDateToCompare);
+	}
+
+	@Transactional
+	public void purgeOldSpringMetaDataRecords() {
+		final LocalDateTime createDateToCompare = this.calculateCreateDateBasedOnStaleEventInDays();
+		this.batchStepExecutionRepository.deleteBatchStepContextsByCreateTimeBefore(createDateToCompare);
+		this.batchStepExecutionRepository.deleteBatchStepsByCreateTimeBefore(createDateToCompare);
+
+		this.batchJobExecutionRepository.deleteBatchParamsByCreateTimeBefore(createDateToCompare);
+		this.batchJobExecutionRepository.deleteBatchContextsByCreateTimeBefore(createDateToCompare);
+		this.batchJobExecutionRepository.deleteBatchJobsByCreateTimeBefore(createDateToCompare);
+		this.batchJobExecutionRepository.deleteBatchInstancesNotInBatchJobs();
+	}
+
+
+	private LocalDateTime calculateCreateDateBasedOnStaleEventInDays() {
+		final LocalDateTime currentTime = LocalDateTime.now();
+		return currentTime.minusDays(this.constants.getRecordsStaleInDays());
+	}
 }
diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/util/EducGradBatchGraduationApiConstants.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/util/EducGradBatchGraduationApiConstants.java
index 9fb53945..bf5e6de8 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/util/EducGradBatchGraduationApiConstants.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/util/EducGradBatchGraduationApiConstants.java
@@ -273,4 +273,7 @@ public class EducGradBatchGraduationApiConstants {
     @Value("${splunk.log-helper.enabled}")
     private boolean splunkLogHelperEnabled;
 
+    @Value("${batch.purge-old-records.staleInDays}")
+    private int recordsStaleInDays;
+
 }
diff --git a/api/src/main/resources/application.yaml b/api/src/main/resources/application.yaml
index 4014ad4e..e47d58a0 100644
--- a/api/src/main/resources/application.yaml
+++ b/api/src/main/resources/application.yaml
@@ -158,6 +158,9 @@ batch:
       routines:
         lockAtLeastFor: ${CRON_SYSTEM_SCHEDULED_ROUTINES_LOCK_AT_LEAST_FOR}
         lockAtMostFor: ${CRON_SYSTEM_SCHEDULED_ROUTINES_LOCK_AT_MOST_FOR}
+  purge-old-records:
+    cron: ${CRON_SCHEDULED_PURGE_OLD_RECORDS}
+    staleInDays: ${RECORDS_STALE_IN_DAYS}
 
 #Endpoints
 endpoint:
diff --git a/api/src/test/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardServiceTest.java b/api/src/test/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardServiceTest.java
index 8be796a2..0a7d8a95 100644
--- a/api/src/test/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardServiceTest.java
+++ b/api/src/test/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardServiceTest.java
@@ -228,4 +228,26 @@ public void testGetBatchSummary() {
         assertThat(res.getBatchJobList()).hasSize(1);
     }
 
+    @Test
+    public void testPurgeOldBatchHistoryRecords() {
+        boolean isExceptionThrown = false;
+        try {
+            gradDashboardService.purgeOldBatchHistoryRecords();
+        } catch (Exception e) {
+            isExceptionThrown = true;
+        }
+        assertThat(isExceptionThrown).isFalse();
+    }
+
+    @Test
+    public void testPurgeOldSpringMetaDataRecords() {
+        boolean isExceptionThrown = false;
+        try {
+            gradDashboardService.purgeOldSpringMetaDataRecords();
+        } catch (Exception e) {
+            isExceptionThrown = true;
+        }
+        assertThat(isExceptionThrown).isFalse();
+    }
+
 }
diff --git a/api/src/test/resources/application.yaml b/api/src/test/resources/application.yaml
index 67671136..67ef0b8d 100644
--- a/api/src/test/resources/application.yaml
+++ b/api/src/test/resources/application.yaml
@@ -87,6 +87,9 @@ batch:
       routines:
         lockAtLeastFor: PT1M
         lockAtMostFor: PT600M
+  purge-old-records:
+    cron: 0 30 0 * * *
+    staleInDays: 90
 
 #Endpoints
 endpoint: