Skip to content

Commit

Permalink
Merge pull request #490 from bcgov/feature/GRAD2-2724
Browse files Browse the repository at this point in the history 
GRAD2-2724: task is completed.
  • Loading branch information
@infstar
infstar authored May 23, 2024
2 parents 7b6a857 + 4c6ee9e commit 7e83065
Show file tree
Hide file tree
Showing 22 changed files with 298 additions and 211 deletions.
6 changes: 0 additions & 6 deletions api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
<org.mapstruct.version>1.3.1.Final</org.mapstruct.version>
<springdoc.version>2.0.2</springdoc.version>
<log4j.version>2.18.0</log4j.version>
<resilience4j.version>2.0.2</resilience4j.version>
<shedlock.version>4.38.0</shedlock.version>
</properties>

Expand Down Expand Up @@ -129,11 +128,6 @@
<artifactId>modelmapper</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-spring-boot2</artifactId>
<version>${resilience4j.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -892,7 +892,7 @@ public Step masterStepCertRegen(JobRepository jobRepository, PlatformTransaction
@Bean
public Step certRegenJobStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SkipSQLTransactionExceptionsListener skipListener) {
return new StepBuilder("certRegenJobStep", jobRepository)
.<UUID, Integer>chunk(1, transactionManager)
.<StudentCredentialDistribution, Integer>chunk(1, transactionManager)
.reader(itemReaderCertRegen())
.processor(itemProcessorCertRegen())
.writer(itemWriterCertRegen())
Expand Down
39 changes: 30 additions & 9 deletions ...ava/ca/bc/gov/educ/api/batchgraduation/processor/RunCertificateRegenerationProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,21 @@

import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
import ca.bc.gov.educ.api.batchgraduation.model.GraduationStudentRecordDistribution;
import ca.bc.gov.educ.api.batchgraduation.model.StudentCredentialDistribution;
import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils;
import jakarta.validation.constraints.NotNull;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.lang.Nullable;

import java.io.IOException;
import java.util.UUID;

public class RunCertificateRegenerationProcessor implements ItemProcessor<UUID, Integer> {
public class RunCertificateRegenerationProcessor implements ItemProcessor<StudentCredentialDistribution, Integer> {

private static final Logger LOGGER = LoggerFactory.getLogger(RunCertificateRegenerationProcessor.class);

Expand All @@ -25,27 +30,43 @@ public class RunCertificateRegenerationProcessor implements ItemProcessor<UUID,
Long batchId;

@Override
public Integer process(UUID key) throws Exception {
GraduationStudentRecordDistribution stuRec = restUtils.getStudentData(key.toString());
if (stuRec != null) {
LOGGER.info("Processing partitionData: studentID = {}", stuRec.getStudentID());
@Nullable
public Integer process(@NotNull StudentCredentialDistribution item) throws Exception {
if (item.getStudentID() != null) {
LOGGER.info("Processing partitionData: studentID = {}", item.getStudentID());
summaryDTO.setBatchId(batchId);
try {
if (StringUtils.isBlank(item.getPen())) {
item.setPen(getPenNumber(item.getStudentID()));
}
summaryDTO.setProcessedCount(summaryDTO.getProcessedCount() + 1L);
Integer count = restUtils.runRegenerateStudentCertificate(stuRec.getPen());
Integer count = restUtils.runRegenerateStudentCertificate(item.getPen());
if (count > 0) {
restUtils.updateStudentGradRecord(key, batchId, "CERTREGEN");
restUtils.updateStudentGradRecord(item.getStudentID(), batchId, "CERTREGEN");
}
return count;
} catch(Exception e) {
LOGGER.error("Unexpected Error: {}", e.getLocalizedMessage());
summaryDTO.updateError(key,"GRAD-GRADUATION-API IS DOWN","Graduation API is unavailable at this moment");
summaryDTO.updateError(item.getStudentID(), "Unexpected Error in GRAD-GRADUATION-API", e.getLocalizedMessage());
summaryDTO.setProcessedCount(summaryDTO.getProcessedCount() - 1L);
LOGGER.info("Failed STU-ID:{} Errors:{}",key,summaryDTO.getErrors().size());
LOGGER.info("Failed STU-ID:{} Errors:{}",item.getStudentID(),summaryDTO.getErrors().size());
return null;
}
}
LOGGER.warn("Skipped STU-ID:null for pen# {},Errors:{}",item.getPen(),summaryDTO.getErrors().size());
return null;
}

private String getPenNumber(UUID studentID) throws IOException {
GraduationStudentRecordDistribution stuRec = null;
try {
stuRec = restUtils.getStudentData(studentID.toString());
} catch (RuntimeException se) {
LOGGER.error("Unknown exception occurred in PEN API: {}", se.getLocalizedMessage());
}
if (stuRec != null) {
return stuRec.getPen();
}
throw new IOException(String.format("Student not found in PEN API: STU-ID:%s", studentID));
}
}
27 changes: 16 additions & 11 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,33 +3,38 @@
import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj;
import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.item.ItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;

import java.util.List;
import java.util.UUID;

public abstract class BaseReader implements ItemReader<UUID> {
@Slf4j
public abstract class BaseReader {

@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
@Autowired
RestUtils restUtils;

@Value("#{stepExecutionContext['index']}")
protected Integer nxtStudentForProcessing;

@Value("#{stepExecutionContext['data']}")
protected List<UUID> studentList;

@Value("#{stepExecutionContext['summary']}")
AlgorithmSummaryDTO summaryDTO;

@Value("#{stepExecution.jobExecution}")
JobExecution jobExecution;

protected void aggregate(String summaryContextName) {
AlgorithmSummaryDTO totalSummaryDTO = (AlgorithmSummaryDTO)jobExecution.getExecutionContext().get(summaryContextName);
if (totalSummaryDTO == null) {
totalSummaryDTO = new AlgorithmSummaryDTO();
jobExecution.getExecutionContext().put(summaryContextName, totalSummaryDTO);
}
totalSummaryDTO.setBatchId(summaryDTO.getBatchId());
totalSummaryDTO.setReadCount(totalSummaryDTO.getReadCount() + summaryDTO.getReadCount());
totalSummaryDTO.setProcessedCount(totalSummaryDTO.getProcessedCount() + summaryDTO.getProcessedCount());
totalSummaryDTO.getErrors().putAll(summaryDTO.getErrors());
}

protected void fetchAccessToken() {
log.info("Fetching the access token from KeyCloak API");
ResponseObj res = restUtils.getTokenResponseObject();
if (res != null) {
summaryDTO.setAccessToken(res.getAccess_token());
Expand Down
38 changes: 38 additions & 0 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseStudentReader.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package ca.bc.gov.educ.api.batchgraduation.reader;

import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj;
import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.item.ItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;

import java.util.List;
import java.util.UUID;

public abstract class BaseStudentReader implements ItemReader<UUID> {

@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
@Autowired
RestUtils restUtils;

@Value("#{stepExecutionContext['index']}")
protected Integer nxtStudentForProcessing;

@Value("#{stepExecutionContext['data']}")
protected List<UUID> studentList;

@Value("#{stepExecutionContext['summary']}")
AlgorithmSummaryDTO summaryDTO;

@Value("#{stepExecution.jobExecution}")
JobExecution jobExecution;

protected void fetchAccessToken() {
ResponseObj res = restUtils.getTokenResponseObject();
if (res != null) {
summaryDTO.setAccessToken(res.getAccess_token());
}
}
}
2 changes: 1 addition & 1 deletion ...ava/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import java.util.UUID;

public class RecalculateProjectedGradRunErrorReader extends BaseReader {
public class RecalculateProjectedGradRunErrorReader extends BaseStudentReader {

private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateProjectedGradRunErrorReader.class);

Expand Down
2 changes: 1 addition & 1 deletion ...a/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorRetryReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import java.util.UUID;

public class RecalculateProjectedGradRunErrorRetryReader extends BaseReader {
public class RecalculateProjectedGradRunErrorRetryReader extends BaseStudentReader {

private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateProjectedGradRunErrorRetryReader.class);

Expand Down
2 changes: 1 addition & 1 deletion ...ain/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import java.util.UUID;

public class RecalculateProjectedGradRunReader extends BaseReader {
public class RecalculateProjectedGradRunReader extends BaseStudentReader {

private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateProjectedGradRunReader.class);

Expand Down
2 changes: 1 addition & 1 deletion ...rc/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import java.util.UUID;

public class RecalculateStudentErrorReader extends BaseReader {
public class RecalculateStudentErrorReader extends BaseStudentReader {

private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateStudentErrorReader.class);

Expand Down
2 changes: 1 addition & 1 deletion ...in/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorRetryReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import java.util.UUID;

public class RecalculateStudentErrorRetryReader extends BaseReader {
public class RecalculateStudentErrorRetryReader extends BaseStudentReader {

private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateStudentErrorRetryReader.class);

Expand Down
2 changes: 1 addition & 1 deletion api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import java.util.UUID;

public class RecalculateStudentReader extends BaseReader {
public class RecalculateStudentReader extends BaseStudentReader {

private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateStudentReader.class);

Expand Down
38 changes: 24 additions & 14 deletions ...main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateCertificatePartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import reactor.core.publisher.Mono;

import java.util.*;
import java.util.stream.Collectors;

import static ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants.SEARCH_REQUEST;

Expand Down Expand Up @@ -43,37 +42,39 @@ public Map<String, ExecutionContext> partition(int gridSize) {
if(parallelDTO != null) {
credentialList.addAll(parallelDTO.certificateList());
}
} else if (certificateRegenerationRequest.getPens() != null && !certificateRegenerationRequest.getPens().isEmpty()) {
certificateRegenerationRequest.getPens().forEach(p -> {
StudentCredentialDistribution scd = new StudentCredentialDistribution();
scd.setPen(p);
scd.setStudentID(getStudentIDByPen(p));
if (scd.getStudentID() != null)
credentialList.add(scd);
});
} else {
credentialList.addAll(getStudentsForUserReqRun(certificateRegenerationRequest));
}

Set<UUID> studentSet = new HashSet<>();
if(!credentialList.isEmpty()) {
studentSet = credentialList.stream().map(StudentCredentialDistribution::getStudentID).collect(Collectors.toSet());
}

List<UUID> studentList = new ArrayList<>(studentSet);
if(!studentList.isEmpty()) {
createTotalSummaryDTO("regenCertSummaryDTO");
updateBatchJobHistory(createBatchJobHistory(), (long) studentList.size());
int partitionSize = studentList.size()/gridSize + 1;
List<List<UUID>> partitions = new LinkedList<>();
for (int i = 0; i < studentList.size(); i += partitionSize) {
partitions.add(studentList.subList(i, Math.min(i + partitionSize, studentList.size())));
updateBatchJobHistory(createBatchJobHistory(), (long) credentialList.size());
int partitionSize = credentialList.size()/gridSize + 1;
List<List<StudentCredentialDistribution>> partitions = new LinkedList<>();
for (int i = 0; i < credentialList.size(); i += partitionSize) {
partitions.add(credentialList.subList(i, Math.min(i + partitionSize, credentialList.size())));
}
Map<String, ExecutionContext> map = new HashMap<>(partitions.size());
for (int i = 0; i < partitions.size(); i++) {
ExecutionContext executionContext = new ExecutionContext();
AlgorithmSummaryDTO summaryDTO = new AlgorithmSummaryDTO();
List<UUID> data = partitions.get(i);
List<StudentCredentialDistribution> data = partitions.get(i);
executionContext.put("data", data);
summaryDTO.setReadCount(data.size());
executionContext.put("summary", summaryDTO);
executionContext.put("index",0);
String key = "partition" + i;
map.put(key, executionContext);
}
LOGGER.info("Found {} in total running on {} partitions",studentList.size(),map.size());
LOGGER.info("Found {} in total running on {} partitions",credentialList.size(),map.size());
return map;
}
LOGGER.info("No Certificates are found to process them with null distribution date");
Expand All @@ -89,6 +90,15 @@ private List<StudentCredentialDistribution> getStudentsForUserReqRun(Certificate
}
}

private UUID getStudentIDByPen(String pen) {
try {
String accessToken = restUtils.fetchAccessToken();
return restUtils.getStudentIDByPen(pen, accessToken);
} catch (Exception e) {
return null;
}
}

@Override
protected JobExecution getJobExecution() {
return context;
Expand Down
42 changes: 19 additions & 23 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateCertificateReader.java
View file
Original file line number Diff line number Diff line change
@@ -1,41 +1,37 @@
package ca.bc.gov.educ.api.batchgraduation.reader;

import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
import ca.bc.gov.educ.api.batchgraduation.model.StudentCredentialDistribution;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.beans.factory.annotation.Value;

import java.util.UUID;
import java.util.List;

public class RegenerateCertificateReader extends BaseReader {
public class RegenerateCertificateReader extends BaseReader implements ItemReader<StudentCredentialDistribution> {

private static final Logger LOGGER = LoggerFactory.getLogger(RegenerateCertificateReader.class);

@Value("#{stepExecutionContext['index']}")
private Integer nxtCredentialForProcessing;

@Value("#{stepExecutionContext['data']}")
List<StudentCredentialDistribution> credentialList;

@Override
public UUID read() throws Exception {
public StudentCredentialDistribution read() throws Exception {
fetchAccessToken();
summaryDTO.setReadCount(studentList.size());
summaryDTO.setReadCount(credentialList.size());

UUID nextStudent = null;
StudentCredentialDistribution nextCredential = null;

if (nxtStudentForProcessing < studentList.size()) {
nextStudent = studentList.get(nxtStudentForProcessing);
LOGGER.info("StudID:{} - {} of {}", nextStudent, nxtStudentForProcessing + 1, summaryDTO.getReadCount());
nxtStudentForProcessing++;
if (nxtCredentialForProcessing < credentialList.size()) {
nextCredential = credentialList.get(nxtCredentialForProcessing);
LOGGER.info("StudID:{} - {} of {}", nextCredential, nxtCredentialForProcessing + 1, summaryDTO.getReadCount());
nxtCredentialForProcessing++;
} else {
aggregate("regenCertSummaryDTO");
}
return nextStudent;
}

public void aggregate(String summaryContextName) {
AlgorithmSummaryDTO totalSummaryDTO = (AlgorithmSummaryDTO)jobExecution.getExecutionContext().get(summaryContextName);
if (totalSummaryDTO == null) {
totalSummaryDTO = new AlgorithmSummaryDTO();
jobExecution.getExecutionContext().put(summaryContextName, totalSummaryDTO);
}
totalSummaryDTO.setBatchId(summaryDTO.getBatchId());
totalSummaryDTO.setReadCount(totalSummaryDTO.getReadCount() + summaryDTO.getReadCount());
totalSummaryDTO.setProcessedCount(totalSummaryDTO.getProcessedCount() + summaryDTO.getProcessedCount());
totalSummaryDTO.getErrors().putAll(summaryDTO.getErrors());
return nextCredential;
}
}
2 changes: 1 addition & 1 deletion api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpecialGradRunStudentReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import java.util.UUID;

public class SpecialGradRunStudentReader extends BaseReader {
public class SpecialGradRunStudentReader extends BaseStudentReader {

private static final Logger LOGGER = LoggerFactory.getLogger(SpecialGradRunStudentReader.class);

Expand Down
Loading

0 comments on commit 7e83065

Please sign in to comment.