Skip to content

Commit

Permalink
Merge pull request #443 from bcgov/feature/GRAD2-2395
Browse files Browse the repository at this point in the history 
GRAD2-2395: task is complete
  • Loading branch information
@kamal-mohammed
kamal-mohammed authored Nov 10, 2023
2 parents 77db3e3 + e98831c commit ca76182
Show file tree
Hide file tree
Showing 11 changed files with 159 additions and 216 deletions.
129 changes: 0 additions & 129 deletions .github/workflows/deploy.to.dev.jinil.yml
View file

This file was deleted.

2 changes: 1 addition & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ ARG DEPENDENCY=/workspace/app/target/dependency
COPY --from=build ${DEPENDENCY}/BOOT-INF/lib /app/lib
COPY --from=build ${DEPENDENCY}/META-INF /app/META-INF
COPY --from=build ${DEPENDENCY}/BOOT-INF/classes /app
ENTRYPOINT ["java","-Duser.name=EDUC_GRAD_BATCH_GRADUATION_API","-Xms4000m","-Xmx4000m","-noverify","-XX:TieredStopAtLevel=1",\
ENTRYPOINT ["java","-Duser.name=EDUC_GRAD_BATCH_GRADUATION_API","-Xms3000m","-Xmx3800m","-noverify","-XX:TieredStopAtLevel=1",\
"-XX:+UseParallelGC","-XX:MinHeapFreeRatio=20","-XX:MaxHeapFreeRatio=40","-XX:GCTimeRatio=4",\
"-XX:AdaptiveSizePolicyWeight=90","-XX:MaxMetaspaceSize=500m","-XX:ParallelGCThreads=1",\
"-Djava.util.concurrent.ForkJoinPool.common.parallelism=1","-XX:CICompilerCount=2",\
Expand Down
12 changes: 3 additions & 9 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/AsyncConfig.java
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,18 @@
package ca.bc.gov.educ.api.batchgraduation.config;

import ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

@Configuration
@EnableAsync
@Profile("!test")
public class AsyncConfig {
@Bean(name = "asyncExecutor")
public TaskExecutor asyncExecutor(EducGradBatchGraduationApiConstants constants) {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(constants.getNumberOfPartitions());
executor.setMaxPoolSize(constants.getNumberOfPartitions());
executor.setThreadNamePrefix("async-");
executor.initialize();
return executor;
public TaskExecutor asyncExecutor() {
return new SimpleAsyncTaskExecutor("async-");
}
}
2 changes: 1 addition & 1 deletion api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public class BatchConfig {
public JobLauncher asyncJobLauncher(JobRepository jobRepository) throws Exception {
TaskExecutorJobLauncher jobLauncher = new TaskExecutorJobLauncher();
jobLauncher.setJobRepository(jobRepository);
jobLauncher.setTaskExecutor(new SimpleAsyncTaskExecutor());
jobLauncher.setTaskExecutor(new SimpleAsyncTaskExecutor("asyncTask-"));
jobLauncher.afterPropertiesSet();
return jobLauncher;
}
Expand Down
55 changes: 25 additions & 30 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionException;

Expand Down Expand Up @@ -87,7 +87,7 @@ public Step masterStepRegGrad(JobRepository jobRepository, EducGradBatchGraduati
.partitioner(graduationJobStep(jobRepository, skipListener).getName(), partitionerRegGrad())
.step(graduationJobStep(jobRepository, skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -97,7 +97,7 @@ public Step masterStepRegGradError(JobRepository jobRepository, EducGradBatchGra
.partitioner(graduationJobErrorStep(jobRepository, skipListener).getName(), partitionerRegGradRetry())
.step(graduationJobErrorStep(jobRepository, skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -107,7 +107,7 @@ public Step masterStepRegGradErrorRetry(JobRepository jobRepository, EducGradBat
.partitioner(graduationJobErrorRetryStep(jobRepository, skipListener).getName(), partitionerRegGradRetry())
.step(graduationJobErrorRetryStep(jobRepository, skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -230,7 +230,7 @@ public Step masterStepTvrRun(JobRepository jobRepository, EducGradBatchGraduatio
.partitioner(tvrJobStep(jobRepository,skipListener).getName(), partitionerTvrRun())
.step(tvrJobStep(jobRepository,skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -240,7 +240,7 @@ public Step masterStepTvrRunError(JobRepository jobRepository, EducGradBatchGrad
.partitioner(tvrJobErrorStep(jobRepository,skipListener).getName(), partitionerTvrRunRetry())
.step(tvrJobErrorStep(jobRepository,skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -250,7 +250,7 @@ public Step masterStepTvrRunErrorRetry(JobRepository jobRepository, EducGradBatc
.partitioner(tvrJobErrorRetryStep(jobRepository,skipListener).getName(), partitionerTvrRunRetry())
.step(tvrJobErrorRetryStep(jobRepository,skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -362,7 +362,7 @@ public Step masterStepSpcRegGrad(JobRepository jobRepository, EducGradBatchGradu
.partitioner(slaveSpcRegGradStep(jobRepository,skipListener).getName(), partitionerSpcRegGrad())
.step(slaveSpcRegGradStep(jobRepository,skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -372,7 +372,7 @@ public Step masterStepSpcRegGradError(JobRepository jobRepository, EducGradBatch
.partitioner(slaveSpcRegGradErrorStep(jobRepository,skipListener).getName(), partitionerSpcRegGradRetry())
.step(slaveSpcRegGradErrorStep(jobRepository,skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -382,7 +382,7 @@ public Step masterStepSpcRegGradErrorRetry(JobRepository jobRepository, EducGrad
.partitioner(slaveSpcRegGradErrorRetryStep(jobRepository,skipListener).getName(), partitionerSpcRegGradRetry())
.step(slaveSpcRegGradErrorRetryStep(jobRepository,skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -491,7 +491,7 @@ public Step masterStepSpcTvrRun(JobRepository jobRepository, EducGradBatchGradua
.partitioner(slaveStepSpcTvrRun(jobRepository,skipListener).getName(), partitionerSpcRegGrad())
.step(slaveStepSpcTvrRun(jobRepository,skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -501,7 +501,7 @@ public Step masterStepSpcTvrRunError(JobRepository jobRepository, EducGradBatchG
.partitioner(slaveStepSpcTvrRunError(jobRepository,skipListener).getName(), partitionerSpcRegGradRetry())
.step(slaveStepSpcTvrRunError(jobRepository,skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -511,7 +511,7 @@ public Step masterStepSpcTvrRunErrorRetry(JobRepository jobRepository, EducGradB
.partitioner(slaveStepSpcTvrRunErrorRetry(jobRepository,skipListener).getName(), partitionerSpcRegGradRetry())
.step(slaveStepSpcTvrRunErrorRetry(jobRepository,skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -669,7 +669,7 @@ public Step masterStepDisRun(JobRepository jobRepository, EducGradBatchGraduatio
.partitioner(slaveStepDisRun(jobRepository).getName(), partitionerDisRun())
.step(slaveStepDisRun(jobRepository))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -701,7 +701,7 @@ public Step masterStepDisRunYearly(JobRepository jobRepository, EducGradBatchGra
.partitioner(slaveStepDisRun(jobRepository).getName(), partitionerDisRunYearly())
.step(slaveStepDisRunYearly(jobRepository))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -727,7 +727,7 @@ public Step masterStepDisRunYearlyNonGrad(JobRepository jobRepository, EducGradB
.partitioner(slaveStepDisRunYearlyNonGradByMincode(jobRepository).getName(), partitionerDisRunYearlyNonGrad())
.step(slaveStepDisRunYearlyNonGradByMincode(jobRepository))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand All @@ -753,7 +753,7 @@ public Step masterStepDisRunSupplemental(JobRepository jobRepository, EducGradBa
.partitioner(slaveStepDisRun(jobRepository).getName(), partitionerDisRunSupplemental())
.step(slaveStepDisRun(jobRepository))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -785,7 +785,7 @@ public Step masterStepUserReqDisRun(JobRepository jobRepository, EducGradBatchGr
.partitioner(slaveStepDisRun(jobRepository).getName(), partitionerDisRunUserReq())
.step(slaveStepDisRun(jobRepository))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -847,7 +847,7 @@ public Step masterStepBlankUserReqDisRun(JobRepository jobRepository, EducGradBa
.partitioner(slaveStepBlankDisRun(jobRepository).getName(), partitionerDisRunBlankUserReq())
.step(slaveStepBlankDisRun(jobRepository))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -910,7 +910,7 @@ public Step masterStepPsiUserReqDisRun(JobRepository jobRepository, EducGradBatc
.partitioner(slaveStepPsiDisRun(jobRepository).getName(), partitionerDisRunPsiUserReq())
.step(slaveStepPsiDisRun(jobRepository))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -958,7 +958,7 @@ public Step masterStepCertRegen(JobRepository jobRepository, EducGradBatchGradua
.partitioner(certRegenJobStep(jobRepository, skipListener).getName(), partitionerCertRegen())
.step(certRegenJobStep(jobRepository, skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -1022,7 +1022,7 @@ public Step masterStepEdwSnapshotSchool(JobRepository jobRepository, EducGradBat
.partitioner(edwSnapshotSchoolJobStep(jobRepository, skipListener).getName(), partitionerEDWSnapshotSchool())
.step(edwSnapshotSchoolJobStep(jobRepository, skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -1072,7 +1072,7 @@ public Step masterStepEdwSnapshot(JobRepository jobRepository, EducGradBatchGrad
.partitioner(edwSnapshotJobStep(jobRepository, skipListener).getName(), partitionerEDWSnapshot())
.step(edwSnapshotJobStep(jobRepository, skipListener))
.gridSize(constants.getNumberOfPartitions())
.taskExecutor(taskExecutor(constants))
.taskExecutor(taskExecutor())
.build();
}

Expand Down Expand Up @@ -1155,12 +1155,7 @@ public JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor() {
}

@Bean
public TaskExecutor taskExecutor(EducGradBatchGraduationApiConstants constants) {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(constants.getNumberOfPartitions());
executor.setMaxPoolSize(constants.getNumberOfPartitions());
executor.setThreadNamePrefix("partition-");
executor.initialize();
return executor;
public TaskExecutor taskExecutor() {
return new SimpleAsyncTaskExecutor("partition-");
}
}
Loading

0 comments on commit ca76182

Please sign in to comment.