Skip to content

Commit

Permalink
Merge pull request #475 from bcgov/feature/reportGen
Browse files Browse the repository at this point in the history
Working enrollment report.
  • Loading branch information
mightycox authored Feb 23, 2024
2 parents 859ae3e + f69740c commit 94ebf13
Show file tree
Hide file tree
Showing 21 changed files with 601 additions and 133 deletions.
4 changes: 4 additions & 0 deletions api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,10 @@
<id>central</id>
<url>https://repo1.maven.org/maven2</url>
</repository>
<repository>
<id>localrepository</id>
<url>file://${project.basedir}/src/main/resources/bcsans.jar</url>
</repository>
</repositories>
<profiles>
<profile>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
public interface ReportGenerationEndpoint {

@GetMapping("/{sdcSchoolCollectionID}/{reportTypeCode}")
@PreAuthorize("hasAuthority('SCOPE_READ_SDC_SCHOOL_COLLECTION')")
@PreAuthorize("hasAuthority('SCOPE_READ_SDC_COLLECTION')")
@Transactional(readOnly = true)
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "NOT FOUND")})
byte[] generateSDCReport(@PathVariable("sdcSchoolCollectionID") UUID sdcSchoolCollectionID, @PathVariable("reportTypeCode") String reportTypeCode);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public void setGradeCodes(Optional<School> school) {
public void setComparisonValues(SdcSchoolCollectionEntity sdcSchoolCollectionEntity, List<HeadcountHeader> headcountHeaderList) {
UUID previousCollectionID = getPreviousSeptemberCollectionID(sdcSchoolCollectionEntity);

List<EnrollmentHeadcountResult> previousCollectionRawData = sdcSchoolCollectionStudentRepository.getEnrollmentHeadcountsBySchoolId(previousCollectionID);
List<EnrollmentHeadcountResult> previousCollectionRawData = sdcSchoolCollectionStudentRepository.getEnrollmentHeadcountsBySdcSchoolCollectionId(previousCollectionID);
HeadcountResultsTable previousCollectionData = convertHeadcountResults(previousCollectionRawData);
List<HeadcountHeader> previousHeadcountHeaderList = Arrays.asList(getStudentsHeadcountTotals(previousCollectionData), getGradesHeadcountTotals(previousCollectionData));
setComparisonValues(headcountHeaderList, previousHeadcountHeaderList);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,51 @@

import ca.bc.gov.educ.studentdatacollection.api.exception.EntityNotFoundException;
import ca.bc.gov.educ.studentdatacollection.api.exception.StudentDataCollectionAPIRuntimeException;
import ca.bc.gov.educ.studentdatacollection.api.mappers.v1.SdcSchoolCollectionMapper;
import ca.bc.gov.educ.studentdatacollection.api.model.v1.SdcSchoolCollectionEntity;
import ca.bc.gov.educ.studentdatacollection.api.properties.ApplicationProperties;
import ca.bc.gov.educ.studentdatacollection.api.repository.v1.SdcSchoolCollectionRepository;
import ca.bc.gov.educ.studentdatacollection.api.repository.v1.SdcSchoolCollectionStudentRepository;
import ca.bc.gov.educ.studentdatacollection.api.rest.RestUtils;
import ca.bc.gov.educ.studentdatacollection.api.struct.v1.District;
import ca.bc.gov.educ.studentdatacollection.api.struct.v1.School;
import ca.bc.gov.educ.studentdatacollection.api.struct.v1.headcounts.EnrollmentHeadcountResult;
import ca.bc.gov.educ.studentdatacollection.api.struct.v1.reports.GradeEnrollementFTENode;
import ca.bc.gov.educ.studentdatacollection.api.struct.v1.reports.GradeEnrollementFTEReportGradesNode;
import ca.bc.gov.educ.studentdatacollection.api.struct.v1.reports.GradeEnrollementFTEReportNode;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.util.concurrent.AtomicDouble;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import net.sf.jasperreports.engine.*;
import net.sf.jasperreports.engine.query.JsonQueryExecuterFactory;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;

@Service
@Slf4j
public class ReportGenerationService {

private final SdcSchoolCollectionRepository sdcSchoolCollectionRepository;
private final SdcSchoolCollectionStudentRepository sdcSchoolCollectionStudentRepository;
private JasperReport gradeEnrollmentFTEReport;
private final RestUtils restUtils;
private ObjectWriter objectWriter = new ObjectMapper().writer().withDefaultPrettyPrinter();
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final String DOUBLE_FORMAT = "%,.4f";

public ReportGenerationService(SdcSchoolCollectionRepository sdcSchoolCollectionRepository) {
public ReportGenerationService(SdcSchoolCollectionRepository sdcSchoolCollectionRepository, SdcSchoolCollectionStudentRepository sdcSchoolCollectionStudentRepository, RestUtils restUtils) {
this.sdcSchoolCollectionRepository = sdcSchoolCollectionRepository;
this.sdcSchoolCollectionStudentRepository = sdcSchoolCollectionStudentRepository;
this.restUtils = restUtils;
}

@PostConstruct
Expand All @@ -41,7 +60,7 @@ private void initialize() {

private void compileJasperReports(){
try {
InputStream input = getClass().getResourceAsStream("/gradeEnrollmentFTEReport.jrxml");
InputStream input = getClass().getResourceAsStream("/reports/gradeEnrollmentFTEReport.jrxml");
gradeEnrollmentFTEReport = JasperCompileManager.compileReport(input);
} catch (JRException e) {
throw new StudentDataCollectionAPIRuntimeException("Compiling Jasper reports has failed :: " + e.getMessage());
Expand All @@ -54,23 +73,101 @@ public byte[] generateGradeEnrollementFTEReport(UUID collectionID){
SdcSchoolCollectionEntity sdcSchoolCollectionEntity = sdcSchoolCollectionEntityOptional.orElseThrow(() ->
new EntityNotFoundException(SdcSchoolCollectionEntity.class, "Collection by Id", collectionID.toString()));

var gradeEnrollmentList = sdcSchoolCollectionStudentRepository.getEnrollmentHeadcountsBySdcSchoolCollectionId(sdcSchoolCollectionEntity.getSdcSchoolCollectionID());

Map<String, Object> params = new HashMap<>();
params.put(JsonQueryExecuterFactory.JSON_DATE_PATTERN, "yyyy-MM-dd");
params.put(JsonQueryExecuterFactory.JSON_NUMBER_PATTERN, "#,##0.##");
params.put(JsonQueryExecuterFactory.JSON_LOCALE, Locale.ENGLISH);
params.put(JRParameter.REPORT_LOCALE, Locale.US);
var schoolColl = SdcSchoolCollectionMapper.mapper.toStructure(sdcSchoolCollectionEntity);
String json = "{\"report\": { \"collectionNameAndYear\": \"September 2023 Collection\", \"reportGeneratedDate\": \"Report Date: 2023-10-03\", \"districtNumberAndName\": \"085 - Vancouver Island\", \"schoolMincodeAndName\": \"08585023 - Eagle View Elementary School\", \"grades\": [{ \"code\": \"KF\", \"schoolAgedHeadcount\": \"100\", \"schoolAgedEligibleForFTE\": \"0\", \"schoolAgedFTETotal\": \"100.0000\", \"adultHeadcount\": \"100\", \"adultEligibleForFTE\": \"0\", \"adultFTETotal\": \"100.0000\", \"allStudentHeadcount\": \"90\", \"allStudentEligibleForFTE\": \"10\", \"allStudentFTETotal\": \"90.0000\"},{ \"code\": \"01\", \"schoolAgedHeadcount\": \"100\", \"schoolAgedEligibleForFTE\": \"50\", \"schoolAgedFTETotal\": \"100.0000\", \"adultHeadcount\": \"60\", \"adultEligibleForFTE\": \"5\", \"adultFTETotal\": \"150.0000\", \"allStudentHeadcount\": \"100\", \"allStudentEligibleForFTE\": \"0\", \"allStudentFTETotal\": \"100.0000\", \"totalCountsCode\": \"Total\", \"totalSchoolAgedHeadcount\": \"300\"} ] }}";
// String json = objectWriter.writeValueAsString(schoolColl);
InputStream targetStream = new ByteArrayInputStream(json.getBytes());

InputStream targetStream = new ByteArrayInputStream(convertToReportJSONString(gradeEnrollmentList, sdcSchoolCollectionEntity).getBytes());
params.put(JsonQueryExecuterFactory.JSON_INPUT_STREAM, targetStream);

JasperPrint jasperPrint = JasperFillManager.fillReport(gradeEnrollmentFTEReport, params);
return JasperExportManager.exportReportToPdf(jasperPrint);
} catch (Exception e) {
} catch (JRException | JsonProcessingException e) {
log.info("Exception occurred while writing PDF report for grade enrollment :: " + e.getMessage());
throw new StudentDataCollectionAPIRuntimeException("Exception occurred while writing PDF report for grade enrollment :: " + e.getMessage());
}
}

private String convertToReportJSONString(List<EnrollmentHeadcountResult> results, SdcSchoolCollectionEntity sdcSchoolCollection) throws JsonProcessingException {
var district = restUtils.getDistrictByDistrictID(sdcSchoolCollection.getDistrictID().toString());
if(district.isEmpty()){
log.info("District could not be found while writing PDF report for grade enrollment :: " + sdcSchoolCollection.getDistrictID().toString());
throw new EntityNotFoundException(District.class, "District could not be found while writing PDF report for grade enrollment :: ", sdcSchoolCollection.getDistrictID().toString());
}

var school = restUtils.getSchoolBySchoolID(sdcSchoolCollection.getSchoolID().toString());
if(school.isEmpty()){
log.info("School could not be found while writing PDF report for grade enrollment :: " + sdcSchoolCollection.getSchoolID().toString());
throw new EntityNotFoundException(School.class, "School could not be found while writing PDF report for grade enrollment :: ", sdcSchoolCollection.getSchoolID().toString());
}

GradeEnrollementFTENode mainNode = new GradeEnrollementFTENode();
GradeEnrollementFTEReportNode reportNode = new GradeEnrollementFTEReportNode();
reportNode.setReportGeneratedDate("Report Generated: " + LocalDate.now().format(formatter));
reportNode.setDistrictNumberAndName(district.get().getDistrictNumber() + " - " + district.get().getDisplayName());
reportNode.setCollectionNameAndYear(StringUtils.capitalize(sdcSchoolCollection.getCollectionEntity().getCollectionTypeCode().toLowerCase()) + " " + sdcSchoolCollection.getCollectionEntity().getOpenDate().getYear() + " Collection");
reportNode.setSchoolMincodeAndName(school.get().getMincode() + " - " + school.get().getDisplayName());
reportNode.setGrades(new ArrayList<>());

AtomicInteger totalSchoolAgedHeadcount = new AtomicInteger(0);
AtomicInteger totalSchoolAgedEligibleForFTE = new AtomicInteger(0);
AtomicDouble totalSchoolAgedFTETotal = new AtomicDouble(0);
AtomicInteger totalAdultHeadcount = new AtomicInteger(0);
AtomicInteger totalAdultEligibleForFTE = new AtomicInteger(0);
AtomicDouble totalAdultFTETotal = new AtomicDouble(0);
AtomicInteger totalAllStudentHeadcount = new AtomicInteger(0);
AtomicInteger totalAllStudentEligibleForFTE = new AtomicInteger(0);
AtomicDouble totalAllStudentFTETotal = new AtomicDouble(0);

results.forEach(hcResult -> {
GradeEnrollementFTEReportGradesNode grade = getGradeEnrollementFTEReportGradesNode(hcResult);
reportNode.getGrades().add(grade);

totalSchoolAgedHeadcount.addAndGet(Integer.valueOf(grade.getSchoolAgedHeadcount()));
totalSchoolAgedEligibleForFTE.addAndGet(Integer.valueOf(grade.getSchoolAgedEligibleForFTE()));
totalSchoolAgedFTETotal.addAndGet(Double.valueOf(grade.getSchoolAgedFTETotal()));
totalAdultHeadcount.addAndGet(Integer.valueOf(grade.getAdultHeadcount()));
totalAdultEligibleForFTE.addAndGet(Integer.valueOf(grade.getAdultEligibleForFTE()));
totalAdultFTETotal.addAndGet(Double.valueOf(grade.getAdultFTETotal()));
totalAllStudentHeadcount.addAndGet(Integer.valueOf(grade.getAllStudentHeadcount()));
totalAllStudentEligibleForFTE.addAndGet(Integer.valueOf(grade.getAllStudentEligibleForFTE()));
totalAllStudentFTETotal.addAndGet(Double.valueOf(grade.getAllStudentFTETotal()));
});

reportNode.getGrades().forEach(grade -> {
grade.setTotalCountsCode("Total");
grade.setTotalSchoolAgedHeadcount(totalSchoolAgedHeadcount.toString());
grade.setTotalSchoolAgedEligibleForFTE(totalSchoolAgedEligibleForFTE.toString());
grade.setTotalSchoolAgedFTETotal(String.format(DOUBLE_FORMAT, totalSchoolAgedFTETotal.doubleValue()));
grade.setTotalAdultsHeadcount(totalAdultHeadcount.toString());
grade.setTotalAdultsEligibleForFTE(totalAdultEligibleForFTE.toString());
grade.setTotalAdultsFTETotal(String.format(DOUBLE_FORMAT, totalAdultFTETotal.doubleValue()));
grade.setTotalAllStudentsHeadcount(totalAllStudentHeadcount.toString());
grade.setTotalAllStudentsEligibleForFTE(totalAllStudentEligibleForFTE.toString());
grade.setTotalAllStudentsFTETotal(String.format(DOUBLE_FORMAT, totalAllStudentFTETotal.doubleValue()));
});

mainNode.setReport(reportNode);
return objectWriter.writeValueAsString(mainNode);
}

private GradeEnrollementFTEReportGradesNode getGradeEnrollementFTEReportGradesNode(EnrollmentHeadcountResult hcResult) {
GradeEnrollementFTEReportGradesNode grade = new GradeEnrollementFTEReportGradesNode();
grade.setCode(hcResult.getEnrolledGradeCode());
grade.setSchoolAgedHeadcount(hcResult.getSchoolAgedHeadcount());
grade.setSchoolAgedEligibleForFTE(hcResult.getSchoolAgedEligibleForFte());
grade.setSchoolAgedFTETotal(String.format(DOUBLE_FORMAT, Double.valueOf(hcResult.getSchoolAgedFteTotal())));
grade.setAdultHeadcount(hcResult.getAdultHeadcount());
grade.setAdultEligibleForFTE(hcResult.getAdultEligibleForFte());
grade.setAdultFTETotal(String.format(DOUBLE_FORMAT, Double.valueOf(hcResult.getAdultFteTotal())));
grade.setAllStudentHeadcount(hcResult.getTotalHeadcount());
grade.setAllStudentEligibleForFTE(hcResult.getTotalEligibleForFte());
grade.setAllStudentFTETotal(String.format(DOUBLE_FORMAT, Double.valueOf(hcResult.getTotalFteTotal())));
return grade;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ NOT IN (SELECT saga.sdcSchoolCollectionStudentID FROM SdcSagaEntity saga WHERE s
"AND s.sdcSchoolCollectionStudentStatusCode NOT IN ('ERROR', 'DELETED') " +
"GROUP BY s.enrolledGradeCode " +
"ORDER BY s.enrolledGradeCode")
List<EnrollmentHeadcountResult> getEnrollmentHeadcountsBySchoolId(@Param("sdcSchoolCollectionID") UUID sdcSchoolCollectionID);
List<EnrollmentHeadcountResult> getEnrollmentHeadcountsBySdcSchoolCollectionId(@Param("sdcSchoolCollectionID") UUID sdcSchoolCollectionID);

@Query("SELECT " +
"s.enrolledGradeCode AS enrolledGradeCode, " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,13 @@ public class RestUtils {
public static final String CLOSE_DATE = "closedDate";
private static final String CONTENT_TYPE = "Content-Type";
private final Map<String, School> schoolMap = new ConcurrentHashMap<>();
private final Map<String, District> districtMap = new ConcurrentHashMap<>();
public static final String PAGE_SIZE = "pageSize";
private final WebClient webClient;
private final MessagePublisher messagePublisher;
private final ObjectMapper objectMapper = new ObjectMapper();
private final ReadWriteLock schoolLock = new ReentrantReadWriteLock();
private final ReadWriteLock districtLock = new ReentrantReadWriteLock();
@Getter
private final ApplicationProperties props;

Expand All @@ -86,6 +88,7 @@ public void init() {

private void initialize() {
this.populateSchoolMap();
this.populateDistrictMap();
}

@Scheduled(cron = "${schedule.jobs.load.school.cron}")
Expand Down Expand Up @@ -116,7 +119,7 @@ public void populateSchoolMap() {
finally {
writeLock.unlock();
}
log.info("loaded {} schools to memory", this.schoolMap.values().size());
log.info("Loaded {} schools to memory", this.schoolMap.values().size());
}

public List<School> getSchools() {
Expand All @@ -130,6 +133,34 @@ public List<School> getSchools() {
.block();
}

public void populateDistrictMap() {
val writeLock = this.districtLock.writeLock();
try {
writeLock.lock();
for (val district : this.getDistricts()) {
this.districtMap.put(district.getDistrictId(), district);
}
}
catch (Exception ex) {
log.error("Unable to load map cache district {}", ex);
}
finally {
writeLock.unlock();
}
log.info("Loaded {} districts to memory", this.districtMap.values().size());
}

public List<District> getDistricts() {
log.info("Calling Institute api to load districts to memory");
return this.webClient.get()
.uri(this.props.getInstituteApiURL() + "/district")
.header(CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
.retrieve()
.bodyToFlux(District.class)
.collectList()
.block();
}

@Retryable(retryFor = {Exception.class}, noRetryFor = {SagaRuntimeException.class}, backoff = @Backoff(multiplier = 2, delay = 2000))
public PenMatchResult getPenMatchResult(UUID correlationID, SdcSchoolCollectionStudentEntity sdcSchoolStudent, String mincode) {
try {
Expand Down Expand Up @@ -248,6 +279,14 @@ public Optional<School> getSchoolBySchoolID(final String schoolID) {
return Optional.ofNullable(this.schoolMap.get(schoolID));
}

public Optional<District> getDistrictByDistrictID(final String districtID) {
if (this.districtMap.isEmpty()) {
log.info("District map is empty reloading schools");
this.populateDistrictMap();
}
return Optional.ofNullable(this.districtMap.get(districtID));
}

private SearchCriteria getCriteria(final String key, final FilterOperation operation, final String value, final ValueType valueType, final Condition condition) {
return SearchCriteria.builder().key(key).operation(operation).value(value).valueType(valueType).condition(condition).build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public SdcSchoolCollectionStudentHeadcounts getEnrollmentHeadcounts(SdcSchoolCol
var sdcSchoolCollectionID = sdcSchoolCollectionEntity.getSdcSchoolCollectionID();
Optional<School> school = this.restUtils.getSchoolBySchoolID(String.valueOf(sdcSchoolCollectionEntity.getSchoolID()));
enrollmentHeadcountHelper.setGradeCodes(school);
List<EnrollmentHeadcountResult> collectionRawData = sdcSchoolCollectionStudentRepository.getEnrollmentHeadcountsBySchoolId(sdcSchoolCollectionID);
List<EnrollmentHeadcountResult> collectionRawData = sdcSchoolCollectionStudentRepository.getEnrollmentHeadcountsBySdcSchoolCollectionId(sdcSchoolCollectionID);
HeadcountResultsTable collectionData = enrollmentHeadcountHelper.convertHeadcountResults(collectionRawData);

List<HeadcountHeader> headcountHeaderList = Arrays.asList(enrollmentHeadcountHelper.getStudentsHeadcountTotals(collectionData), enrollmentHeadcountHelper.getGradesHeadcountTotals(collectionData));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package ca.bc.gov.educ.studentdatacollection.api.struct.v1;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import jakarta.validation.constraints.Email;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Pattern;
import jakarta.validation.constraints.Size;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.SuperBuilder;

import java.io.Serializable;

/**
* The type Student.
*/
@EqualsAndHashCode(callSuper = true)
@Data
@SuperBuilder
@JsonIgnoreProperties(ignoreUnknown = true)
public class District extends BaseRequest implements Serializable {
/**
* The constant serialVersionUID.
*/
private static final long serialVersionUID = 1L;

private String districtId;

@Size(max = 3)
@NotNull(message = "districtNumber can not be null.")
private String districtNumber;

@Size(max = 10)
@Pattern(regexp = "^$|\\d{10}", message = "Invalid phone number format")
private String faxNumber;

@Size(max = 10)
@Pattern(regexp = "^$|\\d{10}", message = "Invalid phone number format")
private String phoneNumber;

@Size(max = 255)
@Email(message = "Email address should be a valid email address")
private String email;

@Size(max = 255)
private String website;

@Size(max = 255)
@NotNull(message = "displayName cannot be null")
private String displayName;

@Size(max = 10)
@NotNull(message = "districtRegionCode cannot be null")
private String districtRegionCode;

@Size(max = 10)
@NotNull(message = "districtStatusCode cannot be null")
private String districtStatusCode;

}
Loading

0 comments on commit 94ebf13

Please sign in to comment.