From 8f7f731b3d880c449f0d34010df7cd49cf8ca869 Mon Sep 17 00:00:00 2001 From: sheldonabrown Date: Tue, 31 Oct 2023 09:26:34 -0400 Subject: [PATCH] MTA-14 fixes and push to s3 bucket --- onebusaway-gtfs-transformer-cli-aws/pom.xml | 2 +- onebusaway-gtfs-transformer/pom.xml | 7 ++- .../impl/CheckForFutureService.java | 2 +- .../impl/CompareToReferenceService.java | 54 ++++++++++++++++--- .../util/CalendarFunctions.java | 34 +++++++++++- pom.xml | 3 +- 6 files changed, 88 insertions(+), 14 deletions(-) diff --git a/onebusaway-gtfs-transformer-cli-aws/pom.xml b/onebusaway-gtfs-transformer-cli-aws/pom.xml index ed56a317..51e55576 100644 --- a/onebusaway-gtfs-transformer-cli-aws/pom.xml +++ b/onebusaway-gtfs-transformer-cli-aws/pom.xml @@ -27,7 +27,7 @@ org.onebusaway onebusaway-cloud-aws - 0.0.10 + ${onebusaway_cloud_version} commons-cli diff --git a/onebusaway-gtfs-transformer/pom.xml b/onebusaway-gtfs-transformer/pom.xml index c8524e58..8e3b0780 100644 --- a/onebusaway-gtfs-transformer/pom.xml +++ b/onebusaway-gtfs-transformer/pom.xml @@ -47,10 +47,15 @@ wsf-api 1.0 + + org.onebusaway + onebusaway-cloud-api + ${onebusaway_cloud_version} + org.onebusaway onebusaway-cloud-noop - 0.0.10 + ${onebusaway_cloud_version} javax.xml.bind diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CheckForFutureService.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CheckForFutureService.java index 41a15f6c..23c8883e 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CheckForFutureService.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CheckForFutureService.java @@ -98,7 +98,7 @@ int hasServiceForDate(GtfsMutableRelationalDao dao, Date testDate) { ServiceDate serviceDate = new ServiceDate(testDate); int numTripsOnDate = 0; for (Trip trip : dao.getAllTrips()) { - if (helper.isTripActive(dao, serviceDate, trip)) + if (helper.isTripActive(dao, serviceDate, trip, false)) numTripsOnDate++; } return numTripsOnDate; diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CompareToReferenceService.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CompareToReferenceService.java index 059b5787..6dcf291e 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CompareToReferenceService.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CompareToReferenceService.java @@ -29,6 +29,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.FileWriter; import java.util.*; /** @@ -47,6 +49,12 @@ public String getName() { @CsvField(ignore = true) private String defaultAgencyId = "1"; + + private String s3BasePath = null; + public void setS3BasePath(String path) { + s3BasePath = path; + } + @CsvField(ignore = true) private CalendarFunctions helper = new CalendarFunctions(); @Override @@ -58,10 +66,11 @@ public void run(TransformContext context, GtfsMutableRelationalDao gtfsDao) { String detailTopic = CloudContextService.getTopic() + "-atis-detail"; ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices(); + String summaryHeader = "depot,unmatched_gtfs_trips,unmatched_reference_trips\n"; + String detailHeader = "depot,unmatched_gtfs_trip_ds,unmatched_reference_trip_ids\n"; + StringBuffer summaryReport = new StringBuffer(); - summaryReport.append("depot,unmatched_gtfs_trips,unmatched_reference_trips\n"); StringBuffer detailReport = new StringBuffer(); - detailReport.append("depot,unmatched_gtfs_trip_ds,unmatched_reference_trip_ids\n"); GtfsMutableRelationalDao referenceDao = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore(); @@ -69,6 +78,12 @@ public void run(TransformContext context, GtfsMutableRelationalDao gtfsDao) { Map> activeTripsByDepot = getTripsByDepot(context, gtfsDao); // get active trip ids for service date for reference GTFS Map> referenceTripsByDepot = getTripsByDepot(context, referenceDao); + String summaryFilename = System.getProperty("java.io.tmpdir") + File.separator + "summary.csv"; + FileWriter summaryFile = new FileWriter(summaryFilename); + summaryFile.write(summaryHeader); + String detailFilename = System.getProperty("java.io.tmpdir") + File.separator + "detail.csv"; + FileWriter detailFile = new FileWriter(detailFilename); + detailFile.write(detailHeader); Set allDepots = new HashSet<>(); allDepots.addAll(activeTripsByDepot.keySet()); @@ -94,9 +109,26 @@ public void run(TransformContext context, GtfsMutableRelationalDao gtfsDao) { summaryReport.append(depot).append(",").append(unmatchedGtfsTrips.size()).append(",").append(unmatchedReferenceTrips.size()).append("\n"); detailReport.append(depot).append(",\"").append(unmatchedGtfsTrips).append("\",\"").append(unmatchedReferenceTrips).append("\"\n"); es.publishMessage(detailTopic, truncate(detailReport.toString())); + detailFile.write(detailReport.toString()); detailReport = new StringBuffer(); } es.publishMessage(summaryTopic, summaryReport.toString()); + summaryFile.write(summaryReport.toString()); + + summaryFile.close(); + detailFile.close(); + + // now copy the reports to an S3 reports directory + Calendar cal = Calendar.getInstance(); + String year = "" + cal.get(Calendar.YEAR); + String month = lpad(cal.get(Calendar.MONTH)+1, 2); + String day = lpad(cal.get(Calendar.DAY_OF_MONTH), 2); + String time = lpad(cal.get(Calendar.HOUR_OF_DAY), 2) + ":" + lpad(cal.get(Calendar.MINUTE),2 ); + String baseurl = s3BasePath + + "/" + year + "/" + month + "/" + day + "/" + time + "-"; + es.putFile(baseurl + "summary.csv", summaryFilename); + es.putFile(baseurl + "detail.csv", detailFilename); + _log.error("{} Unmatched Summary", getName()); _log.error(summaryReport.toString()); @@ -106,14 +138,20 @@ public void run(TransformContext context, GtfsMutableRelationalDao gtfsDao) { } + private String lpad(int numberToFormat, int totalDigits) { + String text = String.valueOf(numberToFormat); + while (text.length() < totalDigits) + text = "0" + text; + return text; + } + private String truncate(String message) { if (message.length() > MAX_MESSAGE_SIZE) return message.substring(0, MAX_MESSAGE_SIZE); return message; } - private String getDepot(Trip trip) { - String tripId = trip.getId().getId(); + private String getDepot(String tripId) { String depot = null; if (tripId.indexOf("-") < 0) { depot = "MISSING"; @@ -133,13 +171,13 @@ private Map> getTripsByDepot(TransformContext context, Map> tripsByDepot = new HashMap<>(); for (Trip trip : dao.getAllTrips()) { if (trip.getServiceId() != null) { - boolean isActive = helper.isTripActive(dao, new ServiceDate(), trip); + boolean isActive = helper.isTripActive(dao, new ServiceDate(), trip, true); if (isActive) { AgencyAndId tripId = trip.getId(); if (trip.getMtaTripId() != null) { tripId = new AgencyAndId(trip.getId().getAgencyId(), trip.getMtaTripId()); } - String depot = getDepot(trip); + String depot = getDepot(tripId.getId()); if (!tripsByDepot.containsKey(depot)) tripsByDepot.put(depot, new ArrayList<>()); tripsByDepot.get(depot).add(sanitize(tripId)); @@ -152,8 +190,8 @@ private Map> getTripsByDepot(TransformContext context, private AgencyAndId sanitize(AgencyAndId tripId) { - if (tripId.getId().contains("SDon-")) { - return new AgencyAndId(defaultAgencyId, tripId.getId().replaceAll("SDon-", "")); + if (tripId.getId().contains("-SDon")) { + return new AgencyAndId(defaultAgencyId, tripId.getId().replaceAll("-SDon", "")); } return new AgencyAndId(defaultAgencyId, tripId.getId()); } diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/util/CalendarFunctions.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/util/CalendarFunctions.java index 27f04977..f917454e 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/util/CalendarFunctions.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/util/CalendarFunctions.java @@ -28,7 +28,8 @@ * Common calendaring functions for Strategies/Transformations. */ public class CalendarFunctions { - public boolean isTripActive(GtfsMutableRelationalDao dao, ServiceDate serviceDate, Trip trip) { + + public boolean isTripActive(GtfsMutableRelationalDao dao, ServiceDate serviceDate, Trip trip, boolean matchDayInCalendar) { Date testDate = serviceDate.getAsDate(); //check for service boolean hasCalDateException = false; @@ -49,13 +50,42 @@ public boolean isTripActive(GtfsMutableRelationalDao dao, ServiceDate serviceDat //if there are no entries in calendarDates, check serviceCalendar if (!hasCalDateException) { ServiceCalendar servCal = dao.getCalendarForServiceId(trip.getServiceId()); + if (servCal == null) { + // do a brute force lookup as agencyIds are tricky + for (ServiceCalendar calendar : dao.getAllCalendars()) { + if (calendar.getServiceId().getId().equals(trip.getServiceId().getId())) { + servCal = calendar; + } + } + + } if (servCal != null) { //check for service using calendar Date start = removeTime(servCal.getStartDate().getAsDate()); Date end = removeTime(servCal.getEndDate().getAsDate()); if (testDate.equals(start) || testDate.equals(end) || (testDate.after(start) && testDate.before(end))) { - return true; + Calendar cal = Calendar.getInstance(); + cal.setTime(testDate); + if (!matchDayInCalendar) return true; + switch (cal.get(Calendar.DAY_OF_WEEK)) { + case Calendar.SUNDAY: + return servCal.getSunday() == 1; + case Calendar.MONDAY: + return servCal.getMonday() == 1; + case Calendar.TUESDAY: + return servCal.getTuesday() == 1; + case Calendar.WEDNESDAY: + return servCal.getWednesday() == 1; + case Calendar.THURSDAY: + return servCal.getThursday() == 1; + case Calendar.FRIDAY: + return servCal.getFriday() == 1; + case Calendar.SATURDAY: + return servCal.getSaturday() == 1; + default: + throw new IllegalStateException("unexected value " + cal.get(Calendar.DAY_OF_WEEK)); + } } } } diff --git a/pom.xml b/pom.xml index 7af685cd..51420a3c 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,8 @@ 1.1.7 1.2.8 - 2.0.6 + 2.0.6 + 0.0.13