MTA-14 fixes and push to s3 bucket

onebusaway-gtfs-transformer-cli-aws/pom.xml

@@ -27,7 +27,7 @@
     <dependency>
       <groupId>org.onebusaway</groupId>
       <artifactId>onebusaway-cloud-aws</artifactId>
-      <version>0.0.10</version>
+      <version>${onebusaway_cloud_version}</version>
     </dependency>
     <dependency>
       <groupId>commons-cli</groupId>

onebusaway-gtfs-transformer/pom.xml

@@ -47,10 +47,15 @@
       <artifactId>wsf-api</artifactId>
       <version>1.0</version>
     </dependency>
+      <dependency>
+          <groupId>org.onebusaway</groupId>
+          <artifactId>onebusaway-cloud-api</artifactId>
+          <version>${onebusaway_cloud_version}</version>
+      </dependency>
     <dependency>
       <groupId>org.onebusaway</groupId>
       <artifactId>onebusaway-cloud-noop</artifactId>
-      <version>0.0.10</version>
+      <version>${onebusaway_cloud_version}</version>
     </dependency>
 	  <dependency>
 	    <groupId>javax.xml.bind</groupId>

onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CheckForFutureService.java

@@ -98,7 +98,7 @@ int hasServiceForDate(GtfsMutableRelationalDao dao, Date testDate) {
         ServiceDate serviceDate = new ServiceDate(testDate);
         int numTripsOnDate = 0;
         for (Trip trip : dao.getAllTrips()) {
-            if (helper.isTripActive(dao, serviceDate, trip))
+            if (helper.isTripActive(dao, serviceDate, trip, false))
                 numTripsOnDate++;
         }
         return numTripsOnDate;

onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CompareToReferenceService.java

@@ -29,6 +29,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.FileWriter;
 import java.util.*;
 
 /**
@@ -47,6 +49,12 @@ public String getName() {
 
   @CsvField(ignore = true)
   private String defaultAgencyId = "1";
+
+  private String s3BasePath = null;
+  public void setS3BasePath(String path) {
+    s3BasePath = path;
+  }
+
   @CsvField(ignore = true)
   private CalendarFunctions helper = new CalendarFunctions();
   @Override
@@ -58,17 +66,24 @@ public void run(TransformContext context, GtfsMutableRelationalDao gtfsDao) {
       String detailTopic = CloudContextService.getTopic() + "-atis-detail";
       ExternalServices es =  new ExternalServicesBridgeFactory().getExternalServices();
 
+      String summaryHeader = "depot,unmatched_gtfs_trips,unmatched_reference_trips\n";
+      String detailHeader = "depot,unmatched_gtfs_trip_ds,unmatched_reference_trip_ids\n";
+
       StringBuffer summaryReport = new StringBuffer();
-      summaryReport.append("depot,unmatched_gtfs_trips,unmatched_reference_trips\n");
       StringBuffer detailReport = new StringBuffer();
-      detailReport.append("depot,unmatched_gtfs_trip_ds,unmatched_reference_trip_ids\n");
 
       GtfsMutableRelationalDao referenceDao = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore();
 
       // get active trip ids for service date from base GTFS
       Map<String, List<AgencyAndId>> activeTripsByDepot = getTripsByDepot(context, gtfsDao);
       // get active trip ids for service date for reference GTFS
       Map<String, List<AgencyAndId>> referenceTripsByDepot = getTripsByDepot(context, referenceDao);
+      String summaryFilename = System.getProperty("java.io.tmpdir") + File.separator + "summary.csv";
+      FileWriter summaryFile = new FileWriter(summaryFilename);
+      summaryFile.write(summaryHeader);
+      String detailFilename = System.getProperty("java.io.tmpdir") + File.separator + "detail.csv";
+      FileWriter detailFile = new FileWriter(detailFilename);
+      detailFile.write(detailHeader);
 
       Set<String> allDepots = new HashSet<>();
       allDepots.addAll(activeTripsByDepot.keySet());
@@ -94,9 +109,26 @@ public void run(TransformContext context, GtfsMutableRelationalDao gtfsDao) {
         summaryReport.append(depot).append(",").append(unmatchedGtfsTrips.size()).append(",").append(unmatchedReferenceTrips.size()).append("\n");
         detailReport.append(depot).append(",\"").append(unmatchedGtfsTrips).append("\",\"").append(unmatchedReferenceTrips).append("\"\n");
         es.publishMessage(detailTopic, truncate(detailReport.toString()));
+        detailFile.write(detailReport.toString());
         detailReport = new StringBuffer();
       }
       es.publishMessage(summaryTopic, summaryReport.toString());
+      summaryFile.write(summaryReport.toString());
+
+      summaryFile.close();
+      detailFile.close();
+
+      // now copy the reports to an S3 reports directory
+      Calendar cal = Calendar.getInstance();
+      String year = "" + cal.get(Calendar.YEAR);
+      String month = lpad(cal.get(Calendar.MONTH)+1, 2);
+      String day = lpad(cal.get(Calendar.DAY_OF_MONTH), 2);
+      String time = lpad(cal.get(Calendar.HOUR_OF_DAY), 2) + ":" + lpad(cal.get(Calendar.MINUTE),2 );
+      String baseurl = s3BasePath
+              + "/" + year + "/" + month + "/" + day + "/" + time + "-";
+      es.putFile(baseurl + "summary.csv", summaryFilename);
+      es.putFile(baseurl + "detail.csv", detailFilename);
+
 
       _log.error("{} Unmatched Summary", getName());
       _log.error(summaryReport.toString());
@@ -106,14 +138,20 @@ public void run(TransformContext context, GtfsMutableRelationalDao gtfsDao) {
 
   }
 
+  private String lpad(int numberToFormat, int totalDigits) {
+    String text = String.valueOf(numberToFormat);
+    while (text.length() < totalDigits)
+      text = "0" + text;
+    return text;
+  }
+
   private String truncate(String message) {
     if (message.length() > MAX_MESSAGE_SIZE)
       return message.substring(0, MAX_MESSAGE_SIZE);
     return message;
   }
 
-  private String getDepot(Trip trip) {
-    String tripId = trip.getId().getId();
+  private String getDepot(String tripId) {
     String depot = null;
     if (tripId.indexOf("-") < 0) {
       depot = "MISSING";
@@ -133,13 +171,13 @@ private Map<String, List<AgencyAndId>> getTripsByDepot(TransformContext context,
     Map<String, List<AgencyAndId>> tripsByDepot = new HashMap<>();
     for (Trip trip : dao.getAllTrips()) {
       if (trip.getServiceId() != null) {
-        boolean isActive = helper.isTripActive(dao, new ServiceDate(), trip);
+        boolean isActive = helper.isTripActive(dao, new ServiceDate(), trip, true);
         if (isActive) {
           AgencyAndId tripId = trip.getId();
           if (trip.getMtaTripId() != null) {
             tripId = new AgencyAndId(trip.getId().getAgencyId(), trip.getMtaTripId());
           }
-          String depot = getDepot(trip);
+          String depot = getDepot(tripId.getId());
           if (!tripsByDepot.containsKey(depot))
             tripsByDepot.put(depot, new ArrayList<>());
           tripsByDepot.get(depot).add(sanitize(tripId));
@@ -152,8 +190,8 @@ private Map<String, List<AgencyAndId>> getTripsByDepot(TransformContext context,
 
 
   private AgencyAndId sanitize(AgencyAndId tripId) {
-    if (tripId.getId().contains("SDon-")) {
-      return new AgencyAndId(defaultAgencyId, tripId.getId().replaceAll("SDon-", ""));
+    if (tripId.getId().contains("-SDon")) {
+      return new AgencyAndId(defaultAgencyId, tripId.getId().replaceAll("-SDon", ""));
     }
     return new AgencyAndId(defaultAgencyId, tripId.getId());
   }

onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/util/CalendarFunctions.java

@@ -28,7 +28,8 @@
  * Common calendaring functions for Strategies/Transformations.
  */
 public class CalendarFunctions {
-  public boolean isTripActive(GtfsMutableRelationalDao dao, ServiceDate serviceDate, Trip trip) {
+
+  public boolean isTripActive(GtfsMutableRelationalDao dao, ServiceDate serviceDate, Trip trip, boolean matchDayInCalendar) {
     Date testDate = serviceDate.getAsDate();
     //check for service
     boolean hasCalDateException = false;
@@ -49,13 +50,42 @@ public boolean isTripActive(GtfsMutableRelationalDao dao, ServiceDate serviceDat
     //if there are no entries in calendarDates, check serviceCalendar
     if (!hasCalDateException) {
       ServiceCalendar servCal = dao.getCalendarForServiceId(trip.getServiceId());
+      if (servCal == null) {
+        // do a brute force lookup as agencyIds are tricky
+        for (ServiceCalendar calendar : dao.getAllCalendars()) {
+          if (calendar.getServiceId().getId().equals(trip.getServiceId().getId())) {
+            servCal = calendar;
+          }
+        }
+
+      }
       if (servCal != null) {
         //check for service using calendar
         Date start = removeTime(servCal.getStartDate().getAsDate());
         Date end = removeTime(servCal.getEndDate().getAsDate());
         if (testDate.equals(start) || testDate.equals(end) ||
                 (testDate.after(start) && testDate.before(end))) {
-          return true;
+          Calendar cal = Calendar.getInstance();
+          cal.setTime(testDate);
+          if (!matchDayInCalendar) return true;
+          switch (cal.get(Calendar.DAY_OF_WEEK)) {
+            case Calendar.SUNDAY:
+              return servCal.getSunday() == 1;
+            case Calendar.MONDAY:
+              return servCal.getMonday() == 1;
+            case Calendar.TUESDAY:
+              return servCal.getTuesday() == 1;
+            case Calendar.WEDNESDAY:
+              return servCal.getWednesday() == 1;
+            case Calendar.THURSDAY:
+              return servCal.getThursday() == 1;
+            case Calendar.FRIDAY:
+              return servCal.getFriday() == 1;
+            case Calendar.SATURDAY:
+              return servCal.getSaturday() == 1;
+            default:
+              throw new IllegalStateException("unexected value " + cal.get(Calendar.DAY_OF_WEEK));
+          }
         }
       }
     }

pom.xml

@@ -17,7 +17,8 @@
   <properties>
     <onebusaway_csv_entities_version>1.1.7</onebusaway_csv_entities_version>
     <onebusaway_collections_version>1.2.8</onebusaway_collections_version>
-      <slf4j_version>2.0.6</slf4j_version>
+    <slf4j_version>2.0.6</slf4j_version>
+    <onebusaway_cloud_version>0.0.13</onebusaway_cloud_version>
   </properties>
 
   <scm>