Importer fixes for dirty fields

nabeelio · Oct 3, 2023 · 773dbe3 · 773dbe3
1 parent 91c0733
commit 773dbe3
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 30 deletions.
diff --git a/app/Models/Airport.php b/app/Models/Airport.php
@@ -32,6 +32,7 @@
  * @property float  lat
  * @property float  lon
  * @property int    elevation
+ * @property bool   hub
  */
 class Airport extends Model
 {

diff --git a/app/Services/ImportExport/AirportImporter.php b/app/Services/ImportExport/AirportImporter.php
@@ -48,13 +48,13 @@ public function import(array $row, $index): bool
         $row['id'] = $row['icao'];
         $row['hub'] = get_truth_state($row['hub']);
 
-        if ($row['ground_handling_cost'] === null && $row['ground_handling_cost'] !== 0.0) {
+        if (!is_numeric($row['ground_handling_cost'])) {
             $row['ground_handling_cost'] = (float) setting('airports.default_ground_handling_cost');
         } else {
             $row['ground_handling_cost'] = (float) $row['ground_handling_cost'];
         }
 
-        if ($row['fuel_jeta_cost'] === null && $row['fuel_jeta_cost'] !== 0.0) {
+        if (!is_numeric($row['fuel_jeta_cost'])) {
             $row['fuel_jeta_cost'] = (float) setting('airports.default_jet_a_fuel_cost');
         } else {
             $row['fuel_jeta_cost'] = (float) $row['fuel_jeta_cost'];
@@ -65,7 +65,7 @@ public function import(array $row, $index): bool
                 'id' => $row['icao'],
             ], $row);
         } catch (\Exception $e) {
-            $this->errorLog('Error in row '.$index.': '.$e->getMessage());
+            $this->errorLog('Error in row '.($index + 1).': '.$e->getMessage());
             return false;
         }
 

diff --git a/app/Services/ImportService.php b/app/Services/ImportService.php
@@ -66,9 +66,11 @@ protected function throwError($error, \Exception $e = null): void
     public function openCsv($csv_file)
     {
         try {
-            $reader = Reader::createFromPath($csv_file);
+            $reader = Reader::createFromPath($csv_file, 'r');
             $reader->setDelimiter(',');
+            $reader->setHeaderOffset(0);
             $reader->setEnclosure('"');
+            $reader->setEscape('\\');
             return $reader;
         } catch (Exception $e) {
             $this->throwError('Error opening CSV: '.$e->getMessage(), $e);
@@ -94,33 +96,13 @@ protected function runImport($file_path, ImportExport $importer): array
         $first_header = $cols[0];
 
         $first = true;
-        $records = $reader->getRecords($cols);
+        $header_rows = $reader->getHeader();
+        $records = $reader->getRecords($header_rows);
         foreach ($records as $offset => $row) {
-            // check if the first row being read is the header
-            if ($first) {
-                $first = false;
-
-                if ($row[$first_header] !== $first_header) {
-                    $this->throwError('CSV file doesn\'t seem to match import type');
-                }
-
-                continue;
-            }
-
-            // Do a sanity check on the number of columns first
-            if (!$importer->checkColumns($row)) {
-                $importer->errorLog('Number of columns in row doesn\'t match');
-                continue;
-            }
-
             // turn it into a collection and run some filtering
             $row = collect($row)->map(function ($val, $index) {
                 $val = trim($val);
-                if ($val === '') {
-                    return;
-                }
-
-                return $val;
+                return str_ireplace(['\\n', '\\r'], '', $val);
             })->toArray();
 
             // Try to validate

diff --git a/tests/ImporterTest.php b/tests/ImporterTest.php
@@ -23,7 +23,6 @@
 use App\Services\ImportExport\FlightExporter;
 use App\Services\ImportService;
 use Illuminate\Support\Facades\Storage;
-use Illuminate\Validation\ValidationException;
 
 class ImporterTest extends TestCase
 {
@@ -375,9 +374,10 @@ public function testFlightExporter(): void
      */
     public function testInvalidFileImport(): void
     {
-        $this->expectException(ValidationException::class);
+        // $this->expectException(ValidationException::class);
         $file_path = base_path('tests/data/aircraft.csv');
-        $this->importSvc->importAirports($file_path);
+        $status = $this->importSvc->importAirports($file_path);
+        $this->assertCount(2, $status['errors']);
     }
 
     /**
@@ -686,6 +686,29 @@ public function testAirportImporter(): void
         $this->assertEquals(setting('airports.default_ground_handling_cost'), $airport->ground_handling_cost);
     }
 
+    public function testAirportImporterInvalidInputs(): void
+    {
+        $file_path = base_path('tests/data/airports_errors.csv');
+        $status = $this->importSvc->importAirports($file_path);
+
+        $this->assertCount(5, $status['success']);
+        $this->assertCount(1, $status['errors']);
+
+        // See if it imported
+        /** @var Airport $airport */
+        $airport = Airport::where([
+            'id' => 'CYAV',
+        ])->first();
+
+        $this->assertNotNull($airport);
+        $this->assertEquals('CYAV', $airport->id);
+        $this->assertEquals('', $airport->iata);
+        $this->assertEquals('America/Winnipeg', $airport->timezone);
+        $this->assertFalse($airport->hub);
+        $this->assertEquals('50.0564003', $airport->lat);
+        $this->assertEquals('-97.03250122', $airport->lon);
+    }
+
     /**
      * Test importing the subfleets
      *

diff --git a/tests/data/airports_errors.csv b/tests/data/airports_errors.csv
@@ -0,0 +1,7 @@
+icao,iata,name,location,region,country,timezone,hub,lat,lon,elevation,ground_handling_cost,fuel_100ll_cost,fuel_jeta_cost,fuel_mogas_cost,notes
+\N,\N,Sun Island Resort and SPA,South Aari Atoll,,Maldives,America/Sao_Paulo,,3.488334,72.862989,,,,,,
+CYAV,\N,Winnipeg / St. Andrews Airport,Winnipeg,,Canada,America/Winnipeg,,50.0564003,-97.03250122,,,,,,
+CYAW,\N,Halifax / CFB Shearwater Heliport,Halifax,,Canada,America/Halifax,,44.639702,-63.499401,,,,,,
+CYDC,\N,Princeton Airport,Princeton,,Canada,America/Vancouver,,49.4681015,-120.5110016,,,,,,
+CYDF,YDF,Deer Lake Airport,Deer Lake,,Canada,America/St_Johns,,49.21080017,-57.39139938,,,,,,
+CYDL,YDL,Dease Lake Airport,Dease Lake,,Canada,America/Vancouver,,58.42219925,-130.0319977,,,,,,