feat: added symbol for worksheet rows + PR nits

api/src/services/import-services/critter/import-critters-service.test.ts

@@ -183,7 +183,7 @@ describe('ImportCrittersService', () => {
 
       const sexHeaderConfig = await service._getSexHeaderConfig();
 
-      expect(getTaxonMeasurementsStub).to.have.been.calledWithExactly(1234);
+      expect(getTaxonMeasurementsStub).to.have.been.calledWithExactly('1234');
       expect(getSexCellValidatorStub).to.have.been.calledWithExactly(
         new NestedRecord({
           1234: { male: 'maleUUID', female: 'femaleUUID' }
@@ -224,7 +224,7 @@ describe('ImportCrittersService', () => {
 
       const config = await service._getCollectionUnitDynamicHeaderConfig();
 
-      expect(findTaxonCollectionUnitsStub).to.have.been.calledOnceWithExactly(1234);
+      expect(findTaxonCollectionUnitsStub).to.have.been.calledOnceWithExactly('1234');
 
       expect(getCollectionUnitCellValidatorStub).to.have.been.calledWithExactly(
         new NestedRecord({ 1234: { category: { unit: 'uuid' } } }),

api/src/services/import-services/critter/import-critters-service.ts

@@ -208,7 +208,7 @@ export class ImportCrittersService extends DBService {
    * @returns {*} {Promise<CSVHeaderConfig>} The TSN header config
    */
   async _getTsnHeaderConfig(): Promise<CSVHeaderConfig> {
-    const rowTsns = this.configUtils.getUniqueCellValues('ITIS_TSN').map((tsn) => Number(tsn));
+    const rowTsns = this.configUtils.getUniqueCellValues('ITIS_TSN').map((tsn) => String(tsn));
     const taxonomy = await this.platformService.getTaxonomyByTsns(rowTsns);
     const allowedTsns = new Set(taxonomy.map((taxon) => taxon.tsn));
 

api/src/services/import-services/marking/marking-header-configs.ts

@@ -76,18 +76,18 @@ export const getMarkingAliasCellValidator = (surveyAliasMap: Map<string, ICritte
  * Rules:
  *  1. The cell must be a valid marking type ie: exists in the markingTypes set
  *
- * @param {Set<string>} _markingTypes The marking types
+ * @param {Set<string>} markingTypes The marking types set (case insensitive)
  * @returns {*} {CSVCellValidator} The validate cell callback
  */
-export const getMarkingTypeCellValidator = (_markingTypes: Set<string>): CSVCellValidator => {
-  const markingTypes = setToLowercase(_markingTypes);
+export const getMarkingTypeCellValidator = (markingTypes: Set<string>): CSVCellValidator => {
+  const markingTypesLowerCased = setToLowercase(markingTypes);
 
   return (params: CSVParams) => {
     if (params.cell === undefined) {
       return [];
     }
 
-    if (!markingTypes.has(String(params.cell).toLowerCase())) {
+    if (!markingTypesLowerCased.has(String(params.cell).toLowerCase())) {
       return [
         {
           error: `Marking type not supported`,
@@ -104,18 +104,18 @@ export const getMarkingTypeCellValidator = (_markingTypes: Set<string>): CSVCell
 /**
  * Get the marking type cell setter.
  *
- * @param {Set<string>} _colours The colours
+ * @param {Set<string>} colours The colours set (case insensitive)
  * @returns {*} {CSVCellSetter} The set cell callback
  */
-export const getMarkingColourCellValidator = (_colours: Set<string>): CSVCellValidator => {
-  const colours = setToLowercase(_colours);
+export const getMarkingColourCellValidator = (colours: Set<string>): CSVCellValidator => {
+  const coloursLowerCased = setToLowercase(colours);
 
   return (params: CSVParams) => {
     if (params.cell === undefined) {
       return [];
     }
 
-    if (colours.has(String(params.cell).toLowerCase())) {
+    if (coloursLowerCased.has(String(params.cell).toLowerCase())) {
       return [];
     }
 

api/src/utils/csv-utils/csv-config-utils.test.ts

@@ -1,5 +1,6 @@
 import { expect } from 'chai';
 import xlsx, { WorkSheet } from 'xlsx';
+import { WorksheetRowIndexSymbol } from '../xlsx-utils/worksheet-utils';
 import { CSVConfigUtils } from './csv-config-utils';
 import { CSVConfig } from './csv-config-validation.interface';
 
@@ -22,9 +23,13 @@ describe('CSVConfigUtils', () => {
       expect(utils).to.be.instanceOf(CSVConfigUtils);
       expect(utils._config).to.be.equal(mockConfig);
       expect(utils.worksheet).to.be.equal(worksheet);
-      expect(utils.worksheetRows).to.be.deep.equal([
-        { TEST: 'cellValue', ALIASED_HEADER: 'cellValue2', DYNAMIC_HEADER: 'dynamicValue' }
-      ]);
+
+      expect(utils.worksheetRows[0]).to.deep.equal({
+        TEST: 'cellValue',
+        ALIASED_HEADER: 'cellValue2',
+        DYNAMIC_HEADER: 'dynamicValue',
+        [WorksheetRowIndexSymbol]: 1
+      });
       expect(utils.worksheetHeaders).to.be.deep.equal(['TEST', 'ALIASED_HEADER', 'DYNAMIC_HEADER']);
       expect(utils.worksheetAliasedStaticHeaders).to.be.deep.equal(['TEST', 'ALIASED_HEADER']);
       expect(utils.worksheetStaticHeaders).to.be.deep.equal(['TEST', 'TEST_ALIAS']);

api/src/utils/csv-utils/csv-config-validation.test.ts

@@ -2,6 +2,7 @@ import chai, { expect } from 'chai';
 import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
 import xlsx, { WorkSheet } from 'xlsx';
+import { WorksheetRowIndexSymbol } from '../xlsx-utils/worksheet-utils';
 import {
   executeSetCellValue,
   executeValidateCell,
@@ -309,7 +310,7 @@ describe('csv-config-validation', () => {
           cell: 'cellValue',
           header: 'TEST_ALIAS',
           rowIndex: 0,
-          row: { TEST_ALIAS: 'cellValue', DYNAMIC_HEADER: 'dynamicValue' },
+          row: { TEST_ALIAS: 'cellValue', DYNAMIC_HEADER: 'dynamicValue', [WorksheetRowIndexSymbol]: 1 },
           staticHeader: 'TEST',
           mutateCell: 'cellValue'
         },
@@ -324,7 +325,7 @@ describe('csv-config-validation', () => {
           cell: 'dynamicValue',
           header: 'DYNAMIC_HEADER',
           rowIndex: 0,
-          row: { TEST_ALIAS: 'cellValue', DYNAMIC_HEADER: 'dynamicValue' },
+          row: { TEST_ALIAS: 'cellValue', DYNAMIC_HEADER: 'dynamicValue', [WorksheetRowIndexSymbol]: 1 },
           staticHeader: undefined, // Dynamic headers have no static header mapping
           mutateCell: 'dynamicValue'
         },
@@ -346,7 +347,7 @@ describe('csv-config-validation', () => {
         cell: 'cellValue',
         header: 'TEST',
         rowIndex: 0,
-        row: { TEST: 'cellValue' },
+        row: { TEST: 'cellValue', [WorksheetRowIndexSymbol]: 1 },
         staticHeader: 'TEST',
         mutateCell: 'cellValue'
       };

api/src/utils/csv-utils/csv-config-validation.ts

@@ -1,5 +1,5 @@
 import { WorkSheet } from 'xlsx';
-import { getWorksheetRowObjects } from '../xlsx-utils/worksheet-utils';
+import { getWorksheetRowObjects, WorksheetRowIndexSymbol } from '../xlsx-utils/worksheet-utils';
 import { CSVConfigUtils } from './csv-config-utils';
 import {
   CSVConfig,
@@ -220,7 +220,8 @@ export const executeValidateCell = (
         values: error.values ?? null,
         cell: (error.cell === undefined ? params.cell : error.cell) ?? null, // Use cell value if intentionally null
         header: (error.header === undefined ? params.header : error.header) ?? null, // Use header value if intentionally null
-        row: error.row ?? params.rowIndex + 2 // headers: 1, data row: 2
+        // WorksheetRowIndexSymbol is the original row index from the worksheet ie: before filtering empty rows
+        row: error.row ?? params.row[WorksheetRowIndexSymbol] + 1 ?? params.rowIndex + 2 // headers: 1, data row: 2
       });
     });
   }

api/src/utils/xlsx-utils/worksheet-utils.ts

@@ -17,7 +17,7 @@ dayjs.extend(customParseFormat);
 
 const defaultLog = getLogger('src/utils/xlsx-utils/worksheet-utils');
 
-const RowIndex = Symbol();
+export const WorksheetRowIndexSymbol = Symbol('WorksheetRowIndex');
 
 export interface IXLSXCSVColumn {
   /**
@@ -223,7 +223,9 @@ export const getWorksheetRowObjects = (worksheet: xlsx.WorkSheet): Record<symbol
       rowHasValues = true;
     }
 
-    rowObject[RowIndex] = i;
+    // Add the original row index to the row object
+    // Symbols are non-enumerable, so they won't be included in the JSON output
+    rowObject[WorksheetRowIndexSymbol] = i;
 
     if (rowHasValues) {
       // Add the row object to the array if it has at least one non-empty cell