PIMS-2002 Properties Export (#2641)

express-api/src/services/properties/propertiesServices.ts

@@ -30,7 +30,7 @@ import {
   constructFindOptionFromQuerySingleSelect,
 } from '@/utilities/helperFunctions';
 import userServices from '../users/usersServices';
-import { Brackets, FindManyOptions, FindOptionsWhere, ILike, In, QueryRunner } from 'typeorm';
+import { Brackets, FindOptionsWhere, ILike, In, QueryRunner } from 'typeorm';
 import { SSOUser } from '@bcgov/citz-imb-sso-express';
 import { PropertyType } from '@/constants/propertyType';
 import { ProjectStatus } from '@/constants/projectStatus';
@@ -799,35 +799,62 @@ const getPropertiesForExport = async (filter: PropertyUnionFilter) => {
   const buildingIds = filteredProperties
     .filter((p) => p.PropertyTypeId === PropertyType.BUILDING)
     .map((b) => b.Id);
-  // Use IDs from filtered properties to get those properites with joins
-  const parcelQueryOptions: FindManyOptions<Parcel> = {
-    relations: {
-      CreatedBy: true,
-      UpdatedBy: true,
-      Evaluations: true,
-      Fiscals: true,
-    },
-    where: {
-      Id: In(parcelIds),
-    },
-  };
-  const buildingQueryOptions: FindManyOptions<Building> = {
-    relations: {
-      CreatedBy: true,
-      UpdatedBy: true,
-      Evaluations: true,
-      Fiscals: true,
-    },
-    where: { Id: In(buildingIds) },
-  };
-  let properties: (Parcel | Building)[] = [];
-  properties = properties.concat(
-    await AppDataSource.getRepository(Parcel).find(parcelQueryOptions),
+
+  /**
+   * For some reason, getting the data in multiple calls and filtering here is faster than letting TypeORM do it.
+   *
+   * Getting evals, fiscals, and filtering separately: 850-1050ms
+   * Getting as as part of joins, WHERE clause with TypeORM: 1587-1672ms
+   */
+
+  const ongoingFinds = [];
+  ongoingFinds.push(AppDataSource.getRepository(Parcel).find());
+  ongoingFinds.push(AppDataSource.getRepository(Building).find());
+  // Order these to guarantee the find operation later gets the most recent one.
+  ongoingFinds.push(
+    AppDataSource.getRepository(ParcelEvaluation).find({ order: { Year: 'DESC' } }),
   );
-  properties = properties.concat(
-    await AppDataSource.getRepository(Building).find(buildingQueryOptions),
+  ongoingFinds.push(
+    AppDataSource.getRepository(ParcelFiscal).find({ order: { FiscalYear: 'DESC' } }),
   );
-  return properties;
+  ongoingFinds.push(
+    AppDataSource.getRepository(BuildingEvaluation).find({ order: { Year: 'DESC' } }),
+  );
+  ongoingFinds.push(
+    AppDataSource.getRepository(BuildingFiscal).find({ order: { FiscalYear: 'DESC' } }),
+  );
+
+  // Wait for all database requests to resolve, then build the parcels and buildings lists
+  // Use IDs from filtered properties above to filter lists
+  const resolvedFinds = await Promise.all(ongoingFinds);
+  const parcelEvaluations = resolvedFinds.at(2) as ParcelEvaluation[];
+  const parcelFiscals = resolvedFinds.at(3) as ParcelFiscal[];
+  const buildingEvaluations = resolvedFinds.at(4) as BuildingEvaluation[];
+  const buildingFiscals = resolvedFinds.at(5) as BuildingFiscal[];
+  const parcels: Parcel[] = (resolvedFinds.at(0) as Parcel[])
+    .filter((p: Parcel) => parcelIds.includes(p.Id))
+    .map((p: Parcel) => {
+      const evaluation = parcelEvaluations.find((pe) => pe.ParcelId === p.Id);
+      const fiscal = parcelFiscals.find((pf) => pf.ParcelId === p.Id);
+      return {
+        ...p,
+        Evaluations: evaluation ? [evaluation] : undefined,
+        Fiscals: fiscal ? [fiscal] : undefined,
+      };
+    });
+  const buildings: Building[] = (resolvedFinds.at(1) as Building[])
+    .filter((b: Building) => buildingIds.includes(b.Id))
+    .map((b: Building) => {
+      const evaluation = buildingEvaluations.find((be) => be.BuildingId === b.Id);
+      const fiscal = buildingFiscals.find((bf) => bf.BuildingId === b.Id);
+      return {
+        ...b,
+        Evaluations: evaluation ? [evaluation] : undefined,
+        Fiscals: fiscal ? [fiscal] : undefined,
+      };
+    });
+
+  return [...parcels, ...buildings];
 };
 
 /**

express-api/tests/testUtils/factories.ts

@@ -217,7 +217,7 @@ export const produceSSO = (props?: Partial<SSOUser>): SSOUser => {
   };
 };
 
-export const produceParcel = (): Parcel => {
+export const produceParcel = (props?: Partial<Parcel>): Parcel => {
   const id = faker.number.int({ max: 10 });
   return {
     Id: faker.number.int({ max: 10 }),
@@ -252,11 +252,12 @@ export const produceParcel = (): Parcel => {
     CreatedBy: undefined,
     UpdatedById: undefined,
     UpdatedBy: undefined,
-    Fiscals: produceParcelFiscal(id),
-    Evaluations: produceParcelEvaluation(id),
+    Fiscals: produceParcelFiscals(id),
+    Evaluations: produceParcelEvaluations(id),
     DeletedById: null,
     DeletedOn: null,
     DeletedBy: undefined,
+    ...props,
   };
 };
 
@@ -284,7 +285,7 @@ export const produceEmail = (props: Partial<IEmail>): IEmail => {
   return email;
 };
 
-export const produceBuilding = (): Building => {
+export const produceBuilding = (props?: Partial<Building>): Building => {
   const agencyId = faker.number.int();
   const id = faker.number.int({ max: 10 });
   return {
@@ -328,17 +329,18 @@ export const produceBuilding = (): Building => {
     CreatedBy: undefined,
     UpdatedById: undefined,
     UpdatedBy: undefined,
-    Fiscals: produceBuildingFiscal(id),
-    Evaluations: produceBuildingEvaluation(id),
+    Fiscals: produceBuildingFiscals(id),
+    Evaluations: produceBuildingEvaluations(id),
     PID: undefined,
     PIN: undefined,
     DeletedById: null,
     DeletedOn: null,
     DeletedBy: undefined,
+    ...props,
   };
 };
 
-export const produceBuildingEvaluation = (
+export const produceBuildingEvaluations = (
   buildingId: number,
   props?: Partial<BuildingEvaluation>,
 ): BuildingEvaluation[] => {
@@ -355,7 +357,7 @@ export const produceBuildingEvaluation = (
   return [evaluation];
 };
 
-export const produceBuildingFiscal = (
+export const produceBuildingFiscals = (
   buildingId: number,
   props?: Partial<BuildingFiscal>,
 ): BuildingFiscal[] => {
@@ -370,7 +372,7 @@ export const produceBuildingFiscal = (
   return [fiscal];
 };
 
-export const produceParcelEvaluation = (
+export const produceParcelEvaluations = (
   parcelId: number,
   props?: Partial<ParcelEvaluation>,
 ): ParcelEvaluation[] => {
@@ -388,7 +390,7 @@ export const produceParcelEvaluation = (
   return [evaluation];
 };
 
-export const produceParcelFiscal = (
+export const produceParcelFiscals = (
   parcelId: number,
   props?: Partial<ParcelFiscal>,
 ): ParcelFiscal[] => {

express-api/tests/unit/services/buildings/buildingService.test.ts

@@ -2,8 +2,8 @@ import { AppDataSource } from '@/appDataSource';
 import { Building } from '@/typeorm/Entities/Building';
 import {
   produceBuilding,
-  produceBuildingEvaluation,
-  produceBuildingFiscal,
+  produceBuildingEvaluations,
+  produceBuildingFiscals,
   produceSSO,
   produceUser,
 } from 'tests/testUtils/factories';
@@ -26,23 +26,23 @@ const _buildingSave = jest
 
 const _buildingFiscalFindOne = jest
   .spyOn(AppDataSource.getRepository(BuildingFiscal), 'findOne')
-  .mockImplementation(async () => produceBuildingFiscal(1)[0]);
+  .mockImplementation(async () => produceBuildingFiscals(1)[0]);
 
 const _buildingEvaluationFindOne = jest
   .spyOn(AppDataSource.getRepository(BuildingEvaluation), 'findOne')
-  .mockImplementation(async () => produceBuildingEvaluation(1)[0]);
+  .mockImplementation(async () => produceBuildingEvaluations(1)[0]);
 
 const _buildingFindOne = jest
   .spyOn(buildingRepo, 'findOne')
   .mockImplementation(async () => produceBuilding());
 
 jest
   .spyOn(AppDataSource.getRepository(BuildingFiscal), 'find')
-  .mockImplementation(async () => produceBuildingFiscal(1));
+  .mockImplementation(async () => produceBuildingFiscals(1));
 
 jest
   .spyOn(AppDataSource.getRepository(BuildingEvaluation), 'find')
-  .mockImplementation(async () => produceBuildingEvaluation(1));
+  .mockImplementation(async () => produceBuildingEvaluations(1));
 
 const _mockStartTransaction = jest.fn(async () => {});
 const _mockRollbackTransaction = jest.fn(async () => {});

express-api/tests/unit/services/parcels/parcelsService.test.ts

@@ -2,8 +2,8 @@ import { AppDataSource } from '@/appDataSource';
 import { Parcel } from '@/typeorm/Entities/Parcel';
 import {
   produceParcel,
-  produceParcelEvaluation,
-  produceParcelFiscal,
+  produceParcelEvaluations,
+  produceParcelFiscals,
   produceSSO,
   produceUser,
 } from 'tests/testUtils/factories';
@@ -28,7 +28,7 @@ const _parcelSave = jest
 const _parcelFindOne = jest.spyOn(parcelRepo, 'findOne').mockImplementation(async () => {
   const parcel = produceParcel();
   const { Id } = parcel;
-  produceParcelFiscal(Id);
+  produceParcelFiscals(Id);
   return parcel;
 });
 
@@ -42,10 +42,10 @@ const _parcelFindOne = jest.spyOn(parcelRepo, 'findOne').mockImplementation(asyn
 
 const _parcelEvaluationFindOne = jest
   .spyOn(AppDataSource.getRepository(ParcelEvaluation), 'findOne')
-  .mockImplementation(async () => produceParcelEvaluation(1)[0]);
+  .mockImplementation(async () => produceParcelEvaluations(1)[0]);
 const _parcelFiscalFindOne = jest
   .spyOn(AppDataSource.getRepository(ParcelFiscal), 'findOne')
-  .mockImplementation(async () => produceParcelFiscal(1)[0]);
+  .mockImplementation(async () => produceParcelFiscals(1)[0]);
 
 // const _parcelFindOne = jest
 //   .spyOn(parcelRepo, 'findOne')
@@ -60,10 +60,10 @@ jest.spyOn(userServices, 'getAgencies').mockImplementation(async () => []);
 
 jest
   .spyOn(AppDataSource.getRepository(ParcelEvaluation), 'find')
-  .mockImplementation(async () => produceParcelEvaluation(1));
+  .mockImplementation(async () => produceParcelEvaluations(1));
 jest
   .spyOn(AppDataSource.getRepository(ParcelFiscal), 'find')
-  .mockImplementation(async () => produceParcelFiscal(1));
+  .mockImplementation(async () => produceParcelFiscals(1));
 
 const _mockStartTransaction = jest.fn(async () => {});
 const _mockRollbackTransaction = jest.fn(async () => {});

express-api/tests/unit/services/properties/propertyServices.test.ts

@@ -1,5 +1,6 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { AppDataSource } from '@/appDataSource';
+import { PropertyType } from '@/constants/propertyType';
 import { Roles } from '@/constants/roles';
 import propertyServices, {
   getAgencyOrThrowIfMismatched,
@@ -36,10 +37,10 @@ import {
   produceAdminArea,
   produceUser,
   produceImportResult,
-  produceParcelEvaluation,
-  produceParcelFiscal,
-  produceBuildingEvaluation,
-  produceBuildingFiscal,
+  produceParcelEvaluations,
+  produceParcelFiscals,
+  produceBuildingEvaluations,
+  produceBuildingFiscals,
   produceSSO,
   produceProjectStatus,
   produceProjectProperty,
@@ -80,8 +81,17 @@ const _propertyUnionCreateQueryBuilder: any = {
   take: () => _propertyUnionCreateQueryBuilder,
   skip: () => _propertyUnionCreateQueryBuilder,
   orderBy: () => _propertyUnionCreateQueryBuilder,
-  getMany: () => [producePropertyUnion()],
-  getManyAndCount: () => [[producePropertyUnion()], 1],
+  getMany: () => [
+    producePropertyUnion({ Id: 1, PropertyTypeId: PropertyType.LAND }),
+    producePropertyUnion({ Id: 1, PropertyTypeId: PropertyType.BUILDING }),
+  ],
+  getManyAndCount: () => [
+    [
+      producePropertyUnion({ Id: 1, PropertyTypeId: PropertyType.LAND }),
+      producePropertyUnion({ Id: 1, PropertyTypeId: PropertyType.BUILDING }),
+    ],
+    1,
+  ],
 };
 
 const _projectStatusCreateQueryBuilder: any = {
@@ -148,12 +158,24 @@ jest
   .spyOn(AppDataSource.getRepository(PropertyUnion), 'createQueryBuilder')
   .mockImplementation(() => _propertyUnionCreateQueryBuilder);
 
-jest
+const parcelRepoSpy = jest
   .spyOn(AppDataSource.getRepository(Parcel), 'find')
   .mockImplementation(async () => [produceParcel()]);
-jest
+const buildingRepoSpy = jest
   .spyOn(AppDataSource.getRepository(Building), 'find')
   .mockImplementation(async () => [produceBuilding()]);
+jest
+  .spyOn(AppDataSource.getRepository(ParcelEvaluation), 'find')
+  .mockImplementation(async () => produceParcelEvaluations(1));
+jest
+  .spyOn(AppDataSource.getRepository(ParcelFiscal), 'find')
+  .mockImplementation(async () => produceParcelFiscals(1));
+jest
+  .spyOn(AppDataSource.getRepository(BuildingEvaluation), 'find')
+  .mockImplementation(async () => produceBuildingEvaluations(1));
+jest
+  .spyOn(AppDataSource.getRepository(BuildingFiscal), 'find')
+  .mockImplementation(async () => produceBuildingFiscals(1));
 jest
   .spyOn(AppDataSource.getRepository(Parcel), 'save')
   .mockImplementation(async () => produceParcel());
@@ -198,13 +220,13 @@ const _mockEntityManager = {
     } else if (entityClass === Building) {
       return produceBuilding();
     } else if (entityClass === ParcelEvaluation) {
-      return produceParcelEvaluation(1, { Year: 2023 });
+      return produceParcelEvaluations(1, { Year: 2023 });
     } else if (entityClass === ParcelFiscal) {
-      return produceParcelFiscal(1, { FiscalYear: 2023 });
+      return produceParcelFiscals(1, { FiscalYear: 2023 });
     } else if (entityClass === BuildingEvaluation) {
-      return produceBuildingEvaluation(1, { Year: 2023 });
+      return produceBuildingEvaluations(1, { Year: 2023 });
     } else if (entityClass === BuildingFiscal) {
-      return produceBuildingFiscal(1, { FiscalYear: 2023 });
+      return produceBuildingFiscals(1, { FiscalYear: 2023 });
     } else {
       return [];
     }
@@ -295,6 +317,8 @@ describe('UNIT - Property Services', () => {
 
   describe('getPropertiesforExport', () => {
     it('should get a list of properties based on the filter', async () => {
+      parcelRepoSpy.mockImplementationOnce(async () => [produceParcel({ Id: 1 })]);
+      buildingRepoSpy.mockImplementationOnce(async () => [produceBuilding({ Id: 1 })]);
       const result = await propertyServices.getPropertiesForExport({
         pid: 'contains,123',
         pin: 'contains,456',

react-app/src/components/property/PropertyTable.tsx

@@ -251,8 +251,9 @@ const PropertyTable = (props: IPropertyTable) => {
           property.AdministrativeAreaId,
         )?.Name,
         Postal: property.Postal,
-        PID: property.PID,
+        PID: pidFormatter(property.PID),
         PIN: property.PIN,
+        'Is Sensitive': property.IsSensitive,
         'Assessed Value': property.Evaluations?.length
           ? property.Evaluations.sort(
               (