From 4f265dc152b2fedfa23584299cde70c1245ae5a7 Mon Sep 17 00:00:00 2001 From: Goeme Nthomiwa Date: Mon, 13 May 2024 11:12:51 -0700 Subject: [PATCH] fix: use custom query to get reports (#473) --- .../routes/external-consumer-routes.spec.ts | 85 ++--- .../external-consumer-service.spec.ts | 174 +++-------- .../v1/services/external-consumer-service.ts | 294 +++++++++++------- 3 files changed, 259 insertions(+), 294 deletions(-) diff --git a/backend/src/v1/routes/external-consumer-routes.spec.ts b/backend/src/v1/routes/external-consumer-routes.spec.ts index 6e91033a8..a2450cb13 100644 --- a/backend/src/v1/routes/external-consumer-routes.spec.ts +++ b/backend/src/v1/routes/external-consumer-routes.spec.ts @@ -4,17 +4,14 @@ import router from './external-consumer-routes'; import { faker } from '@faker-js/faker'; const mockCount = jest.fn(); -const mockFindMany = jest.fn(); +const mockQueryRaw = jest.fn(); jest.mock('../prisma/prisma-client-readonly-replica', () => { return { __esModule: true, default: { $replica: () => { return { - pay_transparency_report: { - count: () => mockCount(), - findMany: (...args) => mockFindMany(...args), - }, + $queryRaw: (...args) => mockQueryRaw(...args), }; }, }, @@ -34,31 +31,21 @@ const REPORT = { report_start_date: faker.date.past(), report_end_date: faker.date.past(), report_status: 'Published', - pay_transparency_company: { - company_name: faker.company.name(), - province: faker.location.state(), - bceid_business_guid: faker.string.uuid(), - country: faker.location.country(), - city: faker.location.city(), - postal_code: faker.location.zipCode(), - address_line1: faker.location.streetAddress(), - address_line2: faker.location.streetAddress(), - }, - employee_count_range: { - employee_count_range: '50-299', - }, - naics_code_pay_transparency_report_naics_codeTonaics_code: { - naics_code: '11', - naics_label: faker.lorem.words(3), - }, - pay_transparency_calculated_data: [ - { - value: faker.number.float(), - is_suppressed: false, - calculation_code: { calculation_code: `${faker.number.int()}` }, - }, - ], - report_history: [], + company_name: faker.company.name(), + province: faker.location.state(), + bceid_business_guid: faker.string.uuid(), + country: faker.location.country(), + city: faker.location.city(), + postal_code: faker.location.zipCode(), + address_line1: faker.location.streetAddress(), + address_line2: faker.location.streetAddress(), + + employee_count_range: '50-299', + naics_label: faker.lorem.words(3), + + value: faker.number.float(), + is_suppressed: false, + calculation_code: faker.number.int(), }; describe('external-consumer-routes', () => { @@ -71,7 +58,7 @@ describe('external-consumer-routes', () => { describe('/ GET', () => { it('should return data if user doeas not send query params', () => { mockCount.mockReturnValue(1); - mockFindMany.mockReturnValue([REPORT]); + mockQueryRaw.mockReturnValue([REPORT]); return request(app) .get('') .expect(200) @@ -79,44 +66,38 @@ describe('external-consumer-routes', () => { expect(body).toEqual({ page: 0, pageSize: 1000, - history: [], records: [ { calculated_data: [ { calculation_code: - REPORT.pay_transparency_calculated_data[0] - .calculation_code.calculation_code, + REPORT.calculation_code, is_suppressed: - REPORT.pay_transparency_calculated_data[0].is_suppressed, - value: REPORT.pay_transparency_calculated_data[0].value, + REPORT.is_suppressed, + value: REPORT.value, }, ], company_address_line1: - REPORT.pay_transparency_company.address_line1, + REPORT.address_line1, company_address_line2: - REPORT.pay_transparency_company.address_line2, + REPORT.address_line2, company_bceid_business_guid: - REPORT.pay_transparency_company.bceid_business_guid, - company_city: REPORT.pay_transparency_company.city, - company_country: REPORT.pay_transparency_company.country, + REPORT.bceid_business_guid, + company_city: REPORT.city, + company_country: REPORT.country, company_id: REPORT.company_id, - company_name: REPORT.pay_transparency_company.company_name, + company_name: REPORT.company_name, company_postal_code: - REPORT.pay_transparency_company.postal_code, - company_province: REPORT.pay_transparency_company.province, + REPORT.postal_code, + company_province: REPORT.province, create_date: REPORT.create_date.toISOString(), data_constraints: REPORT.data_constraints, employee_count_range: - REPORT.employee_count_range.employee_count_range, + REPORT.employee_count_range, naics_code: - REPORT - .naics_code_pay_transparency_report_naics_codeTonaics_code - .naics_code, + REPORT.naics_code, naics_code_label: - REPORT - .naics_code_pay_transparency_report_naics_codeTonaics_code - .naics_label, + REPORT.naics_label, report_end_date: REPORT.report_end_date.toISOString(), report_id: REPORT.report_id, report_start_date: REPORT.report_start_date.toISOString(), @@ -141,7 +122,7 @@ describe('external-consumer-routes', () => { }); }); it('should fail if request fails to get reports', () => { - mockFindMany.mockRejectedValue({}); + mockQueryRaw.mockRejectedValue({}); return request(app).get('').expect(200); }); }); diff --git a/backend/src/v1/services/external-consumer-service.spec.ts b/backend/src/v1/services/external-consumer-service.spec.ts index d8fb00a86..6487bdeff 100644 --- a/backend/src/v1/services/external-consumer-service.spec.ts +++ b/backend/src/v1/services/external-consumer-service.spec.ts @@ -1,17 +1,14 @@ import { faker } from '@faker-js/faker'; import { externalConsumerService } from './external-consumer-service'; -const mockCount = jest.fn(); -const mockFindMany = jest.fn(); + +const mockQueryRaw = jest.fn(); jest.mock('../prisma/prisma-client-readonly-replica', () => { return { __esModule: true, default: { $replica: () => { return { - pay_transparency_report: { - count: () => mockCount(), - findMany: (...args) => mockFindMany(...args), - }, + $queryRaw: (...args) => mockQueryRaw(...args), }; }, }, @@ -30,69 +27,20 @@ const testData = { report_start_date: faker.date.past(), report_end_date: faker.date.past(), report_status: 'Published', - pay_transparency_company: { - company_name: faker.company.name(), - province: faker.location.state(), - bceid_business_guid: faker.string.uuid(), - country: faker.location.country(), - city: faker.location.city(), - postal_code: faker.location.zipCode(), - address_line1: faker.location.streetAddress(), - address_line2: faker.location.streetAddress(), - }, - employee_count_range: { - employee_count_range: '50-299', - }, - naics_code_pay_transparency_report_naics_codeTonaics_code: { - naics_code: '11', - naics_label: faker.lorem.words(3), - }, - pay_transparency_calculated_data: [ - { - value: faker.number.float(), - is_suppressed: false, - calculation_code: { calculation_code: `${faker.number.int()}` }, - }, - ], - report_history: [ - { - report_id: faker.string.uuid(), - company_id: faker.string.uuid(), - naics_code: '11', - create_date: faker.date.past(), - update_date: faker.date.past(), - data_constraints: faker.lorem.sentence(), - user_comment: faker.lorem.sentence(), - revision: '12', - report_start_date: faker.date.past(), - report_end_date: faker.date.past(), - report_status: 'Published', - pay_transparency_company: { - company_name: faker.company.name(), - province: faker.location.state(), - bceid_business_guid: faker.string.uuid(), - country: faker.location.country(), - city: faker.location.city(), - postal_code: faker.location.zipCode(), - address_line1: faker.location.streetAddress(), - address_line2: faker.location.streetAddress(), - }, - employee_count_range: { - employee_count_range: '50-299', - }, - naics_code_report_history_naics_codeTonaics_code: { - naics_code: '11', - naics_label: faker.lorem.words(3), - }, - calculated_data_history: [ - { - value: faker.number.float(), - is_suppressed: false, - calculation_code: { calculation_code: `${faker.number.int()}` }, - }, - ], - }, - ], + reporting_year: 2024, + company_name: faker.company.name(), + province: faker.location.state(), + bceid_business_guid: faker.string.uuid(), + country: faker.location.country(), + city: faker.location.city(), + postal_code: faker.location.zipCode(), + address_line1: faker.location.streetAddress(), + address_line2: faker.location.streetAddress(), + employee_count_range: '50-299', + naics_label: faker.lorem.words(3), + value: faker.number.float(), + is_suppressed: false, + calculation_code: faker.number.int() }; describe('external-consumer-service', () => { @@ -101,99 +49,51 @@ describe('external-consumer-service', () => { }); it('should return reports with defaults values', async () => { - mockCount.mockReturnValue(1); - mockFindMany.mockReturnValue([testData]); + mockQueryRaw.mockReturnValue([testData]); const results = await externalConsumerService.exportDataWithPagination(); expect(results.page).toBe(0); expect(results.records[0]).toStrictEqual({ calculated_data: [ { is_suppressed: - testData.pay_transparency_calculated_data[0].is_suppressed, - value: testData.pay_transparency_calculated_data[0].value, + testData.is_suppressed, + value: testData.value, calculation_code: - testData.pay_transparency_calculated_data[0].calculation_code - .calculation_code, + testData.calculation_code, }, ], - company_address_line1: testData.pay_transparency_company.address_line1, - company_address_line2: testData.pay_transparency_company.address_line2, + company_address_line1: testData.address_line1, + company_address_line2: testData.address_line2, company_bceid_business_guid: - testData.pay_transparency_company.bceid_business_guid, - company_city: testData.pay_transparency_company.city, - company_country: testData.pay_transparency_company.country, + testData.bceid_business_guid, + company_city: testData.city, + company_country: testData.country, company_id: testData.company_id, - company_name: testData.pay_transparency_company.company_name, - company_postal_code: testData.pay_transparency_company.postal_code, - company_province: testData.pay_transparency_company.province, + company_name: testData.company_name, + company_postal_code: testData.postal_code, + company_province: testData.province, create_date: testData.create_date, data_constraints: testData.data_constraints, - employee_count_range: testData.employee_count_range.employee_count_range, + employee_count_range: testData.employee_count_range, naics_code: - testData.naics_code_pay_transparency_report_naics_codeTonaics_code + testData .naics_code, naics_code_label: - testData.naics_code_pay_transparency_report_naics_codeTonaics_code + testData .naics_label, report_end_date: testData.report_end_date, report_id: testData.report_id, report_start_date: testData.report_start_date, report_status: testData.report_status, + reporting_year: testData.reporting_year, revision: testData.revision, update_date: testData.update_date, user_comment: testData.user_comment, }); - - expect(results.history[0]).toStrictEqual({ - calculated_data: [ - { - is_suppressed: - testData.report_history[0].calculated_data_history[0].is_suppressed, - value: testData.report_history[0].calculated_data_history[0].value, - calculation_code: - testData.report_history[0].calculated_data_history[0] - .calculation_code.calculation_code, - }, - ], - company_address_line1: - testData.report_history[0].pay_transparency_company.address_line1, - company_address_line2: - testData.report_history[0].pay_transparency_company.address_line2, - company_bceid_business_guid: - testData.report_history[0].pay_transparency_company.bceid_business_guid, - company_city: testData.report_history[0].pay_transparency_company.city, - company_country: - testData.report_history[0].pay_transparency_company.country, - company_id: testData.report_history[0].company_id, - company_name: - testData.report_history[0].pay_transparency_company.company_name, - company_postal_code: - testData.report_history[0].pay_transparency_company.postal_code, - company_province: - testData.report_history[0].pay_transparency_company.province, - create_date: testData.report_history[0].create_date, - data_constraints: testData.report_history[0].data_constraints, - employee_count_range: - testData.report_history[0].employee_count_range.employee_count_range, - naics_code: - testData.report_history[0] - .naics_code_report_history_naics_codeTonaics_code.naics_code, - naics_code_label: - testData.report_history[0] - .naics_code_report_history_naics_codeTonaics_code.naics_label, - report_end_date: testData.report_history[0].report_end_date, - report_id: testData.report_history[0].report_id, - report_start_date: testData.report_history[0].report_start_date, - report_status: testData.report_history[0].report_status, - revision: testData.report_history[0].revision, - update_date: testData.report_history[0].update_date, - user_comment: testData.report_history[0].user_comment, - }); }); it('should parse date strings', async () => { - mockCount.mockReturnValue(1); - mockFindMany.mockReturnValue([testData]); + mockQueryRaw.mockReturnValue([testData]); const results = await externalConsumerService.exportDataWithPagination( '2024-01-01', '2024-01-01', @@ -204,8 +104,7 @@ describe('external-consumer-service', () => { }); it('should fail parse invalid date strings', async () => { - mockCount.mockReturnValue(1); - mockFindMany.mockReturnValue([testData]); + mockQueryRaw.mockReturnValue([testData]); try { await externalConsumerService.exportDataWithPagination( '20241-01-01', @@ -220,8 +119,7 @@ describe('external-consumer-service', () => { } }); it('should fail when endDate is before the startDate', async () => { - mockCount.mockReturnValue(1); - mockFindMany.mockReturnValue([testData]); + mockQueryRaw.mockReturnValue([testData]); try { await externalConsumerService.exportDataWithPagination( '2024-01-01', diff --git a/backend/src/v1/services/external-consumer-service.ts b/backend/src/v1/services/external-consumer-service.ts index 6ca276624..1b96dd685 100644 --- a/backend/src/v1/services/external-consumer-service.ts +++ b/backend/src/v1/services/external-consumer-service.ts @@ -6,55 +6,80 @@ import { convert, nativeJs, } from '@js-joda/core'; -import pick from 'lodash/pick'; -import flatten from 'lodash/flatten'; +import groupBy from 'lodash/groupBy'; +import keys from 'lodash/keys'; import { PayTransparencyUserError } from './file-upload-service'; +import { Prisma } from '@prisma/client'; -const denormalizeCompany = (company) => { - return { - company_name: company.company_name, - company_province: company.province, - company_bceid_business_guid: company.bceid_business_guid, - company_city: company.city, - company_country: company.country, - company_postal_code: company.postal_code, - company_address_line1: company.address_line1, - company_address_line2: company.address_line2, - }; +type RawQueryResult = { + report_id: string; + report_change_id: string; + company_id: string; + user_id: string; + user_comment: any; + employee_count_range_id: string; + naics_code: string; + report_start_date: string; + report_end_date: string; + create_date: string; + update_date: string; + create_user: string; + update_user: string; + report_status: string; + revision: number; + data_constraints: any; + is_unlocked: boolean; + reporting_year: number; + report_unlock_date: any; + naics_label: string; + effective_date: string; + expiry_date: any; + naics_year: string; + employee_count_range: string; + calculation_code_id: string; + value: string; + is_suppressed: boolean; + calculation_code: string; + company_name: string; + province: string; + bceid_business_guid: string; + city: string; + country: string; + postal_code: string; + address_line1: string; + address_line2: string; }; -const denormalizeReport = ( - report, - getNaicsCode: (report) => { naics_code: string; naics_label: string }, - getCalculatedData: (report) => { - value: string; - is_suppressed: string; - calculation_code: any; - }[], -) => { +const buildReport = (data: RawQueryResult[]) => { + const first = data[0]; + return { - ...pick(report, [ - 'report_id', - 'company_id', - 'naics_code', - 'create_date', - 'update_date', - 'data_constraints', - 'user_comment', - 'revision', - 'report_start_date', - 'report_end_date', - 'report_status', - 'reporting_year', - ]), - ...denormalizeCompany(report.pay_transparency_company), - employee_count_range: report.employee_count_range.employee_count_range, - naics_code: getNaicsCode(report).naics_code, - naics_code_label: getNaicsCode(report).naics_label, - calculated_data: getCalculatedData(report).map((data) => ({ - value: data.value, - is_suppressed: data.is_suppressed, - calculation_code: data.calculation_code.calculation_code, + report_id: first.report_id, + company_id: first.company_id, + naics_code: first.naics_code, + create_date: first.create_date, + update_date: first.update_date, + data_constraints: first.data_constraints, + user_comment: first.user_comment, + revision: first.revision, + report_start_date: first.report_start_date, + report_end_date: first.report_end_date, + report_status: first.report_status, + reporting_year: first.reporting_year, + company_name: first.company_name, + company_province: first.province, + company_bceid_business_guid: first.bceid_business_guid, + company_city: first.city, + company_country: first.country, + company_postal_code: first.postal_code, + company_address_line1: first.address_line1, + company_address_line2: first.address_line2, + employee_count_range: first.employee_count_range, + naics_code_label: first.naics_label, + calculated_data: data.map((item) => ({ + value: item.value, + is_suppressed: item.is_suppressed, + calculation_code: item.calculation_code, })), }; }; @@ -120,72 +145,133 @@ const externalConsumerService = { ); } + /** + * 1) Create a union of the pay_transparency_report and report_history table as reports + * 2) Create a union of the pay_transparency_calculated_data and calculated_data_history as calculated + * 3) Paginate the reports + * 4) Join reports and calculated_data based on report_change_id + */ + const getReportsQuery = Prisma.sql`select * + from ((select report.report_id, + report.report_id as report_change_id, + report.company_id, + report.user_id, + report.user_comment, + report.employee_count_range_id, + report.naics_code, + report.report_start_date, + report.report_end_date, + report.create_date, + report.update_date, + report.create_user, + report.update_user, + report.report_status, + report.revision, + report.data_constraints, + report.is_unlocked, + report.reporting_year, + report.report_unlock_date, + naics_code.naics_label, + company.company_id, + company.company_name, + company.bceid_business_guid, + company.address_line1, + company.address_line2, + company.city, + company.province, + company.country, + company.postal_code, + employee_count_range.employee_count_range + from pay_transparency_report as report + left join naics_code as naics_code on naics_code.naics_code = report.naics_code + left join pay_transparency_company as company on company.company_id = report.company_id + left join employee_count_range as employee_count_range on employee_count_range.employee_count_range_id = report.employee_count_range_id + where report_status = 'Published' + and (report.update_date >= ${convert(startDt).toDate()} + and report.update_date < ${convert(endDt).toDate()}) + union + (select report.report_id, + report.report_history_id as report_change_id, + report.company_id, + report.user_id, + report.user_comment, + report.employee_count_range_id, + report.naics_code, + report.report_start_date, + report.report_end_date, + report.create_date, + report.update_date, + report.create_user, + report.update_user, + report.report_status, + report.revision, + report.data_constraints, + report.is_unlocked, + report.reporting_year, + report.report_unlock_date, + naics_code.naics_label, + company.company_id, + company.company_name, + company.bceid_business_guid, + company.address_line1, + company.address_line2, + company.city, + company.province, + company.country, + company.postal_code, + employee_count_range.employee_count_range + from report_history as report + left join naics_code as naics_code on naics_code.naics_code = report.naics_code + left join pay_transparency_company as company on company.company_id = report.company_id + left join employee_count_range as employee_count_range on employee_count_range.employee_count_range_id = report.employee_count_range_id + where report_status = 'Published' + and (report.update_date >= ${convert(startDt).toDate()} + and report.update_date < ${convert(endDt).toDate()}))) + order by update_date + offset ${offset} + limit ${limit}) as reports + +left join + (select data.report_id as calculated_data_report_id, + data.calculation_code_id, + data.value, + data.is_suppressed, + code.calculation_code + from + (select data.report_id, + data.calculation_code_id, + data.value, + data.is_suppressed + from pay_transparency_calculated_data as data where data.update_date >= ${convert(startDt).toDate()} + and data.update_date < ${convert(endDt).toDate()} + union + (select data.report_history_id as report_id, + data.calculation_code_id, + data.value, + data.is_suppressed + from calculated_data_history as data where data.update_date >= ${convert(startDt).toDate()} + and data.update_date < ${convert(endDt).toDate()})) as data + left join calculation_code as code on code.calculation_code_id = data.calculation_code_id) as calculated_data on calculated_data.calculated_data_report_id = reports.report_change_id`; + const results = await prismaReadOnlyReplica .$replica() - .pay_transparency_report.findMany({ - where: { - update_date: { - gte: convert(startDt).toDate(), - lte: convert(endDt).toDate(), - }, - report_status: 'Published', - }, - include: { - naics_code_pay_transparency_report_naics_codeTonaics_code: true, - employee_count_range: true, - pay_transparency_calculated_data: { - include: { - calculation_code: true, - }, - }, - pay_transparency_company: true, - report_history: { - include: { - naics_code_report_history_naics_codeTonaics_code: true, - employee_count_range: true, - calculated_data_history: { - include: { - calculation_code: true, - }, - }, - pay_transparency_company: true, - }, - where: { - update_date: { - gte: convert(startDt).toDate(), - lte: convert(endDt).toDate(), - }, - }, - }, - }, - skip: offset, - take: limit, - }); + .$queryRaw(getReportsQuery); + const uniqueReports: Record = groupBy( + results, + (x) => x.report_change_id, + ); + + const uniqueReportIds: string[] = keys(uniqueReports); + + const reports = uniqueReportIds.map((id_rev) => { + const data = uniqueReports[id_rev]; + return buildReport(data); + }); return { page: offset, pageSize: limit, - history: flatten(results.map((r) => r.report_history)).map((report) => { - return { - ...denormalizeReport( - report, - (r) => r.naics_code_report_history_naics_codeTonaics_code, - (r) => r.calculated_data_history, - ), - }; - }), - records: [ - ...results.map((report) => { - return { - ...denormalizeReport( - report, - (r) => - r.naics_code_pay_transparency_report_naics_codeTonaics_code, - (r) => r.pay_transparency_calculated_data, - ), - }; - }), - ], + records: reports, }; }, };