From 3fb2a87e292ea6c67936d60234768345649f66c5 Mon Sep 17 00:00:00 2001 From: Macgregor Aubertin-Young <108430771+mauberti-bc@users.noreply.github.com> Date: Wed, 28 Aug 2024 10:08:15 -0700 Subject: [PATCH] Endpoint for retrieving hierarchy information for multiple TSNs (#250) * Add taxon hierarchy endpoint --- .../taxonomy/taxon/tsn/hierarchy.test.ts | 80 +++++++++++ api/src/paths/taxonomy/taxon/tsn/hierarchy.ts | 108 +++++++++++++++ api/src/services/itis-service.test.ts | 131 +++++++++++++++++- api/src/services/itis-service.ts | 94 ++++++++++++- 4 files changed, 410 insertions(+), 3 deletions(-) create mode 100644 api/src/paths/taxonomy/taxon/tsn/hierarchy.test.ts create mode 100644 api/src/paths/taxonomy/taxon/tsn/hierarchy.ts diff --git a/api/src/paths/taxonomy/taxon/tsn/hierarchy.test.ts b/api/src/paths/taxonomy/taxon/tsn/hierarchy.test.ts new file mode 100644 index 00000000..13167f1c --- /dev/null +++ b/api/src/paths/taxonomy/taxon/tsn/hierarchy.test.ts @@ -0,0 +1,80 @@ +import Ajv from 'ajv'; +import chai, { expect } from 'chai'; +import { describe } from 'mocha'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { GET } from '.'; +import * as db from '../../../../database/db'; +import { ItisService } from '../../../../services/itis-service'; +import { getMockDBConnection, getRequestHandlerMocks } from '../../../../__mocks__/db'; +import { getHierarchyForTSNs } from './hierarchy'; + +chai.use(sinonChai); + +describe('taxonomy/taxon/tsn/hierarchy', () => { + describe('openapi schema', () => { + const ajv = new Ajv(); + + it('is valid openapi v3 schema', () => { + expect(ajv.validateSchema(GET.apiDoc as unknown as object)).to.be.true; + }); + }); + + describe('getHierarchyForTSNs', () => { + afterEach(() => { + sinon.restore(); + }); + + it('returns an empty array if no species are found', async () => { + const dbConnectionObj = getMockDBConnection(); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + const getHierarchyForTSNsStub = sinon.stub(ItisService.prototype, 'getHierarchyForTSNs').resolves([]); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + mockReq.query = { + tsn: ['1', '2'] + }; + + const requestHandler = getHierarchyForTSNs(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(getHierarchyForTSNsStub).to.have.been.calledWith([1, 2]); + + expect(mockRes.statusValue).to.equal(200); + expect(mockRes.jsonValue).to.eql([]); + }); + + it('returns an array of species', async () => { + const dbConnectionObj = getMockDBConnection(); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + const mock1 = { + tsn: 1, + commonNames: ['something'], + scientificName: 'string', + hierarchy: [3, 2, 1] + } as unknown as any; + const mock2 = { tsn: '2', commonNames: [], scientificName: 'string', hierarchy: [3, 2] } as unknown as any; + + const getHierarchyForTSNsStub = sinon.stub(ItisService.prototype, 'getHierarchyForTSNs').resolves([mock1, mock2]); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + mockReq.query = { + tsn: ['1', '2'] + }; + + const requestHandler = getHierarchyForTSNs(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(getHierarchyForTSNsStub).to.have.been.calledWith([1, 2]); + + expect(mockRes.jsonValue).to.eql([mock1, mock2]); + expect(mockRes.statusValue).to.equal(200); + }); + }); +}); diff --git a/api/src/paths/taxonomy/taxon/tsn/hierarchy.ts b/api/src/paths/taxonomy/taxon/tsn/hierarchy.ts new file mode 100644 index 00000000..8f20c57c --- /dev/null +++ b/api/src/paths/taxonomy/taxon/tsn/hierarchy.ts @@ -0,0 +1,108 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { getAPIUserDBConnection } from '../../../../database/db'; +import { ItisService } from '../../../../services/itis-service'; +import { getLogger } from '../../../../utils/logger'; + +const defaultLog = getLogger('paths/taxonomy/taxon/tsn/hierarchy'); + +export const GET: Operation = [getHierarchyForTSNs()]; + +GET.apiDoc = { + description: 'Get taxon hierarchy information by TSN ids.', + tags: ['taxon_id'], + security: [], + parameters: [ + { + description: 'Taxon TSN ids.', + in: 'query', + name: 'tsn', + schema: { + type: 'array', + description: 'One or more Taxon TSN ids.', + items: { + type: 'integer', + minimum: 0, + minItems: 1, + maxItems: 100 + } + }, + required: true + } + ], + responses: { + 200: { + description: 'Taxonomy response.', + content: { + 'application/json': { + schema: { + type: 'array', + items: { + title: 'Species', + type: 'object', + description: 'Taxon hierarchy response object with an array of parent TSNs', + required: ['tsn', 'hierarchy'], + properties: { + tsn: { + type: 'integer' + }, + hierarchy: { + type: 'array', + description: + 'Array of parent TSNs in descending order, where the highest-ranking parent is first and the TSN for which the hierarchy was requested is last.', + items: { type: 'integer' } + } + }, + additionalProperties: false + } + } + } + } + }, + 400: { + $ref: '#/components/responses/400' + }, + 401: { + $ref: '#/components/responses/401' + }, + 500: { + $ref: '#/components/responses/500' + }, + default: { + $ref: '#/components/responses/default' + } + } +}; + +/** + * Get taxon by ITIS TSN. + * + * @returns {RequestHandler} + */ +export function getHierarchyForTSNs(): RequestHandler { + return async (req, res) => { + defaultLog.debug({ label: 'getHierarchyForTSNs', message: 'query params', query: req.query }); + + const connection = getAPIUserDBConnection(); + + const tsnIds: number[] = (req.query.tsn as string[]).map(Number); + + try { + await connection.open(); + + const itisService = new ItisService(); + + const response = await itisService.getHierarchyForTSNs(tsnIds); + + connection.commit(); + + res.status(200).json(response); + } catch (error) { + defaultLog.error({ label: 'getHierarchyForTSNs', message: 'error', error }); + connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/services/itis-service.test.ts b/api/src/services/itis-service.test.ts index be8b8dd7..92c2c037 100644 --- a/api/src/services/itis-service.test.ts +++ b/api/src/services/itis-service.test.ts @@ -3,7 +3,7 @@ import chai, { expect } from 'chai'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; import { ApiGeneralError } from '../errors/api-error'; -import { ItisService } from './itis-service'; +import { ItisService, ItisSolrSearchResponseHierarchy } from './itis-service'; chai.use(sinonChai); @@ -182,6 +182,65 @@ describe('ItisService', () => { }); }); + describe('getHierarchyForTSNs', async () => { + it('returns array of hierarchy objects for tsns', async () => { + const mockTsns = [1, 2]; + const mockTsnHierarchies = [ + { tsn: mockTsns[0], hierarchyTSN: ['$3$2$1$'] }, + { tsn: mockTsns[1], hierarchyTSN: ['$3$2$'] } + ]; + const mockAxiosResponse = { + data: { + response: { + docs: mockTsnHierarchies + } + } + }; + + const getItisSolrTsnHierarchyUrlStub = sinon + .stub(ItisService.prototype, 'getItisSolrTsnHierarchyUrl') + .returns('url'); + + const axiosStub = sinon.stub(axios, 'get').resolves(mockAxiosResponse); + + const itisService = new ItisService(); + + const response = await itisService.getHierarchyForTSNs(mockTsns); + + expect(getItisSolrTsnHierarchyUrlStub).to.have.been.calledWith(mockTsns); + expect(axiosStub).to.have.been.calledWith('url'); + expect(response).to.eql([ + { + tsn: mockTsns[0], + hierarchy: [3, 2, 1] + }, + { + tsn: mockTsns[1], + hierarchy: [3, 2] + } + ]); + }); + + it('catches and re-throws an error', async () => { + const mockTsns = [1, 2]; + sinon.stub(axios, 'get').rejects(new Error('a test error')); + + const itisService = new ItisService(); + const getItisSolrTsnHierarchyUrlStub = sinon + .stub(ItisService.prototype, 'getItisSolrTsnHierarchyUrl') + .resolves('url'); + + try { + await itisService.getHierarchyForTSNs(mockTsns); + + expect.fail(); + } catch (error) { + expect((error as ApiGeneralError).message).to.equal('a test error'); + expect(getItisSolrTsnHierarchyUrlStub).to.have.been.calledWith(mockTsns); + } + }); + }); + describe('getItisSolrTermSearchUrl', () => { it('throws an error when itis solr url is not set', async () => { process.env.ITIS_SOLR_URL = ''; @@ -210,6 +269,35 @@ describe('ItisService', () => { }); }); + describe('getItisSolrTsnHierarchyUrl', () => { + const mockTsns = [1]; + it('throws an error when itis solr url is not set', async () => { + process.env.ITIS_SOLR_URL = ''; + + const itisService = new ItisService(); + + try { + await itisService.getItisSolrTsnHierarchyUrl(mockTsns); + + expect.fail(); + } catch (error) { + expect((error as ApiGeneralError).message).to.equal('Failed to build ITIS query.'); + } + }); + + it('returns a valid url', async () => { + process.env.ITIS_SOLR_URL = 'https://services.itis.gov/'; + + const itisService = new ItisService(); + + const response = await itisService.getItisSolrTsnHierarchyUrl(mockTsns); + + expect(response).to.equal( + 'https://services.itis.gov/??wt=json&sort=kingdom+asc&rows=150&omitHeader=true&fl=tsn+hierarchyTSN&&q=tsn:1' + ); + }); + }); + describe('getItisSolrTsnSearchUrl', () => { it('throws an error when itis solr url is not set', async () => { process.env.ITIS_SOLR_URL = ''; @@ -237,4 +325,45 @@ describe('ItisService', () => { ); }); }); + + describe('getItisSolrTsnSearchUrl', () => { + it('throws an error when itis solr url is not set', async () => { + process.env.ITIS_SOLR_URL = ''; + + const itisService = new ItisService(); + + try { + await itisService.getItisSolrTsnSearchUrl([123]); + + expect.fail(); + } catch (error) { + expect((error as ApiGeneralError).message).to.equal('Failed to build ITIS query.'); + } + }); + + it('returns a valid url', async () => { + process.env.ITIS_SOLR_URL = 'https://services.itis.gov/'; + + const itisService = new ItisService(); + + const response = await itisService.getItisSolrTsnSearchUrl([123]); + + expect(response).to.equal( + 'https://services.itis.gov/??wt=json&sort=kingdom+asc&rows=150&omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage+rank&&q=tsn:123' + ); + }); + }); + + describe('_sanitizeHierarchyData', () => { + it('turns an ITIS hierarchy string into an array'), + () => { + const mockData: ItisSolrSearchResponseHierarchy[] = [{ tsn: '1', hierarchyTSN: ['$3$2$1$'] }]; + + const itisService = new ItisService(); + + const result = itisService._sanitizeHierarchyData(mockData); + + expect(result).to.eql([3, 2, 1]); + }; + }); }); diff --git a/api/src/services/itis-service.ts b/api/src/services/itis-service.ts index 52704e72..290df3b8 100644 --- a/api/src/services/itis-service.ts +++ b/api/src/services/itis-service.ts @@ -17,6 +17,25 @@ export type ItisSolrSearchResponse = { rank: string; }; +export type ItisSolrSearchResponseHierarchy = { + tsn: string; + hierarchyTSN: [string]; // Array with one item +}; + +export type TSNWithHierarchy = { + tsn: number; + hierarchy: number[]; +}; + +/** + * Generic base type for the ITIS Solr service + */ +type ItisSolrResponseBase = { + response: { + docs: T; + }; +}; + /** * Service for retrieving and processing taxonomic data from the Integrated Taxonomic Information System (ITIS). * @@ -38,7 +57,7 @@ export class ItisService { defaultLog.debug({ label: 'searchItisByTerm', message: 'url', url }); - const response = await axios.get(url); + const response = await axios.get>(url); if (!response.data || !response.data.response || !response.data.response.docs) { return []; @@ -66,7 +85,7 @@ export class ItisService { defaultLog.debug({ label: 'searchItisByTSN', message: 'url', url }); - const response = await axios.get(url); + const response = await axios.get>(url); if (!response.data || !response.data.response || !response.data.response.docs) { return []; @@ -75,10 +94,32 @@ export class ItisService { return response.data.response.docs; } + /** + * Returns the parent hierarchy for multiple TSNs + * + * @param {number[]} tsnIds + * @return {*} {Promise} + * @memberof ItisService + */ + async getHierarchyForTSNs(tsnIds: number[]): Promise { + const url = this.getItisSolrTsnHierarchyUrl(tsnIds); + + defaultLog.debug({ label: 'getHierarchyForTSNs', message: 'url', url }); + + const response = await axios.get>(url); + + if (!response.data || !response.data.response || !response.data.response.docs) { + return []; + } + + return this._sanitizeHierarchyData(response.data.response.docs); + } + /** * Cleans up the ITIS search response data * * @param {ItisSolrSearchResponse[]} data + * @return {*} {Promise} * @memberof ItisService */ _sanitizeItisData = (data: ItisSolrSearchResponse[]): TaxonSearchResult[] => { @@ -95,6 +136,23 @@ export class ItisService { }); }; + /** + * Cleans up the ITIS hierarchy response data + * + * @param {ItisSolrSearchResponse[]} data + * @return {TSNWithHierarchy[]} + * @memberof ItisService + */ + _sanitizeHierarchyData = (data: ItisSolrSearchResponseHierarchy[]): TSNWithHierarchy[] => { + return data.map((item: ItisSolrSearchResponseHierarchy) => ({ + tsn: Number(item.tsn), + hierarchy: item.hierarchyTSN[0] + .split('$') + .filter((part) => part !== '') + .map((tsn) => Number(tsn)) + })); + }; + /** * Get the ITIS SORL search-by-term URL. * @@ -139,6 +197,28 @@ export class ItisService { )}&${this._getItisSolrFilterParam()}&&q=${this._getItisSolrTsnSearch(searchTsnIds)}`; } + /** + * Get the ITIS SOLR search-by-tsn URL for hierarchy information + * + * @param {number[]} tsnIds + * @return {*} {string} + * @memberof ItisService + */ + getItisSolrTsnHierarchyUrl(tsnIds: number[]): string { + const itisUrl = this._getItisSolrUrl(); + + if (!itisUrl) { + defaultLog.debug({ label: 'getItisTsnHierarchyUrl', message: 'Environment variable ITIS_URL is not defined.' }); + throw new Error('Failed to build ITIS query.'); + } + + return `${itisUrl}??${this._getItisSolrTypeParam()}&${this._getItisSolrSortParam( + ['kingdom'], + ['asc'], + 150 + )}&${this._getItisSolrHierarchyParam()}&&q=${this._getItisSolrTsnSearch(tsnIds)}`; + } + /** * Get ITIS SOLR base URL. * @@ -182,6 +262,16 @@ export class ItisService { return 'omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage+rank'; } + /** + * Get ITIS SOLR filter param. + * + * @return {*} {string} + * @memberof ItisService + */ + _getItisSolrHierarchyParam(): string { + return 'omitHeader=true&fl=tsn+hierarchyTSN'; + } + /** * Get ITIS SOLR query by search term param. *