diff --git a/.vscode/settings.json b/.vscode/settings.json index 3b6641073..23830fb42 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,3 @@ { - "git.ignoreLimitWarning": true -} \ No newline at end of file + "git.ignoreLimitWarning": true +} diff --git a/api/.pipeline/config.js b/api/.pipeline/config.js index 122576a51..677ebcfad 100644 --- a/api/.pipeline/config.js +++ b/api/.pipeline/config.js @@ -83,6 +83,7 @@ const phases = { elasticsearchURL: 'http://es01:9200', elasticsearchEmlIndex: 'eml', elasticsearchTaxonomyIndex: 'taxonomy_3.0.0', + itisSolrUrl: 'https://services.itis.gov', s3KeyPrefix: (isStaticDeployment && 'biohub') || `local/${deployChangeId}/biohub`, tz: config.timezone.api, sso: config.sso.dev, @@ -111,6 +112,7 @@ const phases = { elasticsearchURL: 'http://es01.a0ec71-dev:9200', // TODO: Update to test instance (es is not yet deployed to test) elasticsearchEmlIndex: 'eml', elasticsearchTaxonomyIndex: 'taxonomy_3.0.0', + itisSolrUrl: 'https://services.itis.gov', s3KeyPrefix: 'biohub', tz: config.timezone.api, sso: config.sso.test, @@ -139,6 +141,7 @@ const phases = { elasticsearchURL: 'http://es01:9200', elasticsearchEmlIndex: 'eml', elasticsearchTaxonomyIndex: 'taxonomy_3.0.0', + itisSolrUrl: 'https://services.itis.gov', s3KeyPrefix: 'biohub', tz: config.timezone.api, sso: config.sso.prod, diff --git a/api/.pipeline/lib/api.deploy.js b/api/.pipeline/lib/api.deploy.js index c89d3be6e..aac11a3d1 100644 --- a/api/.pipeline/lib/api.deploy.js +++ b/api/.pipeline/lib/api.deploy.js @@ -38,6 +38,8 @@ const apiDeploy = async (settings) => { ELASTICSEARCH_URL: phases[phase].elasticsearchURL, ELASTICSEARCH_EML_INDEX: phases[phase].elasticsearchEmlIndex, ELASTICSEARCH_TAXONOMY_INDEX: phases[phase].elasticsearchTaxonomyIndex, + // ITIS SOLR + ITIS_SOLR_URL: phases[phase].itisSolrUrl, // S3 (Object Store) S3_KEY_PREFIX: phases[phase].s3KeyPrefix, OBJECT_STORE_SECRETS: 'biohubbc-object-store', diff --git a/api/.pipeline/templates/api.dc.yaml b/api/.pipeline/templates/api.dc.yaml index 791fd86fa..6951951bc 100644 --- a/api/.pipeline/templates/api.dc.yaml +++ b/api/.pipeline/templates/api.dc.yaml @@ -54,6 +54,11 @@ parameters: description: Application timezone required: false value: 'America/Vancouver' + # ITIS SOLR + - name: ITIS_SOLR_URL + description: ITIS SOLR API URL + value: 'https://services.itis.gov' + required: true # Keycloak - name: KEYCLOAK_HOST description: Key clock login url @@ -263,6 +268,9 @@ objects: value: ${ELASTICSEARCH_EML_INDEX} - name: ELASTICSEARCH_TAXONOMY_INDEX value: ${ELASTICSEARCH_TAXONOMY_INDEX} + # ITIS SOLR + - name: ITIS_SOLR_URL + value: ${ITIS_SOLR_URL} - name: S3_KEY_PREFIX value: ${S3_KEY_PREFIX} - name: TZ diff --git a/api/package-lock.json b/api/package-lock.json index e15f9c164..9fda741dc 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -5688,9 +5688,9 @@ "dev": true }, "jsonpath-plus": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz", - "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==" + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-8.0.0.tgz", + "integrity": "sha512-+AOBHcQvRr8DcWVIkfOCCCLSlYgQuNZ+gFNqwkBrNpdUfdfkcrbO4ml3F587fWUMFOmoy6D9c+5wrghgjN3mbg==" }, "jsonwebtoken": { "version": "8.5.1", diff --git a/api/src/__mocks__/db.ts b/api/src/__mocks__/db.ts index 4d6d044a6..98f1f1e6b 100644 --- a/api/src/__mocks__/db.ts +++ b/api/src/__mocks__/db.ts @@ -77,6 +77,18 @@ export class MockRes { return this; }); + /** + * The value of the last `.setHeader()` call. + * + * @memberof MockRes + */ + headerValue: any; + setHeader = sinon.fake((header: any) => { + this.headerValue = header; + + return this; + }); + /** * The value of the last `.json()` call. * diff --git a/api/src/paths/submission/intake.ts b/api/src/paths/submission/intake.ts index 60a106b7b..53d65840a 100644 --- a/api/src/paths/submission/intake.ts +++ b/api/src/paths/submission/intake.ts @@ -147,7 +147,7 @@ export function submissionIntake(): RequestHandler { const searchIndexService = new SearchIndexService(connection); const regionService = new RegionService(connection); - // validate theubmission + // validate the submission if (!(await validationService.validateSubmissionFeatures([submissionFeature]))) { throw new HTTP400('Invalid submission'); // TODO return details on why the submission is invalid } diff --git a/api/src/paths/taxonomy/species/list.ts b/api/src/paths/taxonomy/species/list.ts deleted file mode 100644 index 518c69dad..000000000 --- a/api/src/paths/taxonomy/species/list.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { RequestHandler } from 'express'; -import { Operation } from 'express-openapi'; -import qs from 'qs'; -import { TaxonomyService } from '../../../services/taxonomy-service'; -import { getLogger } from '../../../utils/logger'; - -const defaultLog = getLogger('paths/taxonomy/list'); - -export const GET: Operation = [getSpeciesFromIds()]; - -GET.apiDoc = { - description: 'Gets the labels of the taxonomic units identified by the provided list of ids.', - tags: ['taxonomy'], - parameters: [ - { - description: 'Taxonomy ids.', - in: 'query', - name: 'ids', - required: true, - schema: { - type: 'string' - } - } - ], - responses: { - 200: { - description: 'Taxonomy search response object.', - content: { - 'application/json': { - schema: { - type: 'object', - properties: { - searchResponse: { - type: 'array', - items: { - title: 'Species', - type: 'object', - required: ['id', 'label'], - properties: { - id: { - type: 'string' - }, - label: { - type: 'string' - } - } - } - } - } - } - } - } - }, - 400: { - $ref: '#/components/responses/400' - }, - 500: { - $ref: '#/components/responses/500' - }, - default: { - $ref: '#/components/responses/default' - } - } -}; - -/** - * Get taxonomic search results. - * - * @returns {RequestHandler} - */ -export function getSpeciesFromIds(): RequestHandler { - return async (req, res) => { - defaultLog.debug({ label: 'getSearchResults', message: 'request body', req_body: req.query }); - - const ids = Object.values(qs.parse(req.query.ids?.toString() || '')); - - try { - const taxonomyService = new TaxonomyService(); - const response = await taxonomyService.getSpeciesFromIds(ids as string[]); - - res.status(200).json({ searchResponse: response }); - } catch (error) { - defaultLog.error({ label: 'getSearchResults', message: 'error', error }); - throw error; - } - }; -} diff --git a/api/src/paths/taxonomy/species/search.ts b/api/src/paths/taxonomy/species/search.ts deleted file mode 100644 index 196b6e668..000000000 --- a/api/src/paths/taxonomy/species/search.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { RequestHandler } from 'express'; -import { Operation } from 'express-openapi'; -import { TaxonomyService } from '../../../services/taxonomy-service'; -import { getLogger } from '../../../utils/logger'; - -const defaultLog = getLogger('paths/taxonomy/search'); - -export const GET: Operation = [searchSpecies()]; - -GET.apiDoc = { - description: 'Gets a list of taxonomic units.', - tags: ['taxonomy'], - parameters: [ - { - description: 'Taxonomy search parameters.', - in: 'query', - name: 'terms', - required: true, - schema: { - type: 'string' - } - } - ], - responses: { - 200: { - description: 'Taxonomy search response object.', - content: { - 'application/json': { - schema: { - type: 'object', - properties: { - searchResponse: { - type: 'array', - items: { - title: 'Species', - type: 'object', - required: ['id', 'code', 'label'], - properties: { - id: { - type: 'string' - }, - code: { - type: 'string' - }, - label: { - type: 'string' - } - } - } - } - } - } - } - } - }, - 400: { - $ref: '#/components/responses/400' - }, - 500: { - $ref: '#/components/responses/500' - }, - default: { - $ref: '#/components/responses/default' - } - } -}; - -/** - * Get taxonomic search results. - * - * @returns {RequestHandler} - */ -export function searchSpecies(): RequestHandler { - return async (req, res) => { - defaultLog.debug({ label: 'getSearchResults', message: 'request params', req_params: req.query.terms }); - - const term = String(req.query.terms) || ''; - try { - const taxonomySearch = new TaxonomyService(); - const response = await taxonomySearch.searchSpecies(term.toLowerCase()); - - res.status(200).json({ searchResponse: response }); - } catch (error) { - defaultLog.error({ label: 'getSearchResults', message: 'error', error }); - throw error; - } - }; -} diff --git a/api/src/paths/taxonomy/taxon/index.test.ts b/api/src/paths/taxonomy/taxon/index.test.ts new file mode 100644 index 000000000..a1e0d7ae0 --- /dev/null +++ b/api/src/paths/taxonomy/taxon/index.test.ts @@ -0,0 +1,97 @@ +import Ajv from 'ajv'; +import chai, { expect } from 'chai'; +import { describe } from 'mocha'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { findTaxonBySearchTerms, GET } from '.'; +import * as db from '../../../database/db'; +import { HTTPError } from '../../../errors/http-error'; +import { ItisService } from '../../../services/itis-service'; +import { getMockDBConnection, getRequestHandlerMocks } from '../../../__mocks__/db'; + +chai.use(sinonChai); + +describe('taxon', () => { + describe('openapi schema', () => { + const ajv = new Ajv(); + + it('is valid openapi v3 schema', () => { + expect(ajv.validateSchema(GET.apiDoc as unknown as object)).to.be.true; + }); + }); + + describe('findTaxonBySearchTerms', () => { + afterEach(() => { + sinon.restore(); + }); + + it('returns an empty array if no species are found', async () => { + const dbConnectionObj = getMockDBConnection(); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + const getSpeciesFromIdsStub = sinon.stub(ItisService.prototype, 'searchItisByTerm').resolves([]); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + mockReq.query = { + terms: '' + }; + + const requestHandler = findTaxonBySearchTerms(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(getSpeciesFromIdsStub).to.have.been.calledWith(''); + + expect(mockRes.statusValue).to.equal(200); + expect(mockRes.jsonValue).to.eql({ searchResponse: [] }); + }); + + it('returns an array of species', async () => { + const dbConnectionObj = getMockDBConnection(); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + const mock1 = { id: '1', commonName: 'something', scientificName: 'string' } as unknown as any; + const mock2 = { id: '2', commonName: null, scientificName: 'string' } as unknown as any; + + const getSpeciesFromIdsStub = sinon.stub(ItisService.prototype, 'searchItisByTerm').resolves([mock1, mock2]); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + mockReq.query = { + terms: 't' + }; + + const requestHandler = findTaxonBySearchTerms(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(getSpeciesFromIdsStub).to.have.been.calledWith('t'); + + expect(mockRes.jsonValue).to.eql({ searchResponse: [mock1, mock2] }); + expect(mockRes.statusValue).to.equal(200); + }); + + it('catches error, and re-throws error', async () => { + const dbConnectionObj = getMockDBConnection({ rollback: sinon.stub(), release: sinon.stub() }); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + sinon.stub(ItisService.prototype, 'searchItisByTerm').rejects(new Error('a test error')); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + mockReq.query = { + ids: 'a' + }; + + try { + const requestHandler = findTaxonBySearchTerms(); + + await requestHandler(mockReq, mockRes, mockNext); + expect.fail(); + } catch (actualError) { + expect((actualError as HTTPError).message).to.equal('a test error'); + } + }); + }); +}); diff --git a/api/src/paths/taxonomy/taxon/index.ts b/api/src/paths/taxonomy/taxon/index.ts new file mode 100644 index 000000000..aaec03249 --- /dev/null +++ b/api/src/paths/taxonomy/taxon/index.ts @@ -0,0 +1,110 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { ItisService } from '../../../services/itis-service'; +import { getLogger } from '../../../utils/logger'; + +const defaultLog = getLogger('paths/taxonomy/taxon'); + +export const GET: Operation = [findTaxonBySearchTerms()]; + +GET.apiDoc = { + description: 'Find taxon records by search criteria.', + tags: ['taxonomy'], + security: [ + { + Bearer: [] + } + ], + parameters: [ + { + description: 'Taxonomy search terms.', + in: 'query', + name: 'terms', + required: true, + schema: { + type: 'array', + description: 'One or more search terms.', + items: { + type: 'string', + minLength: 1 + }, + minItems: 1 + } + } + ], + responses: { + 200: { + description: 'Taxonomy response.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + searchResponse: { + type: 'array', + items: { + title: 'Taxon', + type: 'object', + required: ['tsn', 'commonName', 'scientificName'], + properties: { + tsn: { + type: 'integer' + }, + commonName: { + type: 'string', + nullable: true + }, + scientificName: { + type: 'string' + } + }, + additionalProperties: false + } + } + }, + additionalProperties: false + } + } + } + }, + 400: { + $ref: '#/components/responses/400' + }, + 401: { + $ref: '#/components/responses/401' + }, + 500: { + $ref: '#/components/responses/500' + }, + default: { + $ref: '#/components/responses/default' + } + } +}; + +/** + * Get taxon by search terms. + * + * @returns {RequestHandler} + */ +export function findTaxonBySearchTerms(): RequestHandler { + return async (req, res) => { + defaultLog.debug({ label: 'findTaxonBySearchTerms', message: 'query params', query: req.query }); + + const searchTerms = req.query.terms as string[]; + + try { + const itisService = new ItisService(); + + const response = await itisService.searchItisByTerm(searchTerms); + + // Overwrite default cache-control header, allow caching up to 7 days + res.setHeader('Cache-Control', 'max-age=604800'); + + res.status(200).json({ searchResponse: response }); + } catch (error) { + defaultLog.error({ label: 'findTaxonBySearchTerms', message: 'error', error }); + throw error; + } + }; +} diff --git a/api/src/paths/taxonomy/taxon/tsn/index.test.ts b/api/src/paths/taxonomy/taxon/tsn/index.test.ts new file mode 100644 index 000000000..7102e9266 --- /dev/null +++ b/api/src/paths/taxonomy/taxon/tsn/index.test.ts @@ -0,0 +1,97 @@ +import Ajv from 'ajv'; +import chai, { expect } from 'chai'; +import { describe } from 'mocha'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { GET, getTaxonByTSN } from '.'; +import * as db from '../../../../database/db'; +import { HTTPError } from '../../../../errors/http-error'; +import { TaxonomyService } from '../../../../services/taxonomy-service'; +import { getMockDBConnection, getRequestHandlerMocks } from '../../../../__mocks__/db'; + +chai.use(sinonChai); + +describe('tsn', () => { + describe('openapi schema', () => { + const ajv = new Ajv(); + + it('is valid openapi v3 schema', () => { + expect(ajv.validateSchema(GET.apiDoc as unknown as object)).to.be.true; + }); + }); + + describe('getTaxonByTSN', () => { + afterEach(() => { + sinon.restore(); + }); + + it('returns an empty array if no species are found', async () => { + const dbConnectionObj = getMockDBConnection(); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + const getTaxonByTsnIdsStub = sinon.stub(TaxonomyService.prototype, 'getTaxonByTsnIds').resolves([]); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + mockReq.query = { + tsn: ['1', '2'] + }; + + const requestHandler = getTaxonByTSN(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(getTaxonByTsnIdsStub).to.have.been.calledWith([1, 2]); + + expect(mockRes.statusValue).to.equal(200); + expect(mockRes.jsonValue).to.eql({ searchResponse: [] }); + }); + + it('returns an array of species', async () => { + const dbConnectionObj = getMockDBConnection(); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + const mock1 = { tsn: '1', commonName: 'something', scientificName: 'string' } as unknown as any; + const mock2 = { tsn: '2', commonName: null, scientificName: 'string' } as unknown as any; + + const getTaxonByTsnIdsStub = sinon.stub(TaxonomyService.prototype, 'getTaxonByTsnIds').resolves([mock1, mock2]); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + mockReq.query = { + tsn: ['1', '2'] + }; + + const requestHandler = getTaxonByTSN(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(getTaxonByTsnIdsStub).to.have.been.calledWith([1, 2]); + + expect(mockRes.jsonValue).to.eql({ searchResponse: [mock1, mock2] }); + expect(mockRes.statusValue).to.equal(200); + }); + + it('catches error, and re-throws error', async () => { + const dbConnectionObj = getMockDBConnection({ rollback: sinon.stub(), release: sinon.stub() }); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + sinon.stub(TaxonomyService.prototype, 'getTaxonByTsnIds').rejects(new Error('a test error')); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + mockReq.query = { + tsn: ['1', '2'] + }; + + try { + const requestHandler = getTaxonByTSN(); + + await requestHandler(mockReq, mockRes, mockNext); + expect.fail(); + } catch (actualError) { + expect((actualError as HTTPError).message).to.equal('a test error'); + } + }); + }); +}); diff --git a/api/src/paths/taxonomy/taxon/tsn/index.ts b/api/src/paths/taxonomy/taxon/tsn/index.ts new file mode 100644 index 000000000..7ac992dac --- /dev/null +++ b/api/src/paths/taxonomy/taxon/tsn/index.ts @@ -0,0 +1,121 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { getAPIUserDBConnection } from '../../../../database/db'; +import { TaxonomyService } from '../../../../services/taxonomy-service'; +import { getLogger } from '../../../../utils/logger'; + +const defaultLog = getLogger('paths/taxonomy/taxon/tsn'); + +export const GET: Operation = [getTaxonByTSN()]; + +GET.apiDoc = { + description: 'Get taxon records by TSN ids.', + tags: ['taxonomy'], + security: [ + { + Bearer: [] + } + ], + parameters: [ + { + description: 'Taxon TSN ids.', + in: 'query', + name: 'tsn', + schema: { + type: 'array', + description: 'One or more Taxon TSN ids.', + items: { + type: 'integer', + minimum: 0, + minItems: 1, + maxItems: 100 + } + }, + required: true + } + ], + responses: { + 200: { + description: 'Taxonomy response.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + searchResponse: { + type: 'array', + items: { + title: 'Species', + type: 'object', + required: ['tsn', 'commonName', 'scientificName'], + properties: { + tsn: { + type: 'integer' + }, + commonName: { + type: 'string', + nullable: true + }, + scientificName: { + type: 'string' + } + }, + additionalProperties: false + } + } + }, + additionalProperties: false + } + } + } + }, + 400: { + $ref: '#/components/responses/400' + }, + 401: { + $ref: '#/components/responses/401' + }, + 500: { + $ref: '#/components/responses/500' + }, + default: { + $ref: '#/components/responses/default' + } + } +}; + +/** + * Get taxon by ITIS TSN. + * + * @returns {RequestHandler} + */ +export function getTaxonByTSN(): RequestHandler { + return async (req, res) => { + defaultLog.debug({ label: 'getTaxonByTSN', message: 'query params', query: req.query }); + + const connection = getAPIUserDBConnection(); + + const tsnIds: number[] = (req.query.tsn as (string | number)[]).map(Number); + + try { + await connection.open(); + + const taxonomyService = new TaxonomyService(connection); + + const response = await taxonomyService.getTaxonByTsnIds(tsnIds); + + connection.commit(); + + // Overwrite default cache-control header, allow caching up to 7 days + res.setHeader('Cache-Control', 'max-age=604800'); + + res.status(200).json({ searchResponse: response }); + } catch (error) { + defaultLog.error({ label: 'getTaxonByTSN', message: 'error', error }); + connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/repositories/taxonomy-repository.test.ts b/api/src/repositories/taxonomy-repository.test.ts new file mode 100644 index 000000000..348654b9e --- /dev/null +++ b/api/src/repositories/taxonomy-repository.test.ts @@ -0,0 +1,128 @@ +import chai, { expect } from 'chai'; +import { describe } from 'mocha'; +import { QueryResult } from 'pg'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { getMockDBConnection } from '../__mocks__/db'; +import { TaxonomyRepository, TaxonRecord } from './taxonomy-repository'; + +chai.use(sinonChai); + +describe('TaxonomyRepository', () => { + afterEach(() => { + sinon.restore(); + }); + + describe('getTaxonByTsnIds', () => { + it('should return array of system constants', async () => { + const TaxonRecord = { + taxon_id: 1, + itis_tsn: 1, + bc_taxon_code: 'string', + itis_scientific_name: 'string', + common_name: 'string', + itis_data: {}, + record_effective_date: 'string', + record_end_date: 'string', + create_date: 'string', + create_user: 1, + update_date: 'string', + update_user: 1, + revision_count: 1 + }; + + const mockQueryResponse = { + rowCount: 1, + rows: [TaxonRecord] as unknown as TaxonRecord[] + } as unknown as Promise>; + + const mockDBConnection = getMockDBConnection({ knex: () => mockQueryResponse }); + + const taxonomyRepository = new TaxonomyRepository(mockDBConnection); + + const response = await taxonomyRepository.getTaxonByTsnIds([1]); + + expect(response).to.be.eql([TaxonRecord]); + }); + }); + + describe('addItisTaxonRecord', () => { + it('should return a new taxon record', async () => { + const mockQueryResponse = { + rowCount: 1, + rows: [ + { + taxon_id: 1, + itis_tsn: 1, + bc_taxon_code: 'string', + itis_scientific_name: 'string', + common_name: 'string', + itis_data: {}, + record_effective_date: 'string', + record_end_date: 'string', + create_date: 'string', + create_user: 1, + update_date: 'string', + update_user: 1, + revision_count: 1 + } + ] + } as unknown as Promise>; + + const mockDBConnection = getMockDBConnection({ sql: () => mockQueryResponse }); + + const taxonomyRepository = new TaxonomyRepository(mockDBConnection); + + const response = await taxonomyRepository.addItisTaxonRecord(1, 'string', 'string', {}, 'string'); + + expect(response).to.be.eql({ + taxon_id: 1, + itis_tsn: 1, + bc_taxon_code: 'string', + itis_scientific_name: 'string', + common_name: 'string', + itis_data: {}, + record_effective_date: 'string', + record_end_date: 'string', + create_date: 'string', + create_user: 1, + update_date: 'string', + update_user: 1, + revision_count: 1 + }); + }); + }); + + describe('deleteTaxonRecord', () => { + it('should return a deleted taxon record', async () => { + const mockQueryResponse = { + rowCount: 1, + rows: [ + { + taxon_id: 1, + itis_tsn: 1, + bc_taxon_code: 'string', + itis_scientific_name: 'string', + common_name: 'string', + itis_data: {}, + record_effective_date: 'string', + record_end_date: 'string', + create_date: 'string', + create_user: 1, + update_date: 'string', + update_user: 1, + revision_count: 1 + } + ] + } as unknown as Promise>; + + const mockDBConnection = getMockDBConnection({ sql: () => mockQueryResponse }); + + const taxonomyRepository = new TaxonomyRepository(mockDBConnection); + + const response = await taxonomyRepository.deleteTaxonRecord(1); + + expect(response).to.be.eql(undefined); + }); + }); +}); diff --git a/api/src/repositories/taxonomy-repository.ts b/api/src/repositories/taxonomy-repository.ts new file mode 100644 index 000000000..20f421425 --- /dev/null +++ b/api/src/repositories/taxonomy-repository.ts @@ -0,0 +1,117 @@ +import SQL from 'sql-template-strings'; +import { z } from 'zod'; +import { getKnex } from '../database/db'; +import { ApiExecuteSQLError } from '../errors/api-error'; +import { BaseRepository } from './base-repository'; + +export const TaxonRecord = z.object({ + taxon_id: z.number(), + itis_tsn: z.number(), + bc_taxon_code: z.string().nullable(), + itis_scientific_name: z.string(), + common_name: z.string().nullable(), + itis_data: z.record(z.any()), + record_effective_date: z.string(), + record_end_date: z.string().nullable(), + create_date: z.string(), + create_user: z.number(), + update_date: z.string().nullable(), + update_user: z.number().nullable(), + revision_count: z.number() +}); + +export type TaxonRecord = z.infer; + +/** + * Taxonomy Repository + * + * @export + * @class TaxonomyRepository + * @extends {BaseRepository} + */ +export class TaxonomyRepository extends BaseRepository { + /** + * Get taxon records by TSN id. + * + * @param {number[]} tsnIds + * @return {*} {Promise} + * @memberof TaxonomyRepository + */ + async getTaxonByTsnIds(tsnIds: number[]): Promise { + const queryBuilder = getKnex().queryBuilder().select('*').from('taxon').whereIn('itis_tsn', tsnIds); + + const response = await this.connection.knex(queryBuilder, TaxonRecord); + + return response.rows; + } + + /** + * Insert a new taxon record. + * + * @param {number} itisTsn + * @param {string} itisScientificName + * @param {(string | null)} commonName + * @param {Record} itisData + * @param {string} itisUpdateDate + * @return {*} {Promise} + * @memberof TaxonomyRepository + */ + async addItisTaxonRecord( + itisTsn: number, + itisScientificName: string, + commonName: string | null, + itisData: Record, + itisUpdateDate: string + ): Promise { + const sqlStatement = SQL` + INSERT INTO + taxon + ( + itis_tsn, + itis_scientific_name, + common_name, + itis_data, + itis_update_date + ) + VALUES ( + ${itisTsn}, + ${itisScientificName}, + ${commonName}, + ${itisData}, + ${itisUpdateDate} + ) + RETURNING + *; + `; + + const response = await this.connection.sql(sqlStatement, TaxonRecord); + + if (response.rowCount !== 1) { + throw new ApiExecuteSQLError('Failed to insert new taxon record', [ + 'TaxonomyRepository->addItisTaxonRecord', + 'rowCount was null or undefined, expected rowCount = 1' + ]); + } + + return response.rows[0]; + } + + /** + * Delete an existing taxon record. + * + * @param {number} taxonId + * @memberof TaxonomyRepository + */ + async deleteTaxonRecord(taxonId: number) { + const sqlStatement = SQL` + DELETE FROM + taxon + WHERE + taxon_id = ${taxonId} + RETURNING + *; + `; + + await this.connection.sql(sqlStatement); + } +} diff --git a/api/src/services/itis-service.test.ts b/api/src/services/itis-service.test.ts new file mode 100644 index 000000000..f08ec48ca --- /dev/null +++ b/api/src/services/itis-service.test.ts @@ -0,0 +1,235 @@ +import axios from 'axios'; +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { ApiGeneralError } from '../errors/api-error'; +import { ItisService } from './itis-service'; + +chai.use(sinonChai); + +describe('ItisService', () => { + afterEach(() => { + sinon.restore(); + }); + + describe('searchItisByTerm', async () => { + it('searches itis by term and returns empty when no term recognized', async () => { + const mockAxiosResponse = { + data: {} + }; + + const getItisSolrTermSearchUrlStub = sinon + .stub(ItisService.prototype, 'getItisSolrTermSearchUrl') + .resolves('url'); + + const axiosStub = sinon.stub(axios, 'get').resolves(mockAxiosResponse); + + const itisService = new ItisService(); + + const response = await itisService.searchItisByTerm(['term']); + + expect(response).to.eql([]); + + expect(axiosStub).to.have.been.calledWith('url'); + expect(getItisSolrTermSearchUrlStub).to.have.been.calledWith(['term']); + }); + + it('searches itis by term and returns list of docs', async () => { + const mockAxiosResponse = { + data: { + response: { + docs: [ + { + commonNames: ['$commonName'], + kingdom: 'kingdom', + name: 'name', + parentTSN: 'parentTSN', + scientificName: 'scientificName', + tsn: '123', + updateDate: 'updateDate', + usage: 'usage' + } + ] + } + } + }; + + const getItisSolrTermSearchUrlStub = sinon + .stub(ItisService.prototype, 'getItisSolrTermSearchUrl') + .resolves('url'); + + const axiosStub = sinon.stub(axios, 'get').resolves(mockAxiosResponse); + + const itisService = new ItisService(); + + const response = await itisService.searchItisByTerm(['term']); + + expect(response).to.eql([ + { + tsn: 123, + commonName: 'commonName', + scientificName: 'scientificName' + } + ]); + + expect(axiosStub).to.have.been.calledWith('url'); + expect(getItisSolrTermSearchUrlStub).to.have.been.calledWith(['term']); + }); + + it('catches and re-throws an error', async () => { + sinon.stub(axios, 'get').rejects(new Error('a test error')); + + const itisService = new ItisService(); + const getItisSolrTermSearchUrlStub = sinon + .stub(ItisService.prototype, 'getItisSolrTermSearchUrl') + .resolves('url'); + + try { + await itisService.searchItisByTerm(['term']); + + expect.fail(); + } catch (error) { + expect((error as ApiGeneralError).message).to.equal('a test error'); + expect(getItisSolrTermSearchUrlStub).to.have.been.calledWith(['term']); + } + }); + }); + + describe('searchItisByTSN', async () => { + it('searches itis by tsn and returns empty when no tsn recognized', async () => { + const mockAxiosResponse = { + data: {} + }; + + const getItisSolrTsnSearchUrlStub = sinon.stub(ItisService.prototype, 'getItisSolrTsnSearchUrl').resolves('url'); + + const axiosStub = sinon.stub(axios, 'get').resolves(mockAxiosResponse); + + const itisService = new ItisService(); + + const response = await itisService.searchItisByTSN([123]); + + expect(response).to.eql([]); + + expect(axiosStub).to.have.been.calledWith('url'); + expect(getItisSolrTsnSearchUrlStub).to.have.been.calledWith([123]); + }); + + it('searches itis by tsn and returns list of docs', async () => { + const mockAxiosResponse = { + data: { + response: { + docs: [ + { + commonNames: ['$commonName'], + kingdom: 'kingdom', + name: 'name', + parentTSN: 'parentTSN', + scientificName: 'scientificName', + tsn: '123', + updateDate: 'updateDate', + usage: 'usage' + } + ] + } + } + }; + + const getItisSolrTsnSearchUrlStub = sinon.stub(ItisService.prototype, 'getItisSolrTsnSearchUrl').resolves('url'); + + const axiosStub = sinon.stub(axios, 'get').resolves(mockAxiosResponse); + + const itisService = new ItisService(); + + const response = await itisService.searchItisByTSN([123]); + + expect(response).to.eql([ + { + commonNames: ['$commonName'], + kingdom: 'kingdom', + name: 'name', + parentTSN: 'parentTSN', + scientificName: 'scientificName', + tsn: '123', + updateDate: 'updateDate', + usage: 'usage' + } + ]); + + expect(axiosStub).to.have.been.calledWith('url'); + expect(getItisSolrTsnSearchUrlStub).to.have.been.calledWith([123]); + }); + + it('catches and re-throws an error', async () => { + sinon.stub(axios, 'get').rejects(new Error('a test error')); + + const itisService = new ItisService(); + const getItisSolrTsnSearchUrlStub = sinon.stub(ItisService.prototype, 'getItisSolrTsnSearchUrl').resolves('url'); + + try { + await itisService.searchItisByTSN([123]); + + expect.fail(); + } catch (error) { + expect((error as ApiGeneralError).message).to.equal('a test error'); + expect(getItisSolrTsnSearchUrlStub).to.have.been.calledWith([123]); + } + }); + }); + + describe('getItisSolrTermSearchUrl', () => { + it('throws an error when itis solr url is not set', async () => { + process.env.ITIS_SOLR_URL = ''; + + const itisService = new ItisService(); + + try { + await itisService.getItisSolrTermSearchUrl(['term']); + + expect.fail(); + } catch (error) { + expect((error as ApiGeneralError).message).to.equal('Failed to build ITIS query.'); + } + }); + + it('returns a valid url', async () => { + process.env.ITIS_SOLR_URL = 'https://services.itis.gov/'; + + const itisService = new ItisService(); + + const response = await itisService.getItisSolrTermSearchUrl(['term']); + + expect(response).to.equal( + 'https://services.itis.gov/?wt=json&sort=nameWOInd+asc&rows=25&omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage&q=((nameWOInd:*term*+AND+usage:/(valid|accepted)/)+OR+(vernacular:*term*+AND+usage:/(valid|accepted)/))' + ); + }); + }); + + describe('getItisSolrTsnSearchUrl', () => { + it('throws an error when itis solr url is not set', async () => { + process.env.ITIS_SOLR_URL = ''; + + const itisService = new ItisService(); + + try { + await itisService.getItisSolrTsnSearchUrl([123]); + + expect.fail(); + } catch (error) { + expect((error as ApiGeneralError).message).to.equal('Failed to build ITIS query.'); + } + }); + + it('returns a valid url', async () => { + process.env.ITIS_SOLR_URL = 'https://services.itis.gov/'; + + const itisService = new ItisService(); + + const response = await itisService.getItisSolrTsnSearchUrl([123]); + + expect(response).to.equal( + 'https://services.itis.gov/??wt=json&sort=nameWOInd+asc&rows=25&omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage&&q=tsn:123' + ); + }); + }); +}); diff --git a/api/src/services/itis-service.ts b/api/src/services/itis-service.ts new file mode 100644 index 000000000..306c0801b --- /dev/null +++ b/api/src/services/itis-service.ts @@ -0,0 +1,205 @@ +import axios from 'axios'; +import { getLogger } from '../utils/logger'; +import { TaxonSearchResult } from './taxonomy-service'; + +const defaultLog = getLogger('services/itis-service'); + +export type ItisSolrSearchResponse = { + commonNames: string[]; + kingdom: string; + name: string; + parentTSN: string; + scientificName: string; + tsn: string; + updateDate: string; + usage: string; +}; + +/** + * Service for retrieving and processing taxonomic data from the Integrated Taxonomic Information System (ITIS). + * + * @See https://itis.gov + * + * @export + * @class ItisService + */ +export class ItisService { + /** + * Returns the ITIS search species Query. + * + * @param {string[]} searchTerms + * @return {*} {Promise} + * @memberof ItisService + */ + async searchItisByTerm(searchTerms: string[]): Promise { + const url = await this.getItisSolrTermSearchUrl(searchTerms); + + defaultLog.debug({ label: 'searchItisByTerm', message: 'url', url }); + + const response = await axios.get(url); + + if (!response.data || !response.data.response || !response.data.response.docs) { + return []; + } + + return this._sanitizeItisData(response.data.response.docs); + } + + /** + * Returns the ITIS search by TSN. + * + * @param {number[]} searchTsnIds + * @return {*} {Promise} + * @memberof ItisService + */ + async searchItisByTSN(searchTsnIds: number[]): Promise { + const url = await this.getItisSolrTsnSearchUrl(searchTsnIds); + + defaultLog.debug({ label: 'searchItisByTSN', message: 'url', url }); + + const response = await axios.get(url); + + if (!response.data || !response.data.response || !response.data.response.docs) { + return []; + } + + return response.data.response.docs; + } + + /** + * Cleans up the ITIS search response data. + * + * @param {ItisSolrSearchResponse[]} data + * @memberof ItisService + */ + _sanitizeItisData = (data: ItisSolrSearchResponse[]): TaxonSearchResult[] => { + return data.map((item: ItisSolrSearchResponse) => { + const commonName = item.commonNames ? item.commonNames[0].split('$')[1] : null; + + return { + tsn: Number(item.tsn), + commonName: commonName, + scientificName: item.scientificName + }; + }); + }; + + /** + * Get the ITIS SORL search-by-term URL. + * + * @param {string} searchTerms + * @return {*} {Promise} + * @memberof ItisService + */ + async getItisSolrTermSearchUrl(searchTerms: string[]): Promise { + const itisUrl = this._getItisSolrUrl(); + + if (!itisUrl) { + defaultLog.debug({ label: 'getItisTermSearchUrl', message: 'Environment variable ITIS_URL is not defined.' }); + throw new Error('Failed to build ITIS query.'); + } + + return `${itisUrl}?${this._getItisSolrTypeParam()}&${this._getItisSolrSortParam( + 'nameWOInd', + 'asc', + 25 + )}&${this._getItisSolrFilterParam()}&${this._getItisSolrQueryParam(searchTerms)}`; + } + + /** + * Get the ITIS SOLR search-by-tsn URL. + * + * @param {number[]} searchTsnIds + * @return {*} {Promise} + * @memberof ItisService + */ + async getItisSolrTsnSearchUrl(searchTsnIds: number[]): Promise { + const itisUrl = this._getItisSolrUrl(); + + if (!itisUrl) { + defaultLog.debug({ label: 'getItisTsnSearchUrl', message: 'Environment variable ITIS_URL is not defined.' }); + throw new Error('Failed to build ITIS query.'); + } + + return `${itisUrl}??${this._getItisSolrTypeParam()}&${this._getItisSolrSortParam( + 'nameWOInd', + 'asc', + 25 + )}&${this._getItisSolrFilterParam()}&&q=${this._getItisSolrTsnSearch(searchTsnIds)}`; + } + + /** + * Get ITIS SOLR base URL. + * + * @return {*} {(string | undefined)} + * @memberof ItisService + */ + _getItisSolrUrl(): string | undefined { + return process.env.ITIS_SOLR_URL; + } + + /** + * Get ITIS SOLR type param. + * + * @return {*} {string} + * @memberof ItisService + */ + _getItisSolrTypeParam(): string { + return 'wt=json'; + } + + /** + * Get ITIS SOLR sort param. + * + * @param {string} sortBy + * @param {('asc' | 'desc')} sortDir + * @param {number} [limit=25] + * @return {*} {string} + * @memberof ItisService + */ + _getItisSolrSortParam(sortBy: string, sortDir: 'asc' | 'desc', limit = 25): string { + return `sort=${sortBy}+${sortDir}&rows=${limit}`; + } + + /** + * Get ITIS SOLR filter param. + * + * @return {*} {string} + * @memberof ItisService + */ + _getItisSolrFilterParam(): string { + return 'omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage'; + } + + /** + * Get ITIS SOLR query by search term param. + * + * @param {string[]} searchTerms + * @return {*} {string} + * @memberof ItisService + */ + _getItisSolrQueryParam(searchTerms: string[]): string { + const queryParams = searchTerms + .map((term) => term.trim()) + .filter(Boolean) + .map((term) => { + // Logical OR between scientific name and vernacular name + return `((nameWOInd:*${term}*+AND+usage:/(valid|accepted)/)+OR+(vernacular:*${term}*+AND+usage:/(valid|accepted)/))`; + }) + // Logical AND between sets of search terms + .join('+AND+'); + + return `q=${queryParams}`; + } + + /** + * Get ITIS SOLR query by TSN param + * + * @param {number[]} searchTsnIds + * @return {*} {string} + * @memberof ItisService + */ + _getItisSolrTsnSearch(searchTsnIds: number[]): string { + return searchTsnIds.map((tsn) => `tsn:${tsn}`).join('+'); + } +} diff --git a/api/src/services/taxonomy-service.test.ts b/api/src/services/taxonomy-service.test.ts index 3ca06645e..bb6af9dfe 100644 --- a/api/src/services/taxonomy-service.test.ts +++ b/api/src/services/taxonomy-service.test.ts @@ -1,9 +1,11 @@ -import { AggregationsAggregate, SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import chai, { expect } from 'chai'; import { describe } from 'mocha'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; -import { ITaxonomySource, TaxonomyService } from './taxonomy-service'; +import { TaxonomyRepository } from '../repositories/taxonomy-repository'; +import { getMockDBConnection } from '../__mocks__/db'; +import { ItisService, ItisSolrSearchResponse } from './itis-service'; +import { TaxonomyService } from './taxonomy-service'; chai.use(sinonChai); @@ -12,212 +14,170 @@ describe('TaxonomyService', () => { sinon.restore(); }); - const mockElasticResponse: SearchResponse> | undefined = { - took: 0, - timed_out: false, - _shards: { - failed: 0, - successful: 1, - total: 1 - }, - hits: { - hits: [] + const getItisSolrSearchResponse: ItisSolrSearchResponse[] = [ + { + commonNames: ['$commonName'], + kingdom: 'kingdom', + name: 'name', + parentTSN: 'parentTSN', + scientificName: 'scientificName', + tsn: 'tsn', + updateDate: 'updateDate', + usage: 'usage' } - }; + ]; it('constructs', () => { - const taxonomyService = new TaxonomyService(); + const mockDBConnection = getMockDBConnection(); + + const taxonomyService = new TaxonomyService(mockDBConnection); expect(taxonomyService).to.be.instanceof(TaxonomyService); }); - describe('getTaxonomyFromIds', async () => { - afterEach(() => { - sinon.restore(); - }); + describe('getTaxonByTsnIds', () => { + it('if all records exist in db should return array of taxon records', async () => { + const getTaxonRecord = [ + { + taxon_id: 1, + itis_tsn: 1, + bc_taxon_code: 'bc_taxon_code', + itis_scientific_name: 'itis_scientific_name', + common_name: 'common_name', + itis_data: {}, + record_effective_date: 'record_effective_date', + record_end_date: 'record_end_date', + create_date: 'create_date', + create_user: 1, + update_date: 'update_date', + update_user: 1, + revision_count: 1 + } + ]; - it('should query elasticsearch and return []', async () => { - process.env.ELASTICSEARCH_TAXONOMY_INDEX = 'taxonomy_test_3.0.0'; + const mockDBConnection = getMockDBConnection(); - const taxonomyService = new TaxonomyService(); + const taxonomyService = new TaxonomyService(mockDBConnection); - const elasticSearchStub = sinon.stub(taxonomyService, 'elasticSearch').resolves(undefined); + const repo = sinon.stub(TaxonomyRepository.prototype, 'getTaxonByTsnIds').resolves(getTaxonRecord); - const response = await taxonomyService.getTaxonomyFromIds([1]); + const response = await taxonomyService.getTaxonByTsnIds([1]); - expect(elasticSearchStub).to.be.calledOnce; - expect(response).to.eql([]); + expect(repo).to.be.calledOnce; + expect(response).to.be.eql([{ tsn: 1, commonName: 'common_name', scientificName: 'itis_scientific_name' }]); }); - it('should query elasticsearch and return taxonomy', async () => { - process.env.ELASTICSEARCH_TAXONOMY_INDEX = 'taxonomy_test_3.0.0'; - - const taxonomyService = new TaxonomyService(); - - const taxonDetails: Omit = { - unit_name1: 'A', - unit_name2: 'B', - unit_name3: 'C', - taxon_authority: 'taxon_authority', - code: 'D', - tty_kingdom: 'kingdom', - tty_name: 'name', - english_name: 'animal', - note: null, - parent_id: 1, - parent_hierarchy: [] - }; - - const elasticSearchStub = sinon.stub(taxonomyService, 'elasticSearch').resolves({ - ...mockElasticResponse, - hits: { - hits: [ - { - _index: process.env.ELASTICSEARCH_TAXONOMY_INDEX, - _id: '1', - _source: { - ...taxonDetails, - end_date: null - } - } - ] - } - }); - - const response = await taxonomyService.getTaxonomyFromIds([1]); - - expect(elasticSearchStub).to.be.calledOnce; - - expect(response).to.eql([ + it('if some records do not exist in db should return array of taxon records', async () => { + const getTaxonRecord = [ + { + taxon_id: 1, + itis_tsn: 1, + bc_taxon_code: 'bc_taxon_code', + itis_scientific_name: 'itis_scientific_name', + common_name: 'common_name', + itis_data: {}, + record_effective_date: 'record_effective_date', + record_end_date: 'record_end_date', + create_date: 'create_date', + create_user: 1, + update_date: 'update_date', + update_user: 1, + revision_count: 1 + }, { - _index: process.env.ELASTICSEARCH_TAXONOMY_INDEX, - _id: '1', - _source: { - ...taxonDetails, - end_date: null - } + taxon_id: 2, + itis_tsn: 2, + bc_taxon_code: 'bc_taxon_code', + itis_scientific_name: 'itis_scientific_name', + common_name: 'common_name', + itis_data: {}, + record_effective_date: 'record_effective_date', + record_end_date: 'record_end_date', + create_date: 'create_date', + create_user: 1, + update_date: 'update_date', + update_user: 1, + revision_count: 1 } - ]); - }); - }); + ]; - describe('getSpeciesFromIds', async () => { - afterEach(() => { - sinon.restore(); - }); + const mockDBConnection = getMockDBConnection(); + + const taxonomyService = new TaxonomyService(mockDBConnection); - it('should query elasticsearch and return []', async () => { - process.env.ELASTICSEARCH_TAXONOMY_INDEX = 'taxonomy_test_3.0.0'; + const repo = sinon.stub(TaxonomyRepository.prototype, 'getTaxonByTsnIds').resolves([getTaxonRecord[0]]); - const taxonomyService = new TaxonomyService(); + const searchItisByTSNStub = sinon + .stub(ItisService.prototype, 'searchItisByTSN') + .resolves(getItisSolrSearchResponse); - const elasticSearchStub = sinon.stub(taxonomyService, 'elasticSearch').resolves(undefined); + const itisService = sinon.stub(TaxonomyService.prototype, 'addItisTaxonRecord').resolves(getTaxonRecord[1]); - const response = await taxonomyService.getSpeciesFromIds(['1']); + const response = await taxonomyService.getTaxonByTsnIds([1, 2]); - expect(elasticSearchStub).to.be.calledOnce; - expect(response).to.eql([]); + expect(repo).to.be.calledOnce; + expect(searchItisByTSNStub).to.be.calledOnce; + expect(itisService).to.be.calledOnce; + expect(response).to.be.eql([ + { tsn: 1, commonName: 'common_name', scientificName: 'itis_scientific_name' }, + { tsn: 2, commonName: 'common_name', scientificName: 'itis_scientific_name' } + ]); }); + }); - it('should query elasticsearch and return sanitized data', async () => { - process.env.ELASTICSEARCH_TAXONOMY_INDEX = 'taxonomy_test_3.0.0'; - - const taxonomyService = new TaxonomyService(); - - const taxonDetails: Omit = { - unit_name1: 'A', - unit_name2: 'B', - unit_name3: 'C', - taxon_authority: 'taxon_authority', - code: 'D', - tty_kingdom: 'kingdom', - tty_name: 'name', - english_name: 'animal', - note: null, - parent_id: 1, - parent_hierarchy: [] - }; - - const elasticSearchStub = sinon.stub(taxonomyService, 'elasticSearch').resolves({ - ...mockElasticResponse, - hits: { - hits: [ - { - _index: process.env.ELASTICSEARCH_TAXONOMY_INDEX, - _id: '1', - _source: { - ...taxonDetails, - end_date: null - } - } - ] - } + describe('addItisTaxonRecord', () => { + it('should add a new taxon record', async () => { + const mockDBConnection = getMockDBConnection(); + + const taxonomyService = new TaxonomyService(mockDBConnection); + + const addItisTaxonRecordStub = sinon.stub(TaxonomyRepository.prototype, 'addItisTaxonRecord').resolves({ + taxon_id: 1, + itis_tsn: 1, + bc_taxon_code: null, + itis_scientific_name: 'scientificName', + common_name: 'commonName', + itis_data: {}, + record_effective_date: 'updateDate', + record_end_date: null, + create_date: 'now', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 1 }); - const sanitizeSpeciesDataStub = sinon.spy(taxonomyService, '_sanitizeSpeciesData'); - - const response = await taxonomyService.getSpeciesFromIds(['1']); - - expect(elasticSearchStub).to.be.calledOnce; - expect(sanitizeSpeciesDataStub).to.be.calledOnce; - expect(response).to.eql([{ id: '1', code: 'D', label: 'animal, A B C' }]); + const response = await taxonomyService.addItisTaxonRecord(getItisSolrSearchResponse[0]); + + expect(addItisTaxonRecordStub).to.be.calledOnce; + expect(response).to.be.eql({ + taxon_id: 1, + itis_tsn: 1, + bc_taxon_code: null, + itis_scientific_name: 'scientificName', + common_name: 'commonName', + itis_data: {}, + record_effective_date: 'updateDate', + record_end_date: null, + create_date: 'now', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 1 + }); }); }); - describe('searchSpecies', async () => { - it('should query elasticsearch', async () => { - process.env.ELASTICSEARCH_TAXONOMY_INDEX = 'taxonomy_test_3.0.0'; - - const taxonomyService = new TaxonomyService(); - - const taxonDetails: Omit = { - unit_name1: 'A', - unit_name2: 'B', - unit_name3: 'C', - taxon_authority: 'taxon_authority', - code: 'D', - tty_kingdom: 'kingdom', - tty_name: 'name', - english_name: 'animal', - note: null, - parent_id: 1, - parent_hierarchy: [] - }; - - const elasticSearchStub = sinon.stub(taxonomyService, 'elasticSearch').resolves({ - ...mockElasticResponse, - hits: { - hits: [ - { - _index: process.env.ELASTICSEARCH_TAXONOMY_INDEX, - _id: '1', - _source: { - ...taxonDetails, - end_date: null - } - }, - { - _index: process.env.ELASTICSEARCH_TAXONOMY_INDEX, - _id: '2', - _source: { - ...taxonDetails, - end_date: '2010-01-01' - } - }, - { - _index: process.env.ELASTICSEARCH_TAXONOMY_INDEX, - _id: '3', - _source: { - ...taxonDetails, - end_date: '2040-01-01' - } - } - ] - } - }); + describe('deleteTaxonRecord', () => { + it('should delete a taxon record', async () => { + const mockDBConnection = getMockDBConnection(); + + const taxonomyService = new TaxonomyService(mockDBConnection); + + const deleteTaxonRecordStub = sinon.stub(TaxonomyRepository.prototype, 'deleteTaxonRecord').resolves(); - taxonomyService.searchSpecies('search term'); + await taxonomyService.deleteTaxonRecord(1); - expect(elasticSearchStub).to.be.calledOnce; + expect(deleteTaxonRecordStub).to.be.calledOnce; }); }); }); diff --git a/api/src/services/taxonomy-service.ts b/api/src/services/taxonomy-service.ts index 07d08ae51..3d49ad8ca 100644 --- a/api/src/services/taxonomy-service.ts +++ b/api/src/services/taxonomy-service.ts @@ -1,205 +1,101 @@ -import { - AggregationsAggregate, - QueryDslBoolQuery, - SearchHit, - SearchRequest, - SearchResponse -} from '@elastic/elasticsearch/lib/api/types'; -import { getLogger } from '../utils/logger'; -import { ElasticSearchIndices, ESService } from './es-service'; +import { IDBConnection } from '../database/db'; +import { TaxonomyRepository, TaxonRecord } from '../repositories/taxonomy-repository'; +import { ItisService, ItisSolrSearchResponse } from './itis-service'; -const defaultLog = getLogger('services/taxonomy-service'); - -export interface ITaxonomySource { - unit_name1: string; - unit_name2: string; - unit_name3: string; - taxon_authority: string; - code: string; - tty_kingdom: string; - tty_name: string; - english_name: string; - note: string | null; - end_date: string | null; - parent_id: number | null; - parent_hierarchy: { id: number; level: string }[]; -} +export type TaxonSearchResult = { + tsn: number; + commonName: string | null; + scientificName: string; +}; /** - * Service for retrieving and processing taxonomic data from Elasticsearch. + * Service for retrieving and processing taxonomic data from BioHub. * * @export * @class TaxonomyService - * @extends {ESService} */ -export class TaxonomyService extends ESService { - /** - * Performs a query in Elasticsearch based on the given search criteria - * - * @param {SearchRequest} searchRequest - * @return {*} {(Promise> | undefined>)} - * Promise resolving the search results from Elasticsearch - * @memberof TaxonomyService - */ - async elasticSearch( - searchRequest: SearchRequest - ): Promise> | undefined> { - try { - const esClient = await this.getEsClient(); +export class TaxonomyService { + taxonRepository: TaxonomyRepository; - return esClient.search({ - index: ElasticSearchIndices.TAXONOMY, - ...searchRequest - }); - } catch (error) { - defaultLog.debug({ label: 'elasticSearch', message: 'error', error }); - } + constructor(connection: IDBConnection) { + this.taxonRepository = new TaxonomyRepository(connection); } + /** - * Sanitizes species data retrieved from Elasticsearch. + * Get taxon records by TSN ids. * - * @param {SearchHit[]} data The data response from ElasticSearch - * @return {*} {({ id: string; code: string | undefined; label: string }[])} An ID, code, and label tuple for each - * taxonomic code + * @param {number[]} tsnIds + * @return {*} {Promise} * @memberof TaxonomyService */ - _sanitizeSpeciesData = ( - data: SearchHit[] - ): { id: string; code: string | undefined; label: string }[] => { - return data.map((item: SearchHit) => { - const { _id: id, _source } = item; - - const label = [ - [ - _source?.english_name, - [_source?.unit_name1, _source?.unit_name2, _source?.unit_name3].filter(Boolean).join(' ') - ] - .filter(Boolean) - .join(', ') - ] - .filter(Boolean) - .join(': '); + async getTaxonByTsnIds(tsnIds: number[]): Promise { + // Search for taxon records in the database + const existingTaxonRecords = await this.taxonRepository.getTaxonByTsnIds(tsnIds); + let patchedTaxonRecords: TaxonRecord[] = []; - return { id, code: _source?.code, label: label }; - }); - }; + const missingTsnIds = tsnIds.filter((tsnId) => !existingTaxonRecords.find((item) => item.itis_tsn === tsnId)); - /** - * Searches the taxonomy Elasticsearch index by taxonomic code IDs - * - * @param {string[] | number[]} ids The array of taxonomic code IDs - * @return {Promise[]>} The response from Elasticsearch - * @memberof TaxonomyService - */ - async getTaxonomyFromIds(ids: string[] | number[]): Promise[]> { - const response = await this.elasticSearch({ - query: { - terms: { - _id: ids - } - } - }); + if (missingTsnIds.length) { + // If the local database does not contain a record for all of the requested ids, search ITIS for the missing + // taxon records, patching the missing records in the local database in the process + const itisService = new ItisService(); + const itisResponse = await itisService.searchItisByTSN(missingTsnIds); - if (!response) { - return []; + patchedTaxonRecords = await Promise.all(itisResponse.map(async (item) => this.addItisTaxonRecord(item))); } - return response.hits.hits; + // Missing ids patched, return taxon records for all requested ids + return this._sanitizeTaxonRecordsData(existingTaxonRecords.concat(patchedTaxonRecords)); + } + + _sanitizeTaxonRecordsData(taxonRecords: TaxonRecord[]): TaxonSearchResult[] { + return taxonRecords.map((item: TaxonRecord) => { + return { + tsn: item.itis_tsn, + commonName: item.common_name, + scientificName: item.itis_scientific_name + }; + }); } /** - * Searches the taxonomy Elasticsearch index by taxonomic code IDs and santizes the response + * Adds a new taxon record. * - * @param {string[] | number[]} ids The array of taxonomic code IDs - * @returns {Promise<{ id: string, label: string}[]>} Promise resolving an ID and label pair for each taxonomic code + * @param {ItisSolrSearchResponse} itisSolrResponse + * @return {*} {Promise} * @memberof TaxonomyService */ - async getSpeciesFromIds(ids: string[] | number[]): Promise<{ id: string; label: string }[]> { - const response = await this.elasticSearch({ - query: { - terms: { - _id: ids - } - } - }); + async addItisTaxonRecord(itisSolrResponse: ItisSolrSearchResponse): Promise { + let commonName = null; + if (itisSolrResponse.commonNames) { + commonName = itisSolrResponse.commonNames[0].split('$')[1]; + /* Sample itisResponse: + * commonNames: [ + * '$withered wooly milk-vetch$English$N$152846$2012-12-21 00:00:00$', + * '$woolly locoweed$English$N$124501$2011-06-29 00:00:00$', + * '$Davis Mountains locoweed$English$N$124502$2011-06-29 00:00:00$', + * '$woolly milkvetch$English$N$72035$2012-12-21 00:00:00$' + * ] + */ + } - return response ? this._sanitizeSpeciesData(response.hits.hits) : []; + return this.taxonRepository.addItisTaxonRecord( + Number(itisSolrResponse.tsn), + itisSolrResponse.scientificName, + commonName, + itisSolrResponse, + itisSolrResponse.updateDate + ); } /** - * Maps a taxonomic search term to an Elasticsearch query, then performs the query and sanitizes the response. - * The query also includes a boolean match to only include records whose `end_date` field is either - * undefined/null or is a date that hasn't occurred yet. This filtering is not done on similar ES queries, - * since we must still be able to search by a given taxonomic code ID, even if is one that is expired. + * Delete an existing taxon record. * - * @param {string} term The search term string - * @return {*} {(Promise<{ id: string; code: string | undefined; label: string }[]>)} Promise resolving an ID, code, - * and label tuple for each taxonomic code + * @param {number} taxonId + * @return {*} {Promise} * @memberof TaxonomyService */ - async searchSpecies(term: string): Promise<{ id: string; code: string | undefined; label: string }[]> { - const searchConfig: object[] = []; - - const splitTerms = term.split(' '); - - splitTerms.forEach((item) => { - searchConfig.push({ - wildcard: { - english_name: { value: `*${item}*`, boost: 4.0, case_insensitive: true } - } - }); - searchConfig.push({ - wildcard: { unit_name1: { value: `*${item}*`, boost: 3.0, case_insensitive: true } } - }); - searchConfig.push({ - wildcard: { unit_name2: { value: `*${item}*`, boost: 3.0, case_insensitive: true } } - }); - searchConfig.push({ - wildcard: { unit_name3: { value: `*${item}*`, boost: 3.0, case_insensitive: true } } - }); - searchConfig.push({ wildcard: { code: { value: `*${item}*`, boost: 2, case_insensitive: true } } }); - searchConfig.push({ - wildcard: { tty_kingdom: { value: `*${item}*`, boost: 1.0, case_insensitive: true } } - }); - }); - - const response = await this.elasticSearch({ - query: { - bool: { - must: [ - { - bool: { - should: searchConfig - } - }, - { - bool: { - minimum_should_match: 1, - should: [ - { - bool: { - must_not: { - exists: { - field: 'end_date' - } - } - } - }, - { - range: { - end_date: { - gt: 'now' - } - } - } - ] - } - } - ] - } as QueryDslBoolQuery - } - }); - - return response ? this._sanitizeSpeciesData(response.hits.hits) : []; + async deleteTaxonRecord(taxonId: number): Promise { + return this.taxonRepository.deleteTaxonRecord(taxonId); } } diff --git a/database/src/migrations/20240119000000_taxonomy_tables.ts b/database/src/migrations/20240119000000_taxonomy_tables.ts index 78a3fcea2..b4c1997a0 100644 --- a/database/src/migrations/20240119000000_taxonomy_tables.ts +++ b/database/src/migrations/20240119000000_taxonomy_tables.ts @@ -139,9 +139,10 @@ export async function up(knex: Knex): Promise { CREATE INDEX taxon_alias_fk3 ON taxon_alias(taxon_alias_origin_id); -- Add unique end-date key constraints (taxon) - CREATE UNIQUE INDEX taxon_nuk1 ON taxon(itis_scientific_name, (record_end_date is NULL)) where record_end_date is null; - CREATE UNIQUE INDEX taxon_nuk2 ON taxon(bc_taxon_code, (record_end_date is NULL)) where record_end_date is null; CREATE UNIQUE INDEX taxon_nuk3 ON taxon(itis_tsn, (record_end_date is NULL)) where record_end_date is null; + CREATE UNIQUE INDEX taxon_nuk1 ON taxon(itis_scientific_name, (record_end_date is NULL)) where record_end_date is null; + -- 'bc_taxon_code' can be null, therefore the record_end_date constraint only applies if 'bc_taxon_code' is itself not null + CREATE UNIQUE INDEX taxon_nuk2 ON taxon(bc_taxon_code, (record_end_date is NULL)) where record_end_date is null and bc_taxon_code is not null; -- Add unique end-date key constraints (taxon_alias) CREATE UNIQUE INDEX taxon_alias_nuk1 ON taxon_alias(taxon_id, alias, language_id, taxon_alias_origin_id, (record_end_date is NULL)) where record_end_date is null; diff --git a/docker-compose.yml b/docker-compose.yml index 62401e4ea..c67a7ed01 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,8 @@ services: - ELASTICSEARCH_URL=${ELASTICSEARCH_URL} - ELASTICSEARCH_EML_INDEX=${ELASTICSEARCH_EML_INDEX} - ELASTICSEARCH_TAXONOMY_INDEX=${ELASTICSEARCH_TAXONOMY_INDEX} + # ITIS API + - ITIS_SOLR_URL=${ITIS_SOLR_URL} - S3_KEY_PREFIX=${S3_KEY_PREFIX} - TZ=${API_TZ} - API_HOST=${API_HOST} diff --git a/env_config/env.docker b/env_config/env.docker index c15b277a4..4975fe987 100644 --- a/env_config/env.docker +++ b/env_config/env.docker @@ -150,6 +150,11 @@ ELASTICSEARCH_URL=https://elasticsearch-a0ec71-dev.apps.silver.devops.gov.bc.ca ELASTICSEARCH_EML_INDEX=eml ELASTICSEARCH_TAXONOMY_INDEX=taxonomy_3.0.0 +# ------------------------------------------------------------------------------ +# ITIS Platform API +# ------------------------------------------------------------------------------ +ITIS_SOLR_URL=https://services.itis.gov + # ------------------------------------------------------------------------------ # GeoServer - https://geoserver.org/ # ------------------------------------------------------------------------------