diff --git a/.github/workflows/ci-cd-pims-dev.yml b/.github/workflows/ci-cd-pims-dev.yml index d72ee8e6e2..fc879705e4 100644 --- a/.github/workflows/ci-cd-pims-dev.yml +++ b/.github/workflows/ci-cd-pims-dev.yml @@ -123,7 +123,6 @@ jobs: - name: Deploy mayan shell: bash run: | - ./openshift/4.0/player.sh deploy scheduler $DESTINATION -apply oc tag mayan-bcgov:latest-$DESTINATION mayan-bcgov:$DESTINATION # the command: diff --git a/source/backend/Pims.Scheduler.Test/Pims.Scheduler.Test.csproj b/source/backend/Pims.Scheduler.Test/Pims.Scheduler.Test.csproj new file mode 100644 index 0000000000..2ec4df52a3 --- /dev/null +++ b/source/backend/Pims.Scheduler.Test/Pims.Scheduler.Test.csproj @@ -0,0 +1,31 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/source/backend/Pims.Scheduler.Test/Repositories/PimsDocumentQueueRepositoryTests.cs b/source/backend/Pims.Scheduler.Test/Repositories/PimsDocumentQueueRepositoryTests.cs new file mode 100644 index 0000000000..e794406fcf --- /dev/null +++ b/source/backend/Pims.Scheduler.Test/Repositories/PimsDocumentQueueRepositoryTests.cs @@ -0,0 +1,87 @@ +using FluentAssertions; +using Moq; +using Pims.Api.Models.CodeTypes; +using Pims.Api.Models.Concepts.Document; +using Pims.Api.Models.Requests.Http; +using Pims.Dal.Entities.Models; +using Pims.Scheduler.Repositories; +using Xunit; + +namespace Pims.Scheduler.Test.Repositories +{ + public class PimsDocumentQueueRepositoryTest + { + [Fact] + public async Task PollQueuedDocument_ValidDocument_ReturnsExternalResponse() + { + // Arrange + var document = new DocumentQueueModel { Id = 1 }; + var expectedResponse = new ExternalResponse { Status = ExternalResponseStatus.Success }; + var repositoryMock = new Mock(); + repositoryMock.Setup(x => x.PollQueuedDocument(document)).ReturnsAsync(expectedResponse); + + // Act + var result = await repositoryMock.Object.PollQueuedDocument(document); + + // Assert + result.Should().NotBeNull(); + result.Status.Should().Be(ExternalResponseStatus.Success); + repositoryMock.Verify(x => x.PollQueuedDocument(document), Times.Once); + } + + [Fact] + public async Task UploadQueuedDocument_ValidDocument_ReturnsExternalResponse() + { + // Arrange + var document = new DocumentQueueModel { Id = 1 }; + var expectedResponse = new ExternalResponse { Status = ExternalResponseStatus.Success }; + var repositoryMock = new Mock(); + repositoryMock.Setup(x => x.UploadQueuedDocument(document)).ReturnsAsync(expectedResponse); + + // Act + var result = await repositoryMock.Object.UploadQueuedDocument(document); + + // Assert + result.Should().NotBeNull(); + result.Status.Should().Be(ExternalResponseStatus.Success); + repositoryMock.Verify(x => x.UploadQueuedDocument(document), Times.Once); + } + + [Fact] + public async Task UpdateQueuedDocument_ValidDocument_ReturnsExternalResponse() + { + // Arrange + var documentQueueId = 1; + var document = new DocumentQueueModel { Id = documentQueueId }; + var expectedResponse = new ExternalResponse { Status = ExternalResponseStatus.Success }; + var repositoryMock = new Mock(); + repositoryMock.Setup(x => x.UpdateQueuedDocument(documentQueueId, document)).ReturnsAsync(expectedResponse); + + // Act + var result = await repositoryMock.Object.UpdateQueuedDocument(documentQueueId, document); + + // Assert + result.Should().NotBeNull(); + result.Status.Should().Be(ExternalResponseStatus.Success); + repositoryMock.Verify(x => x.UpdateQueuedDocument(documentQueueId, document), Times.Once); + } + + [Fact] + public async Task SearchQueuedDocumentsAsync_ValidFilter_ReturnsExternalResponse() + { + // Arrange + var filter = new DocumentQueueFilter(); + var expectedResponse = new ExternalResponse> { Status = ExternalResponseStatus.Success }; + var repositoryMock = new Mock(); + repositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(filter)).ReturnsAsync(expectedResponse); + + // Act + var result = await repositoryMock.Object.SearchQueuedDocumentsAsync(filter); + + // Assert + result.Should().NotBeNull(); + result.Status.Should().Be(ExternalResponseStatus.Success); + repositoryMock.Verify(x => x.SearchQueuedDocumentsAsync(filter), Times.Once); + } + } +} diff --git a/source/backend/Pims.Scheduler.Test/Services/DocumentQueueServiceTests.cs b/source/backend/Pims.Scheduler.Test/Services/DocumentQueueServiceTests.cs new file mode 100644 index 0000000000..07d686877b --- /dev/null +++ b/source/backend/Pims.Scheduler.Test/Services/DocumentQueueServiceTests.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using FluentAssertions; +using FluentAssertions.Common; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Pims.Api.Models.Base; +using Pims.Api.Models.CodeTypes; +using Pims.Api.Models.Concepts.Document; +using Pims.Api.Models.Requests.Http; +using Pims.Dal.Entities.Models; +using Pims.Scheduler.Http.Configuration; +using Pims.Scheduler.Models; +using Pims.Scheduler.Repositories; +using Pims.Scheduler.Services; +using Xunit; + +namespace Pims.Scheduler.Test.Services +{ + public class DocumentQueueServiceTests + { + private readonly Mock> _loggerMock; + private readonly Mock _documentQueueRepositoryMock; + private readonly Mock> _uploadOptionsMock; + private readonly Mock> _queryOptionsMock; + private readonly DocumentQueueService _service; + + public DocumentQueueServiceTests() + { + _loggerMock = new Mock>(); + _documentQueueRepositoryMock = new Mock(); + _uploadOptionsMock = new Mock>(); + _queryOptionsMock = new Mock>(); + _uploadOptionsMock.Setup(x => x.CurrentValue).Returns(new UploadQueuedDocumentsJobOptions() { BatchSize = 10, FileSize = 100 }); + _queryOptionsMock.Setup(x => x.CurrentValue).Returns(new QueryProcessingDocumentsJobOptions() { BatchSize = 10, MaxProcessingMinutes = 100 }); + + _service = new DocumentQueueService( + _loggerMock.Object, + _uploadOptionsMock.Object, + _queryOptionsMock.Object, + _documentQueueRepositoryMock.Object + ); + } + + [Fact] + public async Task UploadQueuedDocuments_NoDocumentsToProcess_ReturnsSkipped() + { + // Arrange + var searchResponse = new ExternalResponse> { Status = ExternalResponseStatus.Success, Payload = new List() }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + + // Act + var result = await _service.UploadQueuedDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.SKIPPED); + result.Message.Should().Be("No documents to process, skipping execution."); + } + + [Fact] + public async Task UploadQueuedDocuments_ErrorStatus_ReturnsError() + { + // Arrange + var searchResponse = new ExternalResponse> { Status = ExternalResponseStatus.Error, Message = "Error", Payload = new List() { new DocumentQueueModel() } }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + + // Act + var result = await _service.UploadQueuedDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.ERROR); + result.Message.Should().Be("Received error status from pims document queue service, aborting."); + } + + [Fact] + public async Task UploadQueuedDocuments_SingleDocumentError_ReturnsError() + { + // Arrange + var document = new DocumentQueueModel { Id = 1, DocumentQueueStatusType = new CodeTypeModel() { Id = DocumentQueueStatusTypes.PROCESSING.ToString() } }; + var searchResponse = new ExternalResponse> + { + Status = ExternalResponseStatus.Success, + Payload = new List { document }, + }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + _documentQueueRepositoryMock.Setup(x => x.UploadQueuedDocument(document)).ReturnsAsync(new ExternalResponse + { + Status = ExternalResponseStatus.Error, + Message = "Error uploading document.", + }); + + // Act + var result = await _service.UploadQueuedDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.ERROR); + result.DocumentQueueResponses.FirstOrDefault()?.Message.Should().Be("Received error response from UploadQueuedDocument for queued document 1 status Error message: Error uploading document."); + } + + [Fact] + public async Task UploadQueuedDocuments_SingleDocumentSuccess_ReturnsSuccess() + { + // Arrange + var document = new DocumentQueueModel { Id = 1, DocumentQueueStatusType = new CodeTypeModel() { Id = DocumentQueueStatusTypes.PROCESSING.ToString() } }; + var searchResponse = new ExternalResponse> + { + Status = ExternalResponseStatus.Success, + Payload = new List { document }, + }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + _documentQueueRepositoryMock.Setup(x => x.UploadQueuedDocument(document)).ReturnsAsync(new ExternalResponse + { + Status = ExternalResponseStatus.Success, + Payload = document, + }); + + // Act + var result = await _service.UploadQueuedDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.SUCCESS); + } + + [Fact] + public async Task UploadQueuedDocuments_TwoDocumentsMixedResults_ReturnsPartialSuccess() + { + // Arrange + var document1 = new DocumentQueueModel { Id = 1, DocumentQueueStatusType = new CodeTypeModel() { Id = DocumentQueueStatusTypes.PROCESSING.ToString() } }; + var document2 = new DocumentQueueModel { Id = 2, DocumentQueueStatusType = new CodeTypeModel() { Id = DocumentQueueStatusTypes.PROCESSING.ToString() } }; + var searchResponse = new ExternalResponse> + { + Status = ExternalResponseStatus.Success, + Payload = new List { document1, document2 }, + }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + _documentQueueRepositoryMock.Setup(x => x.UploadQueuedDocument(document1)).ReturnsAsync(new ExternalResponse + { + Status = ExternalResponseStatus.Success, + Payload = document1, + }); + _documentQueueRepositoryMock.Setup(x => x.UploadQueuedDocument(document2)).ReturnsAsync(new ExternalResponse + { + Status = ExternalResponseStatus.Error, + Message = "Error uploading document 2.", + }); + + // Act + var result = await _service.UploadQueuedDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.PARTIAL); + result.DocumentQueueResponses.Should().HaveCount(2); + result.DocumentQueueResponses.ToArray()[1].Message.Should().Be("Received error response from UploadQueuedDocument for queued document 2 status Error message: Error uploading document 2."); + } + + [Fact] + public async Task RetryQueuedDocuments_ErrorStatus_ReturnsError() + { + // Arrange + var searchResponse = new ExternalResponse> { Status = ExternalResponseStatus.Error, Message = "Error", Payload = new List() { new DocumentQueueModel() } }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + + // Act + var result = await _service.RetryQueuedDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.ERROR); + result.Message.Should().Be("Received error status from pims document queue service, aborting."); + } + + [Fact] + public async Task QueryProcessingDocuments_NoDocumentsToProcess_ReturnsSkipped() + { + // Arrange + var searchResponse = new ExternalResponse> { Status = ExternalResponseStatus.Success, Payload = new List() }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + + // Act + var result = await _service.QueryProcessingDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.SKIPPED); + result.Message.Should().Be("No documents to process, skipping execution."); + } + + [Fact] + public async Task QueryProcessingDocuments_ErrorStatus_ReturnsError() + { + // Arrange + var searchResponse = new ExternalResponse> { Status = ExternalResponseStatus.Error, Message = "Error", Payload = new List() { new DocumentQueueModel() } }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + + // Act + var result = await _service.QueryProcessingDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.ERROR); + result.Message.Should().Be("Received error status from pims document queue service, aborting."); + } + + [Fact] + public async Task QueryProcessingDocuments_OneDocumentError_ReturnsError() + { + // Arrange + var document = new DocumentQueueModel { Id = 1, DocumentQueueStatusType = new CodeTypeModel() { Id = DocumentQueueStatusTypes.PROCESSING.ToString() } }; + var searchResponse = new ExternalResponse> + { + Status = ExternalResponseStatus.Success, + Payload = new List { document }, + }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + _documentQueueRepositoryMock.Setup(x => x.PollQueuedDocument(document)).ReturnsAsync(new ExternalResponse + { + Status = ExternalResponseStatus.Error, + Message = "Error processing document.", + }); + + // Act + var result = await _service.QueryProcessingDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.ERROR); + result.DocumentQueueResponses.FirstOrDefault()?.Message.Should().Be("Received error response from PollQueuedDocument for queued document 1 status Error message: Error processing document."); + } + + [Fact] + public async Task QueryProcessingDocuments_OneDocumentSuccess_ReturnsSuccess() + { + // Arrange + var document = new DocumentQueueModel { Id = 1, DocumentQueueStatusType = new CodeTypeModel() { Id = DocumentQueueStatusTypes.PROCESSING.ToString() } }; + var searchResponse = new ExternalResponse> + { + Status = ExternalResponseStatus.Success, + Payload = new List { document }, + }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + _documentQueueRepositoryMock.Setup(x => x.PollQueuedDocument(document)).ReturnsAsync(new ExternalResponse + { + Status = ExternalResponseStatus.Success, + Payload = new DocumentQueueModel { Id = document.Id, DocumentQueueStatusType = new CodeTypeModel() { Id = DocumentQueueStatusTypes.SUCCESS.ToString() } }, + }); + + // Act + var result = await _service.QueryProcessingDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.SUCCESS); + } + + [Fact] + public async Task QueryProcessingDocuments_OneDocumentExceededMaxProcessingTime_ReturnsError() + { + // Arrange + var document = new DocumentQueueModel { Id = 1, DocumentQueueStatusType = new CodeTypeModel() { Id = DocumentQueueStatusTypes.PROCESSING.ToString() }, DocumentProcessStartTimestamp = DateTime.UtcNow.AddDays(-2) }; + var searchResponse = new ExternalResponse> + { + Status = ExternalResponseStatus.Success, + Payload = new List { document }, + }; + _documentQueueRepositoryMock.Setup(x => x.SearchQueuedDocumentsAsync(It.IsAny())).ReturnsAsync(searchResponse); + + // Act + var result = await _service.QueryProcessingDocuments(); + + // Assert + result.Status.Should().Be(TaskResponseStatusTypes.ERROR); + result.DocumentQueueResponses.FirstOrDefault()?.Message.Should().Be("Document processing for document 1 has exceeded maximum processing time of 100"); + } + + + } +} diff --git a/source/backend/api/Areas/Documents/DocumentQueueController.cs b/source/backend/api/Areas/Documents/DocumentQueueController.cs index 9fead6a65e..f166e0bc10 100644 --- a/source/backend/api/Areas/Documents/DocumentQueueController.cs +++ b/source/backend/api/Areas/Documents/DocumentQueueController.cs @@ -1,11 +1,19 @@ +using System; using System.Collections.Generic; +using System.Threading.Tasks; using MapsterMapper; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Pims.Api.Areas.Acquisition.Controllers; +using Pims.Api.Models.Concepts.Document; using Pims.Api.Services; +using Pims.Core.Api.Exceptions; using Pims.Core.Api.Policies; +using Pims.Core.Extensions; using Pims.Core.Json; using Pims.Core.Security; +using Pims.Dal.Entities; using Pims.Dal.Entities.Models; using Swashbuckle.AspNetCore.Annotations; @@ -18,12 +26,13 @@ namespace Pims.Api.Controllers [ApiController] [ApiVersion("1.0")] [Route("v{version:apiVersion}/documents/queue")] - [Route("/documents")] + [Route("/documents/queue")] public class DocumentQueueController : ControllerBase { #region Variables private readonly IDocumentQueueService _documentQueueService; private readonly IMapper _mapper; + private readonly ILogger _logger; #endregion #region Constructors @@ -33,29 +42,128 @@ public class DocumentQueueController : ControllerBase /// /// /// - public DocumentQueueController(IDocumentQueueService documentQueueService, IMapper mapper) + /// + public DocumentQueueController(IDocumentQueueService documentQueueService, IMapper mapper, ILogger logger) { _documentQueueService = documentQueueService; _mapper = mapper; + _logger = logger; } #endregion #region Endpoints + /// + /// Update a Queued Document. + /// + /// + [HttpPut("{documentQueueId:long}")] + [HasPermission(Permissions.SystemAdmin)] + [Produces("application/json")] + [ProducesResponseType(typeof(List), 200)] + [SwaggerOperation(Tags = new[] { "document-types" })] + [TypeFilter(typeof(NullJsonResultFilter))] + public IActionResult Update(long documentQueueId, [FromBody] DocumentQueueModel documentQueue) + { + _logger.LogInformation( + "Request received by Controller: {Controller}, Action: {ControllerAction}, User: {User}, DateTime: {DateTime}", + nameof(DocumentQueueController), + nameof(Update), + User.GetUsername(), + DateTime.Now); + + documentQueue.ThrowIfNull(nameof(documentQueue)); + if (documentQueueId != documentQueue.Id) + { + throw new BadRequestException("Invalid document queue id."); + } + + var queuedDocuments = _documentQueueService.Update(_mapper.Map(documentQueue)); + var updatedDocumentQueue = _mapper.Map(queuedDocuments); + return new JsonResult(updatedDocumentQueue); + } + + /// + /// Poll a queud document to check on the upload status. + /// + /// + [HttpPost("{documentQueueId:long}/poll")] + [HasPermission(Permissions.SystemAdmin)] + [Produces("application/json")] + [ProducesResponseType(typeof(List), 200)] + [SwaggerOperation(Tags = new[] { "document-types" })] + [TypeFilter(typeof(NullJsonResultFilter))] + public async Task Poll(long documentQueueId, [FromBody] DocumentQueueModel documentQueue) + { + _logger.LogInformation( + "Request received by Controller: {Controller}, Action: {ControllerAction}, User: {User}, DateTime: {DateTime}", + nameof(DocumentQueueController), + nameof(Poll), + User.GetUsername(), + DateTime.Now); + + documentQueue.ThrowIfNull(nameof(documentQueue)); + if (documentQueueId != documentQueue.Id) + { + throw new BadRequestException("Invalid document queue id."); + } + + var queuedDocuments = await _documentQueueService.PollForDocument(_mapper.Map(documentQueue)); + var updatedDocumentQueue = _mapper.Map(queuedDocuments); + return new JsonResult(updatedDocumentQueue); + } + + /// + /// Upload a Queued Document. + /// + /// + [HttpPost("{documentQueueId:long}/upload")] + [HasPermission(Permissions.SystemAdmin)] + [Produces("application/json")] + [ProducesResponseType(typeof(List), 200)] + [SwaggerOperation(Tags = new[] { "document-types" })] + [TypeFilter(typeof(NullJsonResultFilter))] + public async Task Upload(long documentQueueId, [FromBody] DocumentQueueModel documentQueue) + { + _logger.LogInformation( + "Request received by Controller: {Controller}, Action: {ControllerAction}, User: {User}, DateTime: {DateTime}", + nameof(DocumentQueueController), + nameof(Upload), + User.GetUsername(), + DateTime.Now); + + documentQueue.ThrowIfNull(nameof(documentQueue)); + if (documentQueueId != documentQueue.Id) + { + throw new BadRequestException("Invalid document queue id."); + } + + var queuedDocuments = await _documentQueueService.Upload(_mapper.Map(documentQueue)); + var updatedDocumentQueue = _mapper.Map(queuedDocuments); + return new JsonResult(updatedDocumentQueue); + } + /// /// Search for Document Queue items via filter. /// /// - [HttpGet("search")] + [HttpPost("search")] [HasPermission(Permissions.SystemAdmin)] [Produces("application/json")] - [ProducesResponseType(typeof(List), 200)] + [ProducesResponseType(typeof(List), 200)] [SwaggerOperation(Tags = new[] { "document-types" })] [TypeFilter(typeof(NullJsonResultFilter))] - public IActionResult GetDocumentTypes([FromBody] DocumentQueueFilter filter) + public IActionResult SearchQueuedDocuments([FromBody] DocumentQueueFilter filter) { + _logger.LogInformation( + "Request received by Controller: {Controller}, Action: {ControllerAction}, User: {User}, DateTime: {DateTime}", + nameof(DocumentQueueController), + nameof(SearchQueuedDocuments), + User.GetUsername(), + DateTime.Now); + var queuedDocuments = _documentQueueService.SearchDocumentQueue(filter); - var documentQueueModels = _mapper.Map>(queuedDocuments); + var documentQueueModels = _mapper.Map>(queuedDocuments); return new JsonResult(documentQueueModels); } diff --git a/source/backend/api/Areas/Documents/DocumentRelationshipController.cs b/source/backend/api/Areas/Documents/DocumentRelationshipController.cs index 5202ab1026..4b09228ebf 100644 --- a/source/backend/api/Areas/Documents/DocumentRelationshipController.cs +++ b/source/backend/api/Areas/Documents/DocumentRelationshipController.cs @@ -4,15 +4,15 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Pims.Api.Constants; -using Pims.Core.Api.Exceptions; using Pims.Api.Models.CodeTypes; using Pims.Api.Models.Concepts.Document; using Pims.Api.Models.Requests.Document.Upload; -using Pims.Core.Api.Policies; using Pims.Api.Services; +using Pims.Core.Api.Exceptions; +using Pims.Core.Api.Policies; using Pims.Core.Json; -using Pims.Dal.Entities; using Pims.Core.Security; +using Pims.Dal.Entities; using Swashbuckle.AspNetCore.Annotations; namespace Pims.Api.Controllers diff --git a/source/backend/api/Pims.sln b/source/backend/api/Pims.sln index 0a83937ba1..051693cbc0 100644 --- a/source/backend/api/Pims.sln +++ b/source/backend/api/Pims.sln @@ -44,15 +44,15 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{A0343C94-4 docs\VERSIONING.md = docs\VERSIONING.md EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "..\tests", "{F256F2A5-0DBF-4137-A7D6-21F08111BD4A}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{F256F2A5-0DBF-4137-A7D6-21F08111BD4A}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "unit", "..\unit", "{3D70B211-74A8-484C-9B86-B0A2835C71E7}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "unit", "unit", "{3D70B211-74A8-484C-9B86-B0A2835C71E7}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Pims.Api.Test", "..\tests\unit\api\Pims.Api.Test.csproj", "{1F4E301C-F03B-4A31-A6F2-6A77384A74DA}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Pims.Dal.Test", "..\tests\unit\dal\Pims.Dal.Test.csproj", "{412BF533-2759-4FBE-B4C6-B89DB44FB6B5}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "core", "..\core", "{04780892-FC30-4B6B-A10C-5795C657E574}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "core", "core", "{04780892-FC30-4B6B-A10C-5795C657E574}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Pims.Core.Test", "..\tests\core\Pims.Core.Test.csproj", "{5A83C636-741A-4795-8588-70F033E79B5A}" EndProject @@ -66,6 +66,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Pims.Av", "..\clamav\Pims.A EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Pims.Core.Api", "..\core.api\Pims.Core.Api.csproj", "{89A99CC5-ADFB-4FC2-9136-7B0029EEA2D8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Pims.Scheduler.Test", "..\Pims.Scheduler.Test\Pims.Scheduler.Test.csproj", "{6B20887E-B784-4D78-939B-BDD8206DBE17}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Pims.Scheduler", "..\scheduler\Pims.Scheduler.csproj", "{AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -256,6 +260,30 @@ Global {89A99CC5-ADFB-4FC2-9136-7B0029EEA2D8}.Release|x64.Build.0 = Release|Any CPU {89A99CC5-ADFB-4FC2-9136-7B0029EEA2D8}.Release|x86.ActiveCfg = Release|Any CPU {89A99CC5-ADFB-4FC2-9136-7B0029EEA2D8}.Release|x86.Build.0 = Release|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Debug|x64.ActiveCfg = Debug|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Debug|x64.Build.0 = Debug|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Debug|x86.ActiveCfg = Debug|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Debug|x86.Build.0 = Debug|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Release|Any CPU.Build.0 = Release|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Release|x64.ActiveCfg = Release|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Release|x64.Build.0 = Release|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Release|x86.ActiveCfg = Release|Any CPU + {6B20887E-B784-4D78-939B-BDD8206DBE17}.Release|x86.Build.0 = Release|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Debug|x64.ActiveCfg = Debug|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Debug|x64.Build.0 = Debug|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Debug|x86.ActiveCfg = Debug|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Debug|x86.Build.0 = Debug|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Release|Any CPU.Build.0 = Release|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Release|x64.ActiveCfg = Release|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Release|x64.Build.0 = Release|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Release|x86.ActiveCfg = Release|Any CPU + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -276,6 +304,8 @@ Global {2C31E92C-9C95-45FF-9F95-928C2962F37D} = {3D70B211-74A8-484C-9B86-B0A2835C71E7} {16C06BDA-112F-4D04-82FF-0BBE45072372} = {5237F8A4-67F5-4751-B8B2-B93A06791480} {89A99CC5-ADFB-4FC2-9136-7B0029EEA2D8} = {5237F8A4-67F5-4751-B8B2-B93A06791480} + {6B20887E-B784-4D78-939B-BDD8206DBE17} = {3D70B211-74A8-484C-9B86-B0A2835C71E7} + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F} = {5237F8A4-67F5-4751-B8B2-B93A06791480} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3433C5DD-DC49-4A96-A1AE-90C1A1EBA87C} diff --git a/source/backend/api/Repositories/Cdogs/CdogsAuthRepository.cs b/source/backend/api/Repositories/Cdogs/CdogsAuthRepository.cs index f86a032277..d2bf249d70 100644 --- a/source/backend/api/Repositories/Cdogs/CdogsAuthRepository.cs +++ b/source/backend/api/Repositories/Cdogs/CdogsAuthRepository.cs @@ -1,13 +1,15 @@ using System; using System.Collections.Generic; using System.Net.Http; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Pims.Core.Api.Exceptions; +using Microsoft.Extensions.Options; using Pims.Api.Models.Cdogs; using Pims.Api.Models.CodeTypes; using Pims.Api.Models.Requests.Http; +using Pims.Core.Api.Exceptions; namespace Pims.Api.Repositories.Cdogs { @@ -25,11 +27,13 @@ public class CdogsAuthRepository : CdogsBaseRepository, IDocumentGenerationAuthR /// Injected Logger Provider. /// Injected Httpclient factory. /// The injected configuration provider. + /// The jsonOptions. public CdogsAuthRepository( ILogger logger, IHttpClientFactory httpClientFactory, - IConfiguration configuration) - : base(logger, httpClientFactory, configuration) + IConfiguration configuration, + IOptions jsonOptions) + : base(logger, httpClientFactory, configuration, jsonOptions) { _currentToken = null; _lastSucessfullRequest = DateTime.UnixEpoch; diff --git a/source/backend/api/Repositories/Cdogs/CdogsBaseRepository.cs b/source/backend/api/Repositories/Cdogs/CdogsBaseRepository.cs index d1d192b1d5..185199adc0 100644 --- a/source/backend/api/Repositories/Cdogs/CdogsBaseRepository.cs +++ b/source/backend/api/Repositories/Cdogs/CdogsBaseRepository.cs @@ -1,7 +1,9 @@ using System.Net.Http; using System.Net.Http.Headers; +using System.Text.Json; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Pims.Api.Models.Config; using Pims.Core.Api.Repositories.Rest; @@ -21,11 +23,13 @@ public abstract class CdogsBaseRepository : BaseRestRepository /// Injected Logger Provider. /// Injected Httpclient factory. /// The injected configuration provider. + /// The json options. protected CdogsBaseRepository( ILogger logger, IHttpClientFactory httpClientFactory, - IConfiguration configuration) - : base(logger, httpClientFactory) + IConfiguration configuration, + IOptions jsonOptions) + : base(logger, httpClientFactory, jsonOptions) { _config = new CdogsConfig(); configuration.Bind(CdogsConfigSectionKey, _config); diff --git a/source/backend/api/Repositories/Cdogs/CdogsRepository.cs b/source/backend/api/Repositories/Cdogs/CdogsRepository.cs index 435f0e4c8b..7d3a36d51e 100644 --- a/source/backend/api/Repositories/Cdogs/CdogsRepository.cs +++ b/source/backend/api/Repositories/Cdogs/CdogsRepository.cs @@ -10,6 +10,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Pims.Api.Models.Cdogs; using Pims.Api.Models.CodeTypes; using Pims.Api.Models.Requests.Http; @@ -30,12 +31,14 @@ public class CdogsRepository : CdogsBaseRepository, IDocumentGenerationRepositor /// Injected Httpclient factory. /// Injected repository that handles authentication. /// The injected configuration provider. + /// The jsonOptions. public CdogsRepository( ILogger logger, IHttpClientFactory httpClientFactory, IDocumentGenerationAuthRepository authRepository, - IConfiguration configuration) - : base(logger, httpClientFactory, configuration) + IConfiguration configuration, + IOptions jsonOptions) + : base(logger, httpClientFactory, configuration, jsonOptions) { _authRepository = authRepository; } diff --git a/source/backend/api/Repositories/Mayan/MayanAuthRepository.cs b/source/backend/api/Repositories/Mayan/MayanAuthRepository.cs index 4f534fdc29..952ba6b4d4 100644 --- a/source/backend/api/Repositories/Mayan/MayanAuthRepository.cs +++ b/source/backend/api/Repositories/Mayan/MayanAuthRepository.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Pims.Api.Models.CodeTypes; using Pims.Api.Models.Mayan; using Pims.Api.Models.Requests.Http; @@ -26,11 +27,13 @@ public class MayanAuthRepository : MayanBaseRepository, IEdmsAuthRepository /// Injected Logger Provider. /// Injected Httpclient factory. /// The injected configuration provider. + /// The jsonOptions. public MayanAuthRepository( ILogger logger, IHttpClientFactory httpClientFactory, - IConfiguration configuration) - : base(logger, httpClientFactory, configuration) + IConfiguration configuration, + IOptions jsonOptions) + : base(logger, httpClientFactory, configuration, jsonOptions) { _currentToken = string.Empty; } diff --git a/source/backend/api/Repositories/Mayan/MayanBaseRepository.cs b/source/backend/api/Repositories/Mayan/MayanBaseRepository.cs index ab06e068fc..a57678618a 100644 --- a/source/backend/api/Repositories/Mayan/MayanBaseRepository.cs +++ b/source/backend/api/Repositories/Mayan/MayanBaseRepository.cs @@ -1,8 +1,10 @@ using System.Collections.Generic; using System.Net.Http; using System.Net.Http.Headers; +using System.Text.Json; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Pims.Api.Models.Config; using Pims.Core.Api.Repositories.Rest; @@ -22,11 +24,13 @@ public abstract class MayanBaseRepository : BaseRestRepository /// Injected Logger Provider. /// Injected Httpclient factory. /// The injected configuration provider. + /// The injected json options. protected MayanBaseRepository( ILogger logger, IHttpClientFactory httpClientFactory, - IConfiguration configuration) - : base(logger, httpClientFactory) + IConfiguration configuration, + IOptions jsonOptions) + : base(logger, httpClientFactory, jsonOptions) { _config = new MayanConfig(); configuration.Bind(MayanConfigSectionKey, _config); diff --git a/source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs b/source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs index 89fb6921a3..72a3315968 100644 --- a/source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs +++ b/source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs @@ -11,6 +11,7 @@ using Microsoft.AspNetCore.WebUtilities; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Pims.Api.Models; using Pims.Api.Models.CodeTypes; using Pims.Api.Models.Mayan; @@ -35,12 +36,14 @@ public class MayanDocumentRepository : MayanBaseRepository, IEdmsDocumentReposit /// Injected Httpclient factory. /// Injected repository that handles authentication. /// The injected configuration provider. + /// The jsonOptions. public MayanDocumentRepository( ILogger logger, IHttpClientFactory httpClientFactory, IEdmsAuthRepository authRepository, - IConfiguration configuration) - : base(logger, httpClientFactory, configuration) + IConfiguration configuration, + IOptions jsonOptions) + : base(logger, httpClientFactory, configuration, jsonOptions) { _authRepository = authRepository; } diff --git a/source/backend/api/Repositories/Mayan/MayanMetadataRepository.cs b/source/backend/api/Repositories/Mayan/MayanMetadataRepository.cs index 24e77a0da3..9ecaa04300 100644 --- a/source/backend/api/Repositories/Mayan/MayanMetadataRepository.cs +++ b/source/backend/api/Repositories/Mayan/MayanMetadataRepository.cs @@ -7,6 +7,7 @@ using Microsoft.AspNetCore.WebUtilities; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Pims.Api.Models.Mayan; using Pims.Api.Models.Mayan.Metadata; using Pims.Api.Models.Requests.Http; @@ -28,12 +29,14 @@ public class MayanMetadataRepository : MayanBaseRepository, IEdmsMetadataReposit /// Injected Httpclient factory. /// Injected repository that handles authentication. /// The injected configuration provider. + /// The json options. public MayanMetadataRepository( ILogger logger, IHttpClientFactory httpClientFactory, IEdmsAuthRepository authRepository, - IConfiguration configuration) - : base(logger, httpClientFactory, configuration) + IConfiguration configuration, + IOptions jsonOptions) + : base(logger, httpClientFactory, configuration, jsonOptions) { _authRepository = authRepository; } diff --git a/source/backend/api/Services/DocumentFileService.cs b/source/backend/api/Services/DocumentFileService.cs index 098e3e794c..e0bca85e37 100644 --- a/source/backend/api/Services/DocumentFileService.cs +++ b/source/backend/api/Services/DocumentFileService.cs @@ -104,16 +104,15 @@ public async Task UploadResearchDocumentAsyn UploadResponse = uploadResult, }; - // Throw an error if Mayan returns a null document. This means it wasn't able to store it. ValidateDocumentUploadResponse(uploadResult); - if (uploadResult.Document is not null && uploadResult.Document.Id != 0) + if (uploadRequest.DocumentId != 0) { // Create the pims document research file relationship PimsResearchFileDocument newResearchFileDocument = new PimsResearchFileDocument() { ResearchFileId = researchFileId, - DocumentId = uploadResult.Document.Id, + DocumentId = uploadRequest.DocumentId, }; newResearchFileDocument = researchFileDocumentRepository.AddResearch(newResearchFileDocument); researchFileDocumentRepository.CommitTransaction(); @@ -139,16 +138,15 @@ public async Task UploadAcquisitionDocumentA UploadResponse = uploadResult, }; - // Throw an error if Mayan returns a null document. This means it wasn't able to store it. ValidateDocumentUploadResponse(uploadResult); - if (uploadResult.Document is not null && uploadResult.Document.Id != 0) + if (uploadRequest.DocumentId != 0) { // Create the pims document acquisition file relationship PimsAcquisitionFileDocument newAcquisitionDocument = new PimsAcquisitionFileDocument() { AcquisitionFileId = acquisitionFileId, - DocumentId = uploadResult.Document.Id, + DocumentId = uploadRequest.DocumentId, }; newAcquisitionDocument = acquisitionFileDocumentRepository.AddAcquisition(newAcquisitionDocument); acquisitionFileDocumentRepository.CommitTransaction(); @@ -174,15 +172,14 @@ public async Task UploadProjectDocumentAsync UploadResponse = uploadResult, }; - // Throw an error if Mayan returns a null document. This means it wasn't able to store it. ValidateDocumentUploadResponse(uploadResult); - if (uploadResult.Document is not null && uploadResult.Document.Id != 0) + if (uploadRequest.DocumentId != 0) { PimsProjectDocument newProjectDocument = new() { ProjectId = projectId, - DocumentId = uploadResult.Document.Id, + DocumentId = uploadRequest.DocumentId, }; newProjectDocument = _projectRepository.AddProjectDocument(newProjectDocument); _projectRepository.CommitTransaction(); @@ -208,15 +205,14 @@ public async Task UploadLeaseDocumentAsync(l UploadResponse = uploadResult, }; - // Throw an error if Mayan returns a null document. This means it wasn't able to store it. ValidateDocumentUploadResponse(uploadResult); - if (uploadResult.Document is not null && uploadResult.Document.Id != 0) + if (uploadRequest.DocumentId != 0) { PimsLeaseDocument newDocument = new() { LeaseId = leaseId, - DocumentId = uploadResult.Document.Id, + DocumentId = uploadRequest.DocumentId, }; newDocument = _leaseRepository.AddLeaseDocument(newDocument); _leaseRepository.CommitTransaction(); @@ -245,12 +241,12 @@ public async Task UploadPropertyActivityDocu // Throw an error if Mayan returns a null document. This means it wasn't able to store it. ValidateDocumentUploadResponse(uploadResult); - if (uploadResult.Document is not null && uploadResult.Document.Id != 0) + if (uploadRequest.DocumentId != 0) { PimsPropertyActivityDocument newDocument = new() { PimsPropertyActivityId = propertyActivityId, - DocumentId = uploadResult.Document.Id, + DocumentId = uploadRequest.DocumentId, }; newDocument = _propertyActivityDocumentRepository.AddPropertyActivityDocument(newDocument); _propertyActivityDocumentRepository.CommitTransaction(); @@ -279,12 +275,12 @@ public async Task UploadDispositionDocumentA // Throw an error if Mayan returns a null document. This means it wasn't able to store it. ValidateDocumentUploadResponse(uploadResult); - if (uploadResult.Document is not null && uploadResult.Document.Id != 0) + if (uploadRequest.DocumentId != 0) { PimsDispositionFileDocument newDocument = new() { DispositionFileId = dispositionFileId, - DocumentId = uploadResult.Document.Id, + DocumentId = uploadRequest.DocumentId, }; newDocument = _dispositionFileDocumentRepository.AddDispositionDocument(newDocument); _dispositionFileDocumentRepository.CommitTransaction(); @@ -413,9 +409,9 @@ private static void ValidateZeroLengthFile(DocumentUploadRequest uploadRequest) private static void ValidateDocumentUploadResponse(DocumentUploadResponse uploadResult) { - if (uploadResult.Document is null) + if (uploadResult?.DocumentExternalResponse?.Payload?.Id is null) { - throw new BadRequestException("Unexpected exception uploading file", new System.Exception(uploadResult.DocumentExternalResponse.Message)); + throw new BadRequestException("Unexpected exception uploading file", new BadRequestException(uploadResult?.DocumentExternalResponse?.Message)); } } } diff --git a/source/backend/api/Services/DocumentQueueService.cs b/source/backend/api/Services/DocumentQueueService.cs index 4ffb2c4363..9bd904b5e7 100644 --- a/source/backend/api/Services/DocumentQueueService.cs +++ b/source/backend/api/Services/DocumentQueueService.cs @@ -1,7 +1,23 @@ +using System; +using System.Collections; using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; using System.Security.Claims; +using System.Text.Json; +using System.Threading.Tasks; +using DocumentFormat.OpenXml.InkML; +using Humanizer; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Pims.Api.Models.CodeTypes; +using Pims.Api.Models.Concepts.Document; +using Pims.Api.Models.Mayan.Document; +using Pims.Api.Models.Requests.Document.Upload; +using Pims.Api.Models.Requests.Http; +using Pims.Core.Api.Exceptions; using Pims.Core.Api.Services; using Pims.Core.Extensions; using Pims.Core.Http.Configuration; @@ -9,6 +25,7 @@ using Pims.Dal.Entities; using Pims.Dal.Entities.Models; using Pims.Dal.Repositories; +using Serilog.Filters; namespace Pims.Api.Services { @@ -17,26 +34,276 @@ namespace Pims.Api.Services /// public class DocumentQueueService : BaseService, IDocumentQueueService { - private readonly IDocumentQueueRepository documentQueueRepository; - private readonly IOptionsMonitor keycloakOptions; + private readonly IDocumentQueueRepository _documentQueueRepository; + private readonly IDocumentRepository _documentRepository; + private readonly IDocumentTypeRepository _documentTypeRepository; + private readonly IDocumentService _documentService; + private readonly IOptionsMonitor _keycloakOptions; public DocumentQueueService( ClaimsPrincipal user, ILogger logger, IDocumentQueueRepository documentQueueRepository, + IDocumentRepository documentRepository, + IDocumentTypeRepository documentTypeRepository, + IDocumentService documentService, IOptionsMonitor options) : base(user, logger) { - this.documentQueueRepository = documentQueueRepository; - this.keycloakOptions = options; + this._documentQueueRepository = documentQueueRepository; + this._documentRepository = documentRepository; + this._documentTypeRepository = documentTypeRepository; + this._documentService = documentService; + this._keycloakOptions = options; } + + /// + /// Searches for documents in the document queue based on the specified filter. + /// + /// The filter criteria to apply when searching the document queue. + /// An enumerable collection of that match the filter criteria. + /// Thrown when the user is not authorized to perform this operation. public IEnumerable SearchDocumentQueue(DocumentQueueFilter filter) { this.Logger.LogInformation("Retrieving queued PIMS documents using filter {filter}", filter); - this.User.ThrowIfNotAuthorizedOrServiceAccount(Permissions.SystemAdmin, this.keycloakOptions); + this.User.ThrowIfNotAuthorizedOrServiceAccount(Permissions.SystemAdmin, this._keycloakOptions); + + return _documentQueueRepository.GetAllByFilter(filter); + } + + /// + /// Updates the specified document queue. + /// + /// The document queue object to update. + /// The updated document queue object. + /// Thrown when the user is not authorized to perform this operation. + public PimsDocumentQueue Update(PimsDocumentQueue documentQueue) + { + this.Logger.LogInformation("Updating queued document {documentQueueId}", documentQueue.DocumentQueueId); + this.Logger.LogDebug("Incoming queued document {document}", documentQueue.Serialize()); + + this.User.ThrowIfNotAuthorizedOrServiceAccount(Permissions.SystemAdmin, this._keycloakOptions); + + _documentQueueRepository.Update(documentQueue); + _documentQueueRepository.CommitTransaction(); + return documentQueue; + } + + + /// + /// Polls for the status of a document in mayan, and updates the queue based on the result. + /// + /// The document queue object containing the document details. + /// A task that represents the asynchronous operation. The task result contains the updated document queue object, or null if the polling failed. + /// Thrown when the user is not authorized to perform this operation. + /// Thrown when the document queue does not have a valid document ID or related document. + public async Task PollForDocument(PimsDocumentQueue documentQueue) + { + this.Logger.LogInformation("Polling queued document {documentQueueId}", documentQueue.DocumentQueueId); + this.Logger.LogDebug("Polling queued document {document}", documentQueue.Serialize()); + + this.User.ThrowIfNotAuthorizedOrServiceAccount(Permissions.SystemAdmin, this._keycloakOptions); + if (documentQueue.DocumentId == null) + { + this.Logger.LogError("polled queued document does not have a document Id {documentQueueId}", documentQueue.DocumentQueueId); + throw new InvalidDataException("DocumentId is required to poll for a document."); + } + + var databaseDocumentQueue = _documentQueueRepository.TryGetById(documentQueue.DocumentQueueId); + if (databaseDocumentQueue == null) + { + this.Logger.LogError("Unable to find document queue with {id}", documentQueue.DocumentQueueId); + throw new KeyNotFoundException($"Unable to find document queue with matching id: {documentQueue.DocumentQueueId}"); + } + + var relatedDocument = _documentRepository.TryGet(documentQueue.DocumentId.Value); + + if (relatedDocument?.MayanId == null || relatedDocument?.MayanId < 0) + { + this.Logger.LogError("Queued Document {documentQueueId} has no mayan id and is invalid.", documentQueue.DocumentQueueId); + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.PIMS_ERROR); + return databaseDocumentQueue; + } + + ExternalResponse documentDetailsResponse = await _documentService.GetStorageDocumentDetail(relatedDocument.MayanId.Value); + + if (documentDetailsResponse.Status != ExternalResponseStatus.Success || documentDetailsResponse?.Payload == null) + { + this.Logger.LogError("Polling for queued document {documentQueueId} failed with status {documentDetailsResponseStatus}", documentQueue.DocumentQueueId, documentDetailsResponse.Status); + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.PIMS_ERROR); + return databaseDocumentQueue; + } + + if (documentDetailsResponse.Payload.FileLatest?.Id == null) + { + this.Logger.LogInformation("Polling for queued document {documentQueueId} complete, file still processing", documentQueue.DocumentQueueId); + } + else + { + this.Logger.LogInformation("Polling for queued document {documentQueueId} complete, file uploaded successfully", documentQueue.DocumentQueueId); + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.SUCCESS); + } + + return databaseDocumentQueue; + } + - return documentQueueRepository.GetAllByFilter(filter); + /// + /// Uploads the specified document queue. + /// + /// The document queue object containing the document to upload. + /// A task that represents the asynchronous operation. The task result contains the updated document queue object, or null if the upload failed. + /// Thrown when the user is not authorized to perform this operation. + /// Thrown when the document queue does not have a valid document ID or related document. + public async Task Upload(PimsDocumentQueue documentQueue) + { + this.Logger.LogInformation("Uploading queued document {documentQueueId}", documentQueue.DocumentQueueId); + this.Logger.LogDebug("Uploading queued document {document}", documentQueue.Serialize()); + + this.User.ThrowIfNotAuthorizedOrServiceAccount(Permissions.SystemAdmin, this._keycloakOptions); + + var databaseDocumentQueue = _documentQueueRepository.TryGetById(documentQueue.DocumentQueueId); + if(databaseDocumentQueue == null) + { + this.Logger.LogError("Unable to find document queue with {id}", documentQueue.DocumentQueueId); + throw new KeyNotFoundException($"Unable to find document queue with matching id: {documentQueue.DocumentQueueId}"); + } + databaseDocumentQueue.DocProcessStartDt = DateTime.UtcNow; + + // if the document queued for upload is already in an error state, update the retries. + if (databaseDocumentQueue.DocumentQueueStatusTypeCode == DocumentQueueStatusTypes.PIMS_ERROR.ToString() || databaseDocumentQueue.DocumentQueueStatusTypeCode == DocumentQueueStatusTypes.MAYAN_ERROR.ToString()) + { + databaseDocumentQueue.DocProcessRetries += 1; + databaseDocumentQueue.DocProcessEndDt = null; + } + + bool isValid = ValidateQueuedDocument(databaseDocumentQueue, documentQueue); + if (!isValid) + { + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.PIMS_ERROR); + return databaseDocumentQueue; + } + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.PROCESSING); + + PimsDocument relatedDocument = null; + relatedDocument = _documentRepository.TryGetDocumentRelationships(databaseDocumentQueue.DocumentId.Value); + if (relatedDocument?.DocumentTypeId == null) + { + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.PIMS_ERROR); + this.Logger.LogError("Queued document {documentQueueId} does not have a related PIMS_DOCUMENT {documentId} with valid DocumentType, aborting.", databaseDocumentQueue.DocumentQueueId, relatedDocument?.DocumentId); + return databaseDocumentQueue; + } + else if (relatedDocument?.MayanId != null && relatedDocument?.MayanId > 0) + { + this.Logger.LogInformation("Queued document {documentQueueId} already has a mayan id {mayanid}, no further processing required.", databaseDocumentQueue.DocumentQueueId, relatedDocument.MayanId); + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.SUCCESS); + return databaseDocumentQueue; // The document poll job should pick this up and fix the document queue status. + } + + try + { + PimsDocumentTyp documentTyp = _documentTypeRepository.GetById(relatedDocument.DocumentTypeId); // throws KeyNotFoundException if not found. + + IFormFile file = null; + using MemoryStream memStream = new(databaseDocumentQueue.Document); + file = new FormFile(memStream, 0, databaseDocumentQueue.Document.Length, relatedDocument.FileName, relatedDocument.FileName); + + DocumentUploadRequest request = new DocumentUploadRequest() + { + File = file, + DocumentStatusCode = relatedDocument.DocumentStatusTypeCode, + DocumentTypeId = relatedDocument.DocumentTypeId, + DocumentTypeMayanId = documentTyp.MayanId, + DocumentId = relatedDocument.DocumentId, + DocumentMetadata = databaseDocumentQueue.DocumentMetadata != null ? JsonSerializer.Deserialize>(databaseDocumentQueue.DocumentMetadata) : null, + }; + DocumentUploadResponse response = await _documentService.UploadDocumentAsync(request); + + if (response.DocumentExternalResponse.Status != ExternalResponseStatus.Success || response?.DocumentExternalResponse?.Payload == null) + { + this.Logger.LogError( + "Queued document upload failed {databaseDocumentQueueDocumentQueueId} {databaseDocumentQueueDocumentQueueStatusTypeCode}, {documentExternalResponseStatus}", + databaseDocumentQueue.DocumentQueueId, + databaseDocumentQueue.DocumentQueueStatusTypeCode, + response.DocumentExternalResponse.Status); + + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.MAYAN_ERROR); + return databaseDocumentQueue; + } + response.MetadataExternalResponse.Where(r => r.Status != ExternalResponseStatus.Success).ForEach(r => this.Logger.LogError("url: ${url} status: ${status} message ${message}", r.Payload.Url, r.Status, r.Message)); // Log any metadata errors, but don't fail the upload. + + // Mayan may have already returned a file id from the original upload. If not, this job will remain in the processing state (to be periodically checked for completion in another job). + if (response.DocumentExternalResponse?.Payload?.FileLatest?.Id != null) + { + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.SUCCESS); + } + } + catch (Exception ex) when (ex is BadRequestException || ex is KeyNotFoundException || ex is InvalidDataException || ex is JsonException) + { + this.Logger.LogError($"Error: {ex.Message}"); + UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.PIMS_ERROR); + } + return databaseDocumentQueue; + } + + /// + /// Updates the status of the specified document queue. + /// + /// The document queue object to update. + /// The new status type to set for the document queue. + /// + /// This method updates the document queue's status and commits the transaction. + /// If the status is a final state, it also updates the processing end date. + /// + private void UpdateDocumentQueueStatus(PimsDocumentQueue documentQueue, DocumentQueueStatusTypes statusType) + { + documentQueue.DocumentQueueStatusTypeCode = statusType.ToString(); + bool removeDocument = false; + + // Any final states should update the processing end date. + if (statusType != DocumentQueueStatusTypes.PROCESSING && statusType != DocumentQueueStatusTypes.PENDING) + { + documentQueue.DocProcessEndDt = DateTime.UtcNow; + if (statusType == DocumentQueueStatusTypes.SUCCESS) + { + documentQueue.Document = null; + removeDocument = true; + } + } + _documentQueueRepository.Update(documentQueue, removeDocument); + _documentQueueRepository.CommitTransaction(); + } + + + /// + /// Validates the queued document against the database document queue. + /// + /// The document queue object from the database. + /// The document queue object to validate against the database. + /// True if the queued document is valid; otherwise, false. + /// + /// This method checks if the status type, process retries, and document content are valid. + /// It also ensures that at least one file document ID is associated with the document. + /// + private bool ValidateQueuedDocument(PimsDocumentQueue databaseDocumentQueue, PimsDocumentQueue externalDocument) + { + if (databaseDocumentQueue.DocumentQueueStatusTypeCode != externalDocument.DocumentQueueStatusTypeCode) + { + this.Logger.LogError("Requested document queue status: {documentQueueStatusTypeCode} does not match current database status: {documentQueueStatusTypeCode}", externalDocument.DocumentQueueStatusTypeCode, databaseDocumentQueue.DocumentQueueStatusTypeCode); + return false; + } + else if (databaseDocumentQueue.DocProcessRetries != externalDocument.DocProcessRetries) + { + this.Logger.LogError("Requested document retries: {documentQueueStatusTypeCode} does not match current database retries: {documentQueueStatusTypeCode}", externalDocument.DocumentQueueStatusTypeCode, databaseDocumentQueue.DocumentQueueStatusTypeCode); + return false; + } + else if (databaseDocumentQueue.Document == null || databaseDocumentQueue.DocumentId == null) + { + this.Logger.LogError("Queued document file content is empty, unable to upload."); + return false; + } + return true; } } } diff --git a/source/backend/api/Services/DocumentService.cs b/source/backend/api/Services/DocumentService.cs index 333fdb1c2b..11eeebdae8 100644 --- a/source/backend/api/Services/DocumentService.cs +++ b/source/backend/api/Services/DocumentService.cs @@ -6,6 +6,7 @@ using System.Net.Http; using System.Security.Claims; using System.Threading.Tasks; +using Azure; using MapsterMapper; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Configuration; @@ -136,9 +137,9 @@ public IList GetPimsDocumentTypes(DocumentRelationType relation return documentTypeRepository.GetByCategory(categoryType); } - public async Task UploadDocumentAsync(DocumentUploadRequest uploadRequest) + public async Task UploadDocumentSync(DocumentUploadRequest uploadRequest) { - this.Logger.LogInformation("Uploading document"); + this.Logger.LogInformation("Uploading document and waiting for mayan upload."); this.User.ThrowIfNotAuthorized(Permissions.DocumentAdd); ExternalResponse externalResponse = await UploadDocumentAsync(uploadRequest.DocumentTypeMayanId, uploadRequest.File); @@ -209,6 +210,47 @@ public async Task UploadDocumentAsync(DocumentUploadRequ return response; } + public async Task UploadDocumentAsync(DocumentUploadRequest uploadRequest) + { + this.Logger.LogInformation("Uploading document, do not wait for mayan processing."); + this.User.ThrowIfNotAuthorized(Permissions.DocumentAdd); + + ExternalResponse externalResponse = await UploadDocumentAsync(uploadRequest.DocumentTypeMayanId, uploadRequest.File); + DocumentUploadResponse response = new DocumentUploadResponse() + { + DocumentExternalResponse = externalResponse, + MetadataExternalResponse = new List>(), + }; + + PimsDocument databaseDocument = documentRepository.TryGet(uploadRequest.DocumentId); + response.Document = databaseDocument != null ? mapper.Map(databaseDocument) : null; + + if (response?.DocumentExternalResponse?.Payload?.Id != null && response?.DocumentExternalResponse?.Payload?.Id > 0 && databaseDocument != null) + { + databaseDocument.MayanId = response.DocumentExternalResponse.Payload.Id; + documentRepository.Update(databaseDocument); + documentRepository.CommitTransaction(); + } + else + { + this.Logger.LogError("Failed to update associated PIMS document with uploaded Mayan Id."); + } + + return response; + } + + public PimsDocument AddDocument(PimsDocument newPimsDocument) + { + this.Logger.LogInformation("Adding document uploaded asynchronously."); + this.User.ThrowIfNotAuthorized(Permissions.DocumentAdd); + newPimsDocument.ThrowIfNull(nameof(newPimsDocument)); + + documentRepository.Add(newPimsDocument); + documentRepository.CommitTransaction(); + + return newPimsDocument; + } + public async Task UpdateDocumentAsync(DocumentUpdateRequest updateRequest) { this.Logger.LogInformation("Updating document {documentId}", updateRequest.DocumentId); diff --git a/source/backend/api/Services/FormDocumentService.cs b/source/backend/api/Services/FormDocumentService.cs index 00cab505bd..5d7cc8ccda 100644 --- a/source/backend/api/Services/FormDocumentService.cs +++ b/source/backend/api/Services/FormDocumentService.cs @@ -80,16 +80,16 @@ public async Task UploadFormDocumentTemplate } } - DocumentUploadResponse uploadResult = await _documentService.UploadDocumentAsync(uploadRequest); + DocumentUploadResponse uploadResult = await _documentService.UploadDocumentSync(uploadRequest); DocumentUploadRelationshipResponse relationshipResponse = new DocumentUploadRelationshipResponse() { UploadResponse = uploadResult, }; - if (uploadResult.DocumentExternalResponse.Status == ExternalResponseStatus.Success && uploadResult.Document != null && uploadResult.Document.Id != 0) + if (uploadResult.DocumentExternalResponse.Status == ExternalResponseStatus.Success) { - currentFormType.DocumentId = uploadResult.Document.Id; + currentFormType.DocumentId = uploadRequest.DocumentId; var updatedFormType = _formTypeRepository.SetFormTypeDocument(currentFormType); _formTypeRepository.CommitTransaction(); diff --git a/source/backend/api/Services/IDocumentQueueService.cs b/source/backend/api/Services/IDocumentQueueService.cs index b1f3c09206..afb60b2800 100644 --- a/source/backend/api/Services/IDocumentQueueService.cs +++ b/source/backend/api/Services/IDocumentQueueService.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Threading.Tasks; using Pims.Dal.Entities; using Pims.Dal.Entities.Models; @@ -10,5 +11,11 @@ namespace Pims.Api.Services public interface IDocumentQueueService { public IEnumerable SearchDocumentQueue(DocumentQueueFilter filter); + + public PimsDocumentQueue Update(PimsDocumentQueue documentQueue); + + public Task PollForDocument(PimsDocumentQueue documentQueue); + + public Task Upload(PimsDocumentQueue documentQueue); } } diff --git a/source/backend/api/Services/IDocumentService.cs b/source/backend/api/Services/IDocumentService.cs index ebcd1ded2e..9ddd6ef5bd 100644 --- a/source/backend/api/Services/IDocumentService.cs +++ b/source/backend/api/Services/IDocumentService.cs @@ -40,6 +40,8 @@ public interface IDocumentService Task UploadDocumentAsync(DocumentUploadRequest uploadRequest); + Task UploadDocumentSync(DocumentUploadRequest uploadRequest); + Task UpdateDocumentAsync(DocumentUpdateRequest updateRequest); Task> DeleteDocumentAsync(PimsDocument document); @@ -49,5 +51,7 @@ public interface IDocumentService Task>> GetDocumentFilePageListAsync(long documentId, long documentFileId); Task DownloadFilePageImageAsync(long mayanDocumentId, long mayanFileId, long mayanFilePageId); + + PimsDocument AddDocument(PimsDocument newPimsDocument); } } diff --git a/source/backend/api/appsettings.Development.json b/source/backend/api/appsettings.Development.json index 987052db9c..7df068180a 100644 --- a/source/backend/api/appsettings.Development.json +++ b/source/backend/api/appsettings.Development.json @@ -31,7 +31,7 @@ } }, "ConnectionStrings": { - "PIMS": "Server=localhost,5433;uid=admin;Database=pims;Password=Password12" + "PIMS": "Server=localhost,5433;User ID=admin;Database=pims;TrustServerCertificate=True;Encrypt=false;" }, "Pims": { "Environment": { diff --git a/source/backend/apimodels/CodeTypes/DocumentStatusTypes.cs b/source/backend/apimodels/CodeTypes/DocumentStatusTypes.cs new file mode 100644 index 0000000000..95f6372c7e --- /dev/null +++ b/source/backend/apimodels/CodeTypes/DocumentStatusTypes.cs @@ -0,0 +1,40 @@ +using System.Runtime.Serialization; +using System.Text.Json.Serialization; + +namespace Pims.Api.Models.CodeTypes +{ + [JsonConverter(typeof(JsonStringEnumMemberConverter))] + public enum DocumentStatusTypes + { + + [EnumMember(Value = "AMENDD")] + AMENDD, + + [EnumMember(Value = "APPROVD")] + APPROVD, + + [EnumMember(Value = "CNCLD")] + CNCLD, + + [EnumMember(Value = "DRAFT")] + DRAFT, + + [EnumMember(Value = "FINAL")] + FINAL, + + [EnumMember(Value = "NONE")] + NONE, + + [EnumMember(Value = "RGSTRD")] + RGSTRD, + + [EnumMember(Value = "SENT")] + SENT, + + [EnumMember(Value = "SIGND")] + SIGND, + + [EnumMember(Value = "UNREGD")] + UNREGD, + } +} diff --git a/source/backend/apimodels/Models/Concepts/DocumentQueue/DocumentQueueMap.cs b/source/backend/apimodels/Models/Concepts/DocumentQueue/DocumentQueueMap.cs index 1a98ba0d07..1a78f3157b 100644 --- a/source/backend/apimodels/Models/Concepts/DocumentQueue/DocumentQueueMap.cs +++ b/source/backend/apimodels/Models/Concepts/DocumentQueue/DocumentQueueMap.cs @@ -1,5 +1,6 @@ using Mapster; using Pims.Api.Models.Base; +using Pims.Dal.Entities; using Entity = Pims.Dal.Entities; namespace Pims.Api.Models.Concepts.Document @@ -12,12 +13,15 @@ public void Register(TypeAdapterConfig config) .Map(dest => dest.Id, src => src.DocumentQueueId) .Map(dest => dest.DocumentExternalId, src => src.DocumentExternalId) .Map(dest => dest.DocumentId, src => src.DocumentId) - .Map(dest => dest.DocumentQueueStatusType, src => src.DocumentQueueStatusTypeCodeNavigation) + .Map(dest => dest.DocumentQueueStatusType, src => src.DocumentQueueStatusTypeCodeNavigation == null ? new PimsDocumentQueueStatusType() { Id = src.DocumentQueueStatusTypeCode } : src.DocumentQueueStatusTypeCodeNavigation) .Map(dest => dest.DataSourceTypeCode, src => src.DataSourceTypeCodeNavigation) .Map(dest => dest.DocumentProcessStartTimestamp, src => src.DocProcessStartDt) .Map(dest => dest.DocumentProcessEndTimestamp, src => src.DocProcessEndDt) .Map(dest => dest.DocumentProcessRetries, src => src.DocProcessRetries) .Map(dest => dest.Document, src => src.Document) + .Map(dest => dest.PimsDocument, src => src.DocumentNavigation) + .Map(dest => dest.MayanError, src => src.MayanError) + .Map(dest => dest.FileName, src => src.FileName) .Inherits(); config.NewConfig() @@ -30,6 +34,9 @@ public void Register(TypeAdapterConfig config) .Map(dest => dest.DocProcessEndDt, src => src.DocumentProcessEndTimestamp) .Map(dest => dest.DocProcessRetries, src => src.DocumentProcessRetries) .Map(dest => dest.Document, src => src.Document) + .Map(dest => dest.DocumentNavigation, src => src.PimsDocument) + .Map(dest => dest.MayanError, src => src.MayanError) + .Map(dest => dest.FileName, src => src.FileName) .Inherits(); } } diff --git a/source/backend/apimodels/Models/Concepts/DocumentQueue/DocumentQueueModel.cs b/source/backend/apimodels/Models/Concepts/DocumentQueue/DocumentQueueModel.cs index 73d553f74d..d926200862 100644 --- a/source/backend/apimodels/Models/Concepts/DocumentQueue/DocumentQueueModel.cs +++ b/source/backend/apimodels/Models/Concepts/DocumentQueue/DocumentQueueModel.cs @@ -56,12 +56,21 @@ public class DocumentQueueModel : BaseAuditModel /// public string MayanError { get; set; } + /// + /// get/set - The file name of the document file. + /// + public string FileName { get; set; } + /// /// get/set - The actual document, represented as a byte[]. /// public byte[] Document { get; set; } + /// + /// get/set - The actual document, represented as a byte[]. + /// + public DocumentModel PimsDocument { get; set; } - #endregion - } + #endregion +} } diff --git a/source/backend/apimodels/Models/Requests/Document/Upload/DocumentUploadRequest.cs b/source/backend/apimodels/Models/Requests/Document/Upload/DocumentUploadRequest.cs index 83102292e3..7d08d437e5 100644 --- a/source/backend/apimodels/Models/Requests/Document/Upload/DocumentUploadRequest.cs +++ b/source/backend/apimodels/Models/Requests/Document/Upload/DocumentUploadRequest.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.IO; using Microsoft.AspNetCore.Http; using Pims.Api.Models.Concepts.Document; @@ -21,6 +22,11 @@ public class DocumentUploadRequest /// public long DocumentTypeId { get; set; } + /// + /// get/set - The id of the document to be uploaded (in PIMS). + /// + public long DocumentId { get; set; } + /// /// get/set - Initial status code of the document. /// diff --git a/source/backend/core.api/Pims.Core.Api.csproj b/source/backend/core.api/Pims.Core.Api.csproj index 70c947b9d1..51d218ea46 100644 --- a/source/backend/core.api/Pims.Core.Api.csproj +++ b/source/backend/core.api/Pims.Core.Api.csproj @@ -20,6 +20,7 @@ + diff --git a/source/backend/core.api/Repositories/RestCommon/BaseRestRepository.cs b/source/backend/core.api/Repositories/RestCommon/BaseRestRepository.cs index 214d359f9b..324cd22693 100644 --- a/source/backend/core.api/Repositories/RestCommon/BaseRestRepository.cs +++ b/source/backend/core.api/Repositories/RestCommon/BaseRestRepository.cs @@ -10,6 +10,7 @@ using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Pims.Api.Models; using Pims.Api.Models.CodeTypes; using Pims.Api.Models.Requests.Http; @@ -23,6 +24,7 @@ public abstract class BaseRestRepository : IRestRespository { protected readonly IHttpClientFactory _httpClientFactory; protected readonly ILogger _logger; + protected readonly IOptions _jsonOptions; /// /// Initializes a new instance of the class. @@ -31,10 +33,13 @@ public abstract class BaseRestRepository : IRestRespository /// Injected Httpclient factory. protected BaseRestRepository( ILogger logger, - IHttpClientFactory httpClientFactory) + IHttpClientFactory httpClientFactory, + IOptions jsonOptions + ) { _logger = logger; _httpClientFactory = httpClientFactory; + _jsonOptions = jsonOptions; } public abstract void AddAuthentication(HttpClient client, string authenticationToken = null); @@ -305,7 +310,7 @@ private async Task> ProcessResponse(HttpResponseMessage r }; _logger.LogTrace("Response: {response}", response); - string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(true); + var payload = await response.Content.ReadAsStreamAsync().ConfigureAwait(true); result.HttpStatusCode = response.StatusCode; switch (response.StatusCode) @@ -321,7 +326,7 @@ private async Task> ProcessResponse(HttpResponseMessage r result.Payload = (T)Convert.ChangeType(payload, typeof(T), CultureInfo.InvariantCulture); break; default: - T requestTokenResult = JsonSerializer.Deserialize(payload); + T requestTokenResult = JsonSerializer.Deserialize(payload, _jsonOptions.Value); result.Payload = requestTokenResult; break; } @@ -342,7 +347,7 @@ private async Task> ProcessResponse(HttpResponseMessage r case HttpStatusCode.BadRequest: case HttpStatusCode.MethodNotAllowed: result.Status = ExternalResponseStatus.Error; - result.Message = payload; + result.Message = await response.Content.ReadAsStringAsync(); break; default: result.Status = ExternalResponseStatus.Error; diff --git a/source/backend/dal/Repositories/DocumentQueueRepository.cs b/source/backend/dal/Repositories/DocumentQueueRepository.cs index fa2472427a..0f31e4584e 100644 --- a/source/backend/dal/Repositories/DocumentQueueRepository.cs +++ b/source/backend/dal/Repositories/DocumentQueueRepository.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Linq; using System.Security.Claims; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Pims.Core.Extensions; using Pims.Dal.Entities; @@ -32,14 +33,35 @@ public DocumentQueueRepository( #region Methods + /// + /// Attempts to find a queued document via the documentQueueId. Returns null if not found; + /// + /// + /// + public PimsDocumentQueue TryGetById(long documentQueueId) + { + + return Context.PimsDocumentQueues + .AsNoTracking() + .FirstOrDefault(dq => dq.DocumentQueueId == documentQueueId); + } + /// /// Updates the queued document in the database. /// /// /// - public PimsDocumentQueue Update(PimsDocumentQueue queuedDocument) + public PimsDocumentQueue Update(PimsDocumentQueue queuedDocument, bool removeDocument = false) { queuedDocument.ThrowIfNull(nameof(queuedDocument)); + var existingQueuedDocument = TryGetById(queuedDocument.DocumentQueueId); + + if (!removeDocument) + { + queuedDocument.Document = existingQueuedDocument.Document; + } + + Context.Entry(existingQueuedDocument).CurrentValues.SetValues(queuedDocument); queuedDocument = Context.Update(queuedDocument).Entity; return queuedDocument; @@ -65,25 +87,62 @@ public bool Delete(PimsDocumentQueue queuedDocument) /// public IEnumerable GetAllByFilter(DocumentQueueFilter filter) { - var query = Context.PimsDocumentQueues.Where(q => true); + var query = Context.PimsDocumentQueues + .Include(dq => dq.DocumentNavigation) + .ThenInclude(d => d.DocumentType) + .Include(dq => dq.DocumentQueueStatusTypeCodeNavigation) + .Where(q => true); if (filter.DataSourceTypeCode != null) { - query.Where(d => d.DataSourceTypeCode == filter.DataSourceTypeCode); + query = query.Where(d => d.DataSourceTypeCode == filter.DataSourceTypeCode); } - if (filter.DocumentQueueStatusTypeCode != null) + if (filter.DocumentQueueStatusTypeCodes != null && filter.DocumentQueueStatusTypeCodes.Length > 0) { - query.Where(d => d.DocumentQueueStatusTypeCode == filter.DocumentQueueStatusTypeCode); + query = query.Where(d => filter.DocumentQueueStatusTypeCodes.Any(filterStatus => d.DocumentQueueStatusTypeCode == filterStatus)); } if (filter.DocProcessStartDate != null) { - query.Where(d => d.DocProcessStartDt >= filter.DocProcessStartDate); + query = query.Where(d => d.DocProcessStartDt >= filter.DocProcessStartDate); } if (filter.DocProcessEndDate != null) { - query.Where(d => d.DocProcessEndDt <= filter.DocProcessEndDate); + query = query.Where(d => d.DocProcessEndDt <= filter.DocProcessEndDate); } - return query.ToList(); + if (filter.MaxDocProcessRetries != null) + { + query = query.Where(d => d.DocProcessRetries == null || d.DocProcessRetries < filter.MaxDocProcessRetries); + } + + // Return the PimsDocumentQueue search results without the file contents - to avoid memory issues. + return query.Take(filter.Quantity).Select(dq => new PimsDocumentQueue() + { + DocumentQueueId = dq.DocumentQueueId, + DocumentId = dq.DocumentId, + DocumentQueueStatusTypeCode = dq.DocumentQueueStatusTypeCode, + DocumentQueueStatusTypeCodeNavigation = dq.DocumentQueueStatusTypeCodeNavigation, + DataSourceTypeCode = dq.DataSourceTypeCode, + DataSourceTypeCodeNavigation = dq.DataSourceTypeCodeNavigation, + DocumentExternalId = dq.DocumentExternalId, + DocProcessStartDt = dq.DocProcessStartDt, + DocProcessEndDt = dq.DocProcessEndDt, + DocProcessRetries = dq.DocProcessRetries, + MayanError = dq.MayanError, + FileName = dq.FileName, + AppCreateTimestamp = dq.AppCreateTimestamp, + AppCreateUserDirectory = dq.AppCreateUserDirectory, + AppCreateUserGuid = dq.AppCreateUserGuid, + AppCreateUserid = dq.AppCreateUserid, + AppLastUpdateTimestamp = dq.AppLastUpdateTimestamp, + AppLastUpdateUserDirectory = dq.AppLastUpdateUserDirectory, + AppLastUpdateUserGuid = dq.AppLastUpdateUserGuid, + AppLastUpdateUserid = dq.AppLastUpdateUserid, + DbCreateTimestamp = dq.DbCreateTimestamp, + DbCreateUserid = dq.DbCreateUserid, + DbLastUpdateTimestamp = dq.DbLastUpdateTimestamp, + DbLastUpdateUserid = dq.DbLastUpdateUserid, + ConcurrencyControlNumber = dq.ConcurrencyControlNumber, + }).ToList(); } public int DocumentQueueCount(PimsDocumentQueueStatusType pimsDocumentQueueStatusType) diff --git a/source/backend/dal/Repositories/DocumentRepository.cs b/source/backend/dal/Repositories/DocumentRepository.cs index c9a9716baa..5f48f97ed9 100644 --- a/source/backend/dal/Repositories/DocumentRepository.cs +++ b/source/backend/dal/Repositories/DocumentRepository.cs @@ -5,7 +5,6 @@ using Microsoft.Extensions.Logging; using Pims.Core.Extensions; using Pims.Dal.Entities; -using Pims.Dal.Helpers.Extensions; using Pims.Core.Security; namespace Pims.Dal.Repositories @@ -44,6 +43,23 @@ public PimsDocument TryGet(long documentId) return this.Context.PimsDocuments.AsNoTracking().FirstOrDefault(x => x.DocumentId == documentId); } + public PimsDocument TryGetDocumentRelationships(long documentId) + { + var documentRelationships = Context.PimsDocuments.AsNoTracking() + .Include(d => d.PimsResearchFileDocuments) + .Include(d => d.PimsAcquisitionFileDocuments) + .Include(d => d.PimsProjectDocuments) + .Include(d => d.PimsFormTypes) + .Include(d => d.PimsLeaseDocuments) + .Include(d => d.PimsPropertyActivityDocuments) + .Include(d => d.PimsDispositionFileDocuments) + .Where(d => d.DocumentId == documentId) + .AsNoTracking() + .FirstOrDefault(); + + return documentRelationships; + } + /// /// Adds the passed document to the database. /// diff --git a/source/backend/dal/Repositories/Interfaces/IDocumentQueueRepository.cs b/source/backend/dal/Repositories/Interfaces/IDocumentQueueRepository.cs index 45c14d19ac..4da1e7e0fe 100644 --- a/source/backend/dal/Repositories/Interfaces/IDocumentQueueRepository.cs +++ b/source/backend/dal/Repositories/Interfaces/IDocumentQueueRepository.cs @@ -9,9 +9,12 @@ namespace Pims.Dal.Repositories /// public interface IDocumentQueueRepository : IRepository { + + PimsDocumentQueue TryGetById(long documentQueueId); + IEnumerable GetAllByFilter(DocumentQueueFilter filter); - PimsDocumentQueue Update(PimsDocumentQueue queuedDocument); + PimsDocumentQueue Update(PimsDocumentQueue queuedDocument, bool removeDocument = false); bool Delete(PimsDocumentQueue queuedDocument); diff --git a/source/backend/dal/Repositories/Interfaces/IDocumentRepository.cs b/source/backend/dal/Repositories/Interfaces/IDocumentRepository.cs index 3be4e2b302..5551d41f13 100644 --- a/source/backend/dal/Repositories/Interfaces/IDocumentRepository.cs +++ b/source/backend/dal/Repositories/Interfaces/IDocumentRepository.cs @@ -16,5 +16,7 @@ public interface IDocumentRepository : IRepository bool Delete(PimsDocument document); int DocumentRelationshipCount(long documentId); + + PimsDocument TryGetDocumentRelationships(long documentId); } } diff --git a/source/backend/dal/Repositories/UserRepository.cs b/source/backend/dal/Repositories/UserRepository.cs index 4e4902ffd1..3150d21730 100644 --- a/source/backend/dal/Repositories/UserRepository.cs +++ b/source/backend/dal/Repositories/UserRepository.cs @@ -7,12 +7,12 @@ using Microsoft.Extensions.Options; using Pims.Core.Extensions; using Pims.Core.Http.Configuration; +using Pims.Core.Security; using Pims.Dal.Entities; using Pims.Dal.Entities.Comparers; using Pims.Dal.Entities.Models; using Pims.Dal.Exceptions; using Pims.Dal.Helpers.Extensions; -using Pims.Core.Security; namespace Pims.Dal.Repositories { diff --git a/source/backend/entities/Models/DocumentQueueFilter.cs b/source/backend/entities/Models/DocumentQueueFilter.cs index c9d43c5e61..d0925ba355 100644 --- a/source/backend/entities/Models/DocumentQueueFilter.cs +++ b/source/backend/entities/Models/DocumentQueueFilter.cs @@ -14,7 +14,7 @@ public class DocumentQueueFilter : PageFilter /// /// get/set - The status of the document in the queue, such as 'Pending'. /// - public string DocumentQueueStatusTypeCode { get; set; } + public string[] DocumentQueueStatusTypeCodes { get; set; } /// /// get/set - The date/time that processing of the document started. @@ -26,6 +26,11 @@ public class DocumentQueueFilter : PageFilter /// public DateTime? DocProcessEndDate { get; set; } + /// + /// get/set - The maximum number of times that the system has attempted to upload the document after the initial failure. + /// + public int? MaxDocProcessRetries { get; set; } + #endregion #region Constructors diff --git a/source/backend/entities/Partials/DocumentQueueStatusType.cs b/source/backend/entities/Partials/DocumentQueueStatusType.cs new file mode 100644 index 0000000000..9fd14e1cf7 --- /dev/null +++ b/source/backend/entities/Partials/DocumentQueueStatusType.cs @@ -0,0 +1,33 @@ +using System.ComponentModel.DataAnnotations.Schema; + +namespace Pims.Dal.Entities +{ + /// + /// PimsDocumentQueueStatusType class, provides an entity for the datamodel to manage document queue status types. + /// + public partial class PimsDocumentQueueStatusType : ITypeEntity + { + #region Properties + + /// + /// get/set - Primary key to identify disposition type. + /// + [NotMapped] + public string Id { get => DocumentQueueStatusTypeCode; set => DocumentQueueStatusTypeCode = value; } + #endregion + + #region Constructors + + public PimsDocumentQueueStatusType() { } + + /// + /// Create a new instance of a PimsDocumentQueueStatusType class. + /// + /// + public PimsDocumentQueueStatusType(string id) + { + Id = id; + } + #endregion + } +} diff --git a/source/backend/scheduler/Configuration/QueryProcessingDocumentsJobOptions.cs b/source/backend/scheduler/Configuration/QueryProcessingDocumentsJobOptions.cs new file mode 100644 index 0000000000..3e30f8034d --- /dev/null +++ b/source/backend/scheduler/Configuration/QueryProcessingDocumentsJobOptions.cs @@ -0,0 +1,21 @@ +namespace Pims.Scheduler.Http.Configuration +{ + /// + /// QueryProcessingDocumentsJobOptions class, provides a way to store job configuration. + /// + public class QueryProcessingDocumentsJobOptions + { + #region Properties + + /// + /// get/set - the number of queued documents to pull in a single operation - affects the number of documents that will be uploaded in a single job run. + /// + public int? BatchSize { get; set; } + + /// + /// get/set - the maximum number of minutes a document can be processing for before the upload is considered to be a failure. + /// + public int MaxProcessingMinutes { get; set; } + #endregion + } +} diff --git a/source/backend/scheduler/Configuration/UploadQueuedDocumentsJobOptions.cs b/source/backend/scheduler/Configuration/UploadQueuedDocumentsJobOptions.cs new file mode 100644 index 0000000000..6fec774e1d --- /dev/null +++ b/source/backend/scheduler/Configuration/UploadQueuedDocumentsJobOptions.cs @@ -0,0 +1,21 @@ +namespace Pims.Scheduler.Http.Configuration +{ + /// + /// UploadQueuedDocumentsJobOptions class, provides a way to store job configuration. + /// + public class UploadQueuedDocumentsJobOptions + { + #region Properties + + /// + /// get/set - the number of queued documents to pull in a single operation - affects the number of documents that will be uploaded in a single job run. + /// + public int? BatchSize { get; set; } + + /// + /// get/set - the file size, in mb, that will be processed in a single job run. + /// + public int? FileSize { get; set; } + #endregion + } +} diff --git a/source/backend/scheduler/Models/DocumentQueueResponseModel.cs b/source/backend/scheduler/Models/DocumentQueueResponseModel.cs new file mode 100644 index 0000000000..c7fc1b699f --- /dev/null +++ b/source/backend/scheduler/Models/DocumentQueueResponseModel.cs @@ -0,0 +1,11 @@ +using Pims.Api.Models.CodeTypes; + +namespace Pims.Scheduler.Models +{ + public class DocumentQueueResponseModel + { + public DocumentQueueStatusTypes DocumentQueueStatus { get; set; } + + public string Message { get; set; } + } +} diff --git a/source/backend/scheduler/Models/ScheduledTaskResponseModel.cs b/source/backend/scheduler/Models/ScheduledTaskResponseModel.cs new file mode 100644 index 0000000000..5b2e6c7631 --- /dev/null +++ b/source/backend/scheduler/Models/ScheduledTaskResponseModel.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace Pims.Scheduler.Models +{ + public class ScheduledTaskResponseModel + { + public TaskResponseStatusTypes Status { get; set; } + + public string Message { get; set; } + + public IEnumerable DocumentQueueResponses { get; set; } + } +} diff --git a/source/backend/scheduler/Models/SearchQueuedDocumentsResponseModel.cs b/source/backend/scheduler/Models/SearchQueuedDocumentsResponseModel.cs new file mode 100644 index 0000000000..f37ee8dfe4 --- /dev/null +++ b/source/backend/scheduler/Models/SearchQueuedDocumentsResponseModel.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using Pims.Api.Models.Concepts.Document; +using Pims.Api.Models.Requests.Http; + +namespace Pims.Scheduler.Models +{ + public class SearchQueuedDocumentsResponseModel + { + public ExternalResponse> SearchResults { get; set; } + + public ScheduledTaskResponseModel ScheduledTaskResponseModel { get; set; } + } +} diff --git a/source/backend/scheduler/Models/TaskResponseStatusTypes.cs b/source/backend/scheduler/Models/TaskResponseStatusTypes.cs new file mode 100644 index 0000000000..7f118cdba9 --- /dev/null +++ b/source/backend/scheduler/Models/TaskResponseStatusTypes.cs @@ -0,0 +1,10 @@ +namespace Pims.Scheduler.Models +{ + public enum TaskResponseStatusTypes + { + ERROR, + SUCCESS, + PARTIAL, + SKIPPED, + } +} diff --git a/source/backend/scheduler/Pims.Scheduler.csproj b/source/backend/scheduler/Pims.Scheduler.csproj index e4f4993886..f8553b3712 100644 --- a/source/backend/scheduler/Pims.Scheduler.csproj +++ b/source/backend/scheduler/Pims.Scheduler.csproj @@ -2,7 +2,7 @@ true - 16BC0468-78F6-4C91-87DA-7403C919E646 + {AC4336C5-5631-4D9D-B78F-6C2DF79A6F1F} net8.0 diff --git a/source/backend/scheduler/Repositories/Interfaces/IPimsDocumentRepository.cs b/source/backend/scheduler/Repositories/Interfaces/IPimsDocumentQueueRepository.cs similarity index 50% rename from source/backend/scheduler/Repositories/Interfaces/IPimsDocumentRepository.cs rename to source/backend/scheduler/Repositories/Interfaces/IPimsDocumentQueueRepository.cs index 7505e3576d..c3dd5e33e6 100644 --- a/source/backend/scheduler/Repositories/Interfaces/IPimsDocumentRepository.cs +++ b/source/backend/scheduler/Repositories/Interfaces/IPimsDocumentQueueRepository.cs @@ -4,13 +4,19 @@ using Pims.Api.Models.Requests.Http; using Pims.Dal.Entities.Models; -namespace Pims.Scheduler.Repositories.Pims +namespace Pims.Scheduler.Repositories { /// - /// IPimsDocumentQueueRepository interface, defines the functionality for a pims repository. + /// IPimsDocumentQueueRepository interface, defines the functionality for a repository that interacts with the pims document queue api. /// public interface IPimsDocumentQueueRepository { + Task> UploadQueuedDocument(DocumentQueueModel document); + + Task> PollQueuedDocument(DocumentQueueModel document); + + Task> UpdateQueuedDocument(long documentQueueId, DocumentQueueModel document); + Task>> SearchQueuedDocumentsAsync(DocumentQueueFilter filter); } } diff --git a/source/backend/scheduler/Repositories/PimsBaseRepository.cs b/source/backend/scheduler/Repositories/PimsBaseRepository.cs index 6cdccd99cc..5f822e7eb3 100644 --- a/source/backend/scheduler/Repositories/PimsBaseRepository.cs +++ b/source/backend/scheduler/Repositories/PimsBaseRepository.cs @@ -1,7 +1,9 @@ using System.Collections.Generic; using System.Net.Http; using System.Net.Http.Headers; +using System.Text.Json; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Pims.Core.Api.Repositories.Rest; namespace Pims.Scheduler.Repositories @@ -14,10 +16,12 @@ public abstract class PimsBaseRepository : BaseRestRepository /// /// Injected Logger Provider. /// Injected Httpclient factory. + /// Injected app-wide json options. protected PimsBaseRepository( ILogger logger, - IHttpClientFactory httpClientFactory) - : base(logger, httpClientFactory) + IHttpClientFactory httpClientFactory, + IOptions jsonOptions) + : base(logger, httpClientFactory, jsonOptions) { } diff --git a/source/backend/scheduler/Repositories/PimsDocumentQueueRepository.cs b/source/backend/scheduler/Repositories/PimsDocumentQueueRepository.cs new file mode 100644 index 0000000000..d9f5305ac1 --- /dev/null +++ b/source/backend/scheduler/Repositories/PimsDocumentQueueRepository.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Pims.Api.Models.Concepts.Document; +using Pims.Api.Models.Requests.Http; +using Pims.Core.Extensions; +using Pims.Core.Http; +using Pims.Dal.Entities.Models; +using Pims.Scheduler.Http.Configuration; + +namespace Pims.Scheduler.Repositories +{ + /// + /// PimsDocumentQueueRepository provides document access from the PIMS document queue api. + /// + public class PimsDocumentQueueRepository : PimsBaseRepository, IPimsDocumentQueueRepository + { + private static readonly JsonSerializerOptions SerializerOptions = new() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; + + private readonly IOpenIdConnectRequestClient _authRepository; + private readonly IOptionsMonitor _configuration; + + /// + /// Initializes a new instance of the class. + /// + /// Injected Logger Provider. + /// Injected Httpclient factory. + /// Injected repository that handles authentication. + /// The injected configuration provider. + /// The injected json options. + public PimsDocumentQueueRepository( + ILogger logger, + IHttpClientFactory httpClientFactory, + IOpenIdConnectRequestClient authRepository, + IOptionsMonitor configuration, + IOptions jsonOptions) + : base(logger, httpClientFactory, jsonOptions) + { + _authRepository = authRepository; + _configuration = configuration; + } + + /// + /// Polls the upload status in mayan of a queued document using the provided document model. + /// + /// The document to poll. + /// A task that represents the asynchronous operation. The result is an external response containing the document queue model and status information. + public async Task> PollQueuedDocument(DocumentQueueModel document) + { + _logger.LogDebug("polling queued document with id {documentId}", document.Id); + + string authenticationToken = await _authRepository.RequestAccessToken(); + + Uri endpoint = new($"{_configuration.CurrentValue.Uri}/documents/queue/{document.Id}/poll"); + + string serializedFilter = JsonSerializer.Serialize(document, SerializerOptions); + using var content = new StringContent(serializedFilter, Encoding.UTF8, "application/json"); + + var response = await PostAsync(endpoint, content, authenticationToken); + _logger.LogDebug("queued document poll for document with id {documentId} complete with status: {status} message: {message}", document.Id, response.Status, response.Message); + + return response; + } + + /// + /// Uploads a queued document to the specified endpoint. + /// + /// The document queue model containing the document details. + /// A task that represents the asynchronous operation and returns an external response containing the status of the upload. + public async Task> UploadQueuedDocument(DocumentQueueModel document) + { + _logger.LogDebug("uploading queued document with id {documentId}", document.Id); + + string authenticationToken = await _authRepository.RequestAccessToken(); + + Uri endpoint = new($"{_configuration.CurrentValue.Uri}/documents/queue/{document.Id}/upload"); + + string serializedFilter = JsonSerializer.Serialize(document, SerializerOptions); + using var content = new StringContent(serializedFilter, Encoding.UTF8, "application/json"); + + var response = await PostAsync(endpoint, content, authenticationToken); + _logger.LogDebug("queued document upload for document with id {documentId} complete with status: {status} message: {message}", document.Id, response.Status, response.Message); + + return response; + } + + /// + /// Updates an existing queued document. + /// + /// The ID of the document to update. + /// The updated document details. + /// The result of the update operation. + public async Task> UpdateQueuedDocument(long documentQueueId, DocumentQueueModel document) + { + _logger.LogDebug("updating queued document with id {documentId}", documentQueueId); + + string authenticationToken = await _authRepository.RequestAccessToken(); + + Uri endpoint = new($"{_configuration.CurrentValue.Uri}/documents/queue/{documentQueueId}"); + + string serializedFilter = JsonSerializer.Serialize(document, SerializerOptions); + using var content = new StringContent(serializedFilter, Encoding.UTF8, "application/json"); + + var response = await PutAsync(endpoint, content, authenticationToken); + _logger.LogDebug("queued document update for document with id {documentId} complete with status: {status} message: {message}", documentQueueId, response.Status, response.Message); + + return response; + } + + /// + /// Searches for queued documents based on the provided filter. + /// + /// The filter to apply to the search. + /// A task that represents the asynchronous operation, returning a list of document queue models. + public async Task>> SearchQueuedDocumentsAsync(DocumentQueueFilter filter) + { + _logger.LogDebug("Getting filtered list of queued documents by {filter}", filter); + + string authenticationToken = await _authRepository.RequestAccessToken(); + + Uri endpoint = new($"{_configuration.CurrentValue.Uri}/documents/queue/search"); + + string serializedFilter = JsonSerializer.Serialize(filter, SerializerOptions); + using var content = new StringContent(serializedFilter, Encoding.UTF8, "application/json"); + + var response = await PostAsync>(endpoint, content, authenticationToken); + _logger.LogDebug($"Retrieved list of queued documents based on {filter} ", filter.Serialize()); + + return response; + } + } +} diff --git a/source/backend/scheduler/Repositories/PimsRepository.cs b/source/backend/scheduler/Repositories/PimsRepository.cs deleted file mode 100644 index f6b2b061c6..0000000000 --- a/source/backend/scheduler/Repositories/PimsRepository.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Pims.Api.Models.Concepts.Document; -using Pims.Api.Models.Requests.Http; -using Pims.Core.Http; -using Pims.Dal.Entities.Models; -using Pims.Scheduler.Http.Configuration; - -namespace Pims.Scheduler.Repositories.Pims -{ - /// - /// PimsDocumentQueueRepository provides document access from the PIMS document queue api. - /// - public class PimsDocumentQueueRepository : PimsBaseRepository, IPimsDocumentQueueRepository - { - private readonly IOpenIdConnectRequestClient _authRepository; - private readonly IOptionsMonitor _configuration; - - /// - /// Initializes a new instance of the class. - /// - /// Injected Logger Provider. - /// Injected Httpclient factory. - /// Injected repository that handles authentication. - /// The injected configuration provider. - public PimsDocumentQueueRepository( - ILogger logger, - IHttpClientFactory httpClientFactory, - IOpenIdConnectRequestClient authRepository, - IOptionsMonitor configuration) - : base(logger, httpClientFactory) - { - _authRepository = authRepository; - _configuration = configuration; - } - - public async Task>> SearchQueuedDocumentsAsync(DocumentQueueFilter filter) - { - _logger.LogDebug("Getting filtered list of queued documents by {filter}", filter); - - string authenticationToken = await _authRepository.RequestAccessToken(); - - Uri endpoint = new($"{_configuration.CurrentValue.Uri}/documents/queue/search"); - - var response = await GetAsync>(endpoint, authenticationToken); - _logger.LogDebug($"Retrieved list of queued documents based on {filter} ", filter); - - return response; - } - } -} diff --git a/source/backend/scheduler/Scheduler/JobRescheduler.cs b/source/backend/scheduler/Scheduler/JobRescheduler.cs index 791942711c..03e480f4ec 100644 --- a/source/backend/scheduler/Scheduler/JobRescheduler.cs +++ b/source/backend/scheduler/Scheduler/JobRescheduler.cs @@ -42,16 +42,10 @@ public void LoadSchedules(JobScheduleOptions options) throw new ConfigurationException($"Unable to find TimeZoneInfo : {timezoneId}"); } - var cron = scheduling.Cron; - if (cron == null) - { - throw new ConfigurationException($"Cron is required"); - } - _recurringJobManager.AddOrUpdate( recurringJob.Id, recurringJob.Job, - scheduling.Cron, + scheduling.Cron ?? recurringJob.Cron, new RecurringJobOptions() { TimeZone = timezone }); } } diff --git a/source/backend/scheduler/Services/DocumentQueueService.cs b/source/backend/scheduler/Services/DocumentQueueService.cs index 6e2c2a01c3..55f50bfe76 100644 --- a/source/backend/scheduler/Services/DocumentQueueService.cs +++ b/source/backend/scheduler/Services/DocumentQueueService.cs @@ -1,8 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; +using Hangfire; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Pims.Api.Models.CodeTypes; +using Pims.Api.Models.Concepts.Document; +using Pims.Api.Models.Requests.Http; using Pims.Core.Api.Services; -using Pims.Scheduler.Repositories.Pims; +using Pims.Core.Extensions; +using Pims.Dal.Entities.Models; +using Pims.Scheduler.Http.Configuration; +using Pims.Scheduler.Models; +using Pims.Scheduler.Repositories; namespace Pims.Scheduler.Services { @@ -10,20 +21,154 @@ public class DocumentQueueService : BaseService, IDocumentQueueService { private readonly ILogger _logger; private readonly IPimsDocumentQueueRepository _pimsDocumentQueueRepository; + private readonly IOptionsMonitor _uploadQueuedDocumentsJobOptions; + private readonly IOptionsMonitor _queryProcessingDocumentsJobOptions; public DocumentQueueService( ILogger logger, + IOptionsMonitor uploadQueuedDocumentsJobOptions, + IOptionsMonitor queryProcessingDocumentsJobOptions, IPimsDocumentQueueRepository pimsDocumentQueueRepository) : base(null, logger) { _logger = logger; _pimsDocumentQueueRepository = pimsDocumentQueueRepository; + _uploadQueuedDocumentsJobOptions = uploadQueuedDocumentsJobOptions; + _queryProcessingDocumentsJobOptions = queryProcessingDocumentsJobOptions; } - public async Task UploadQueuedDocuments() + [DisableConcurrentExecution(timeoutInSeconds: 10 * 30)] + public async Task UploadQueuedDocuments() { - var queuedDocuments = await _pimsDocumentQueueRepository.SearchQueuedDocumentsAsync(new Dal.Entities.Models.DocumentQueueFilter() { Quantity = 50, DocumentQueueStatusTypeCode = DocumentQueueStatusTypes.PENDING.ToString() }); - _logger.LogInformation("retrieved {queuedDocuments} documents", queuedDocuments?.Payload?.Count); + var filter = new DocumentQueueFilter() { Quantity = _uploadQueuedDocumentsJobOptions.CurrentValue.BatchSize ?? 50, DocumentQueueStatusTypeCodes = new string[] { DocumentQueueStatusTypes.PENDING.ToString() } }; + var searchResponse = await SearchQueuedDocuments(filter); + if (searchResponse.ScheduledTaskResponseModel != null) + { + return searchResponse.ScheduledTaskResponseModel; + } + + IEnumerable> responses = searchResponse.SearchResults.Payload.Select(qd => + { + _logger.LogInformation("Uploading Queued document {documentQueueId}", qd.Id); + _logger.LogDebug("document contents {document}", qd.Serialize()); + + return _pimsDocumentQueueRepository.UploadQueuedDocument(qd).ContinueWith(response => HandleDocumentQueueResponse("UploadQueuedDocument", qd, response)); + }); + var results = await Task.WhenAll(responses); + return new ScheduledTaskResponseModel() { Status = GetMergedStatus(results), DocumentQueueResponses = results }; + } + + [DisableConcurrentExecution(timeoutInSeconds: 10 * 30)] + public async Task RetryQueuedDocuments() + { + var filter = new DocumentQueueFilter() + { + Quantity = _uploadQueuedDocumentsJobOptions.CurrentValue.BatchSize ?? 50, + DocumentQueueStatusTypeCodes = new string[] { DocumentQueueStatusTypes.PIMS_ERROR.ToString(), DocumentQueueStatusTypes.MAYAN_ERROR.ToString() }, + MaxDocProcessRetries = 3, + }; + var searchResponse = await SearchQueuedDocuments(filter); + if (searchResponse.ScheduledTaskResponseModel != null) + { + return searchResponse.ScheduledTaskResponseModel; + } + IEnumerable> responses = searchResponse.SearchResults.Payload.Select(qd => + { + _logger.LogInformation("Uploading Queued document {documentQueueId}", qd.Id); + _logger.LogDebug("document contents {document}", qd.Serialize()); + return _pimsDocumentQueueRepository.UploadQueuedDocument(qd).ContinueWith(response => HandleDocumentQueueResponse("UploadQueuedDocument", qd, response)); + }); + var results = await Task.WhenAll(responses); + return new ScheduledTaskResponseModel() { Status = GetMergedStatus(results), DocumentQueueResponses = results }; + } + + [DisableConcurrentExecution(timeoutInSeconds: 10 * 30)] + public async Task QueryProcessingDocuments() + { + var filter = new Dal.Entities.Models.DocumentQueueFilter() + { + Quantity = _queryProcessingDocumentsJobOptions.CurrentValue.BatchSize ?? 50, + DocumentQueueStatusTypeCodes = new string[] { DocumentQueueStatusTypes.PROCESSING.ToString() }, + }; + var searchResponse = await SearchQueuedDocuments(filter); + if (searchResponse.ScheduledTaskResponseModel != null) + { + return searchResponse.ScheduledTaskResponseModel; + } + + IEnumerable> responses = searchResponse.SearchResults.Payload.Select(qd => + { + _logger.LogInformation("Querying for queued document {documentQueueId}", qd.Id); + _logger.LogDebug("document contents {document}", qd.Serialize()); + if (qd.DocumentProcessStartTimestamp.HasValue && DateTime.UtcNow.Subtract(qd.DocumentProcessStartTimestamp.Value).TotalMinutes > _queryProcessingDocumentsJobOptions?.CurrentValue?.MaxProcessingMinutes) + { + _logger.LogError("Document processing for document {documentQueueId} has exceeded maximum processing time of {maxProcessingMinutes}", qd.Id, _queryProcessingDocumentsJobOptions?.CurrentValue?.MaxProcessingMinutes); + qd.DocumentQueueStatusType.Id = DocumentQueueStatusTypes.MAYAN_ERROR.ToString(); + qd.DocumentProcessEndTimestamp = DateTime.UtcNow; + _ = _pimsDocumentQueueRepository.UpdateQueuedDocument(qd.Id, qd).ContinueWith(response => + { + _logger.LogInformation("Received response from PIMS document update for queued document {documentQueueId} status {Status} message: {Message}", qd.Id, response?.Result?.Status, response?.Result?.Message); + }); + return Task.FromResult(new DocumentQueueResponseModel() { DocumentQueueStatus = DocumentQueueStatusTypes.PIMS_ERROR, Message = $"Document processing for document {qd.Id} has exceeded maximum processing time of {_queryProcessingDocumentsJobOptions?.CurrentValue?.MaxProcessingMinutes}" }); + } + else + { + return _pimsDocumentQueueRepository.PollQueuedDocument(qd).ContinueWith(response => HandleDocumentQueueResponse("PollQueuedDocument", qd, response)); + } + }); + var results = await Task.WhenAll(responses); + return new ScheduledTaskResponseModel() { Status = GetMergedStatus(results), DocumentQueueResponses = results }; + } + + private static TaskResponseStatusTypes GetMergedStatus(IEnumerable responses) + { + if (responses.All(r => r.DocumentQueueStatus == DocumentQueueStatusTypes.SUCCESS)) + { + return TaskResponseStatusTypes.SUCCESS; + } + else if (responses.All(r => r.DocumentQueueStatus == DocumentQueueStatusTypes.MAYAN_ERROR || r.DocumentQueueStatus == DocumentQueueStatusTypes.PIMS_ERROR)) + { + return TaskResponseStatusTypes.ERROR; + } + return TaskResponseStatusTypes.PARTIAL; + } + + private async Task SearchQueuedDocuments(DocumentQueueFilter filter) + { + ScheduledTaskResponseModel scheduledTaskResponseModel = null; + var queuedDocuments = await _pimsDocumentQueueRepository.SearchQueuedDocumentsAsync(filter); + + if (queuedDocuments.Status != ExternalResponseStatus.Success) + { + _logger.LogError("Received error status from pims document queue search service, aborting. {status} {message}", queuedDocuments.Status, queuedDocuments.Message); + scheduledTaskResponseModel = new ScheduledTaskResponseModel() { Status = TaskResponseStatusTypes.ERROR, Message = "Received error status from pims document queue service, aborting." }; + } + if (queuedDocuments.Payload.Count == 0) + { + _logger.LogInformation("No documents to process, skipping execution."); + scheduledTaskResponseModel = new ScheduledTaskResponseModel() { Status = TaskResponseStatusTypes.SKIPPED, Message = "No documents to process, skipping execution." }; + } + return new SearchQueuedDocumentsResponseModel() { ScheduledTaskResponseModel = scheduledTaskResponseModel, SearchResults = queuedDocuments }; + } + + private DocumentQueueResponseModel HandleDocumentQueueResponse(string httpMethodName, DocumentQueueModel qd, Task> response) + { + var responseObject = response.Result; + if (responseObject.Status == ExternalResponseStatus.Success && (responseObject?.Payload?.DocumentQueueStatusType?.Id == DocumentQueueStatusTypes.PROCESSING.ToString() || responseObject?.Payload?.DocumentQueueStatusType?.Id == DocumentQueueStatusTypes.SUCCESS.ToString())) + { + _logger.LogInformation("Received response from {httpMethodName} for queued document {documentQueueId} status {Status} message: {Message}", httpMethodName, qd.Id, response?.Result?.Status, response?.Result?.Message); + return new DocumentQueueResponseModel() { DocumentQueueStatus = DocumentQueueStatusTypes.SUCCESS }; + } + else if (responseObject?.Payload?.DocumentQueueStatusType?.Id != DocumentQueueStatusTypes.PIMS_ERROR.ToString() && responseObject?.Payload?.DocumentQueueStatusType?.Id != DocumentQueueStatusTypes.MAYAN_ERROR.ToString()) + { + // If the poll failed, but the document is not in an error state, update the document to an error state. + _logger.LogError("Received error response from {httpMethodName} for queued document {documentQueueId} status {Status} message: {Message}", httpMethodName, qd.Id, response?.Result?.Status, response?.Result?.Message); + qd.DocumentQueueStatusType.Id = DocumentQueueStatusTypes.PIMS_ERROR.ToString(); + qd.RowVersion = responseObject?.Payload?.RowVersion ?? qd.RowVersion; + _ = _pimsDocumentQueueRepository.UpdateQueuedDocument(qd.Id, qd); + return new DocumentQueueResponseModel() { DocumentQueueStatus = DocumentQueueStatusTypes.PIMS_ERROR, Message = $"Received error response from {httpMethodName} for queued document {qd.Id} status {response?.Result?.Status} message: {response?.Result?.Message}" }; + } + return new DocumentQueueResponseModel() { DocumentQueueStatus = DocumentQueueStatusTypes.PIMS_ERROR }; } } } diff --git a/source/backend/scheduler/Services/Interfaces/IDocumentQueueService.cs b/source/backend/scheduler/Services/Interfaces/IDocumentQueueService.cs index 8820b3913b..18ecbdae5c 100644 --- a/source/backend/scheduler/Services/Interfaces/IDocumentQueueService.cs +++ b/source/backend/scheduler/Services/Interfaces/IDocumentQueueService.cs @@ -1,9 +1,14 @@ using System.Threading.Tasks; +using Pims.Scheduler.Models; namespace Pims.Scheduler.Services { public interface IDocumentQueueService { - public Task UploadQueuedDocuments(); + public Task UploadQueuedDocuments(); + + public Task RetryQueuedDocuments(); + + public Task QueryProcessingDocuments(); } } diff --git a/source/backend/scheduler/Startup.cs b/source/backend/scheduler/Startup.cs index 3e2cdfe524..e5ec46b355 100644 --- a/source/backend/scheduler/Startup.cs +++ b/source/backend/scheduler/Startup.cs @@ -39,7 +39,7 @@ using Pims.Keycloak.Configuration; using Pims.Scheduler.Http.Configuration; using Pims.Scheduler.Policies; -using Pims.Scheduler.Repositories.Pims; +using Pims.Scheduler.Repositories; using Pims.Scheduler.Rescheduler; using Pims.Scheduler.Services; using Prometheus; @@ -91,7 +91,7 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddSingleton(); - services.AddSingleton(); + services.AddScoped(); services.AddSingleton(); services.AddSerilogging(this.Configuration); @@ -110,6 +110,8 @@ public void ConfigureServices(IServiceCollection services) services.Configure(this.Configuration.GetSection("OpenIdConnect")); services.Configure(this.Configuration.GetSection("Keycloak")); services.Configure(this.Configuration.GetSection("Pims:Environment")); + services.Configure(this.Configuration.GetSection("UploadQueuedDocumentsOptions")); + services.Configure(this.Configuration.GetSection("QueryProcessingDocumentsOptions")); services.AddOptions(); services.AddApiVersioning(options => { @@ -322,7 +324,9 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) private void ScheduleHangfireJobs(IServiceProvider services) { // provide default definition of all jobs. - RecurringJob.AddOrUpdate(nameof(DocumentQueueService.UploadQueuedDocuments), x => x.UploadQueuedDocuments(), Cron.Hourly); + RecurringJob.AddOrUpdate(nameof(DocumentQueueService.UploadQueuedDocuments), x => x.UploadQueuedDocuments(), Cron.Minutely); + RecurringJob.AddOrUpdate(nameof(DocumentQueueService.RetryQueuedDocuments), x => x.RetryQueuedDocuments(), "0 0 * * *"); + RecurringJob.AddOrUpdate(nameof(DocumentQueueService.QueryProcessingDocuments), x => x.QueryProcessingDocuments(), Cron.Minutely); // override scheduled jobs with configuration. JobScheduleOptions jobOptions = this.Configuration.GetSection("JobOptions").Get(); diff --git a/source/backend/scheduler/appsettings.json b/source/backend/scheduler/appsettings.json index 8bf86b8d30..2d1e945ad2 100644 --- a/source/backend/scheduler/appsettings.json +++ b/source/backend/scheduler/appsettings.json @@ -60,11 +60,18 @@ "Schedules": [ { "JobId": "UploadQueuedDocuments", - "IsEnabled": true, - "Cron": "*/5 * * * *" + "IsEnabled": true } ] }, + "UploadQueuedDocumentsOptions": { + "BatchSize": 1, + "FileSize": 100 + }, + "QueryProcessingDocumentsOptions": { + "BatchSize": 20, + "MaxProcessingMinutes": 60, + }, "AllowedHosts": "*", "ContentSecurityPolicy": { "Base": "'none'", diff --git a/source/backend/scheduler/tests/.editorconfig b/source/backend/scheduler/tests/.editorconfig deleted file mode 100644 index 3c1051119d..0000000000 --- a/source/backend/scheduler/tests/.editorconfig +++ /dev/null @@ -1,72 +0,0 @@ -# Editor configuration, see https://editorconfig.org -root = true - -[*] -end_of_line = lf -charset = utf-8 -indent_style = space -insert_final_newline = true -trim_trailing_whitespace = true - -[*.ts] -indent_size = 2 - -[*.md] -max_line_length = off -trim_trailing_whitespace = false - -[*.env] -insert_final_newline = false - -[{Makefile,**.mk}] -# Use tabs for indentation (Makefiles require tabs) -indent_style = tab - -[*.cs] -indent_size = 4 - - -# Test files -# SA1515 Single-line comment should be preceded by blank line -dotnet_diagnostic.SA1515.severity = none -# SA1513: Closing brace should be followed by blank line -dotnet_diagnostic.SA1513.severity = none -# SA1633 The file header is missing or not located at the top of the file -dotnet_diagnostic.SA1633.severity = none -# SA1200 Using directive should appear within a namespace declaration -dotnet_diagnostic.SA1200.severity = none -# SA1124 Do not use regions -dotnet_diagnostic.SA1124.severity = none -# SA1201 A constructor should not follow a property -dotnet_diagnostic.SA1201.severity = none -# SA1309 Field 'X' should not begin with an underscore -dotnet_diagnostic.SA1309.severity = none -# SA1117 The parameters should all be placed on the same line or each parameter should be placed on its own line. -dotnet_diagnostic.SA1117.severity = none - -# -- Set to 'error' before running formatter -# dotnet format --severity error --exclude entities/ef/** --exclude entities/PimsBaseContext.cs -# SA1208: Using directive for X should appear before directive for Y -dotnet_diagnostic.SA1208.severity = warning -# SA1121 Use built-in type alias -dotnet_diagnostic.SA1121.severity = warning -# SA1413 Use trailing comma in multi-line initializers. -dotnet_diagnostic.SA1413.severity = warning -# SA1122 Use string.Empty for empty strings -dotnet_diagnostic.SA1122.severity = warning -# SA1518 Code should not contain blank lines at the end of the file. -dotnet_diagnostic.SA1518.severity = warning -# SA1101 Prefix local calls with this -dotnet_diagnostic.SA1101.severity = warning -# SA1507 Code should not contain multiple blank lines in a row -dotnet_diagnostic.SA1507.severity = warning -# SA1127 Generic type constraints should be on their own line -dotnet_diagnostic.SA1127.severity = warning -# SA1002 Semicolons should be followed by a space. -dotnet_diagnostic.SA1002.severity = warning -# SA1009 Closing parenthesis should not be preceded by a space. -dotnet_diagnostic.SA1009.severity = warning -# SA1508 A closing brace should not be preceded by a blank line -dotnet_diagnostic.SA1508.severity = warning -# SA1005 Single line comment should begin with a space. -dotnet_diagnostic.SA1005.severity = warning \ No newline at end of file diff --git a/source/backend/scheduler/tests/.gitignore b/source/backend/scheduler/tests/.gitignore deleted file mode 100644 index 5923334c0e..0000000000 --- a/source/backend/scheduler/tests/.gitignore +++ /dev/null @@ -1,49 +0,0 @@ -# Environment variables -.env -# Build -.obj - -*.swp -*.*~ -project.lock.json -.DS_Store -*.pyc -nupkg/ - -# IDE - VSCode -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json - -# Rider -.idea - -# User-specific files -*.suo -*.user -*.userosscache -*.sln.docstates - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -build/ -bld/ -[Bb]in/ -[Oo]bj/ -[Oo]ut/ -msbuild.log -msbuild.err -msbuild.wrn - -# Visual Studio 2015 -.vs/ - -# SonarQube -.sonarqube/ diff --git a/source/backend/scheduler/tests/Directory.Build.props b/source/backend/scheduler/tests/Directory.Build.props deleted file mode 100644 index 4357d5e289..0000000000 --- a/source/backend/scheduler/tests/Directory.Build.props +++ /dev/null @@ -1,9 +0,0 @@ - - - net8.0 - 9.0 - - - - - diff --git a/source/backend/tests/core/Entities/DocumentQueueHelper.cs b/source/backend/tests/core/Entities/DocumentQueueHelper.cs new file mode 100644 index 0000000000..58d77d611c --- /dev/null +++ b/source/backend/tests/core/Entities/DocumentQueueHelper.cs @@ -0,0 +1,38 @@ +using System; +using Entity = Pims.Dal.Entities; + +namespace Pims.Core.Test +{ + /// + /// EntityHelper static class, provides helper methods to create test entities. + /// + public static partial class EntityHelper + { + /// + /// Create a new instance of a DocumentQueue. + /// + /// the document queue id. + /// the status of the queued document. + /// the source of the queued document. + /// the filled-out test entity. + public static Entity.PimsDocumentQueue CreateDocumentQueue(long id = 1, string status = "Pending", string dataSourceTypeCd = "PIMS") + { + return new Entity.PimsDocumentQueue() + { + DocumentQueueId = id, + DocumentQueueStatusTypeCode = status, + AppCreateTimestamp = DateTime.Now, + AppCreateUserid = "admin", + AppLastUpdateTimestamp = DateTime.Now, + AppLastUpdateUserid = "admin", + DocumentId = id, + DocumentNavigation = CreateDocument(id: id), + Document = new byte[] { 1, 2, 3 }, + FileName = $"test-{id}.pdf", + DataSourceTypeCode = "PIMS", + DataSourceTypeCodeNavigation = new Entity.PimsDataSourceType() { Id = dataSourceTypeCd ?? $"PIMS-{id}", DbCreateUserid = "test", DbLastUpdateUserid = "test", Description = "desc" }, + DocumentQueueStatusTypeCodeNavigation = new Entity.PimsDocumentQueueStatusType() { Id = status ?? $"PENDING-{id}", DbCreateUserid = "test", DbLastUpdateUserid = "test", Description = "desc" }, + }; + } + } +} diff --git a/source/backend/tests/unit/api/Services/DocumentFileServiceTest.cs b/source/backend/tests/unit/api/Services/DocumentFileServiceTest.cs index da9d935eb4..ae6ce67f05 100644 --- a/source/backend/tests/unit/api/Services/DocumentFileServiceTest.cs +++ b/source/backend/tests/unit/api/Services/DocumentFileServiceTest.cs @@ -310,6 +310,12 @@ public async void UploadDocumentAsync_Project_Success() { Id = 1, }, + DocumentExternalResponse = new() + { + Message = "Mayan test error", + Status = ExternalResponseStatus.Success, + Payload = new Models.Mayan.Document.DocumentDetailModel() { Id = 1 }, + } }); // Act @@ -319,6 +325,7 @@ public async void UploadDocumentAsync_Project_Success() DocumentTypeId = 4, File = this._helper.GetFormFile("Lorem Ipsum"), DocumentStatusCode = "DocumentStatus", + DocumentId = 1, }; await service.UploadProjectDocumentAsync(1, uploadRequest); @@ -406,6 +413,12 @@ public async void UploadDocumentAsync_Acquisition_Success() { Id = 1, }, + DocumentExternalResponse = new() + { + Message = "Mayan test error", + Status = ExternalResponseStatus.Success, + Payload = new Models.Mayan.Document.DocumentDetailModel() { Id = 1 }, + } }); // Act @@ -415,6 +428,7 @@ public async void UploadDocumentAsync_Acquisition_Success() DocumentTypeId = 4, File = this._helper.GetFormFile("Lorem Ipsum"), DocumentStatusCode = "DocumentStatus", + DocumentId = 1, }; await service.UploadAcquisitionDocumentAsync(1, uploadRequest); @@ -502,6 +516,12 @@ public async void UploadDocumentAsync_Research_Success() { Id = 1, }, + DocumentExternalResponse = new() + { + Message = "Mayan test error", + Status = ExternalResponseStatus.Success, + Payload = new Models.Mayan.Document.DocumentDetailModel() { Id = 1 }, + } }); // Act @@ -511,6 +531,7 @@ public async void UploadDocumentAsync_Research_Success() DocumentTypeId = 4, File = this._helper.GetFormFile("Lorem Ipsum"), DocumentStatusCode = "DocumentStatus", + DocumentId = 1, }; await service.UploadResearchDocumentAsync(1, uploadRequest); @@ -598,6 +619,13 @@ public async void UploadDocumentAsync_Lease_Success() { Id = 1, }, + DocumentExternalResponse = new() + { + Message = "Mayan test error", + Status = ExternalResponseStatus.Success, + Payload = new Models.Mayan.Document.DocumentDetailModel() { Id = 1 }, + + } }); // Act @@ -607,6 +635,7 @@ public async void UploadDocumentAsync_Lease_Success() DocumentTypeId = 4, File = this._helper.GetFormFile("Lorem Ipsum"), DocumentStatusCode = "DocumentStatus", + DocumentId = 1, }; await service.UploadLeaseDocumentAsync(1, uploadRequest); @@ -694,6 +723,12 @@ public async void UploadDocumentAsync_PropertyActivity_Success() { Id = 1, }, + DocumentExternalResponse = new() + { + Message = "Mayan test error", + Status = ExternalResponseStatus.Success, + Payload = new Models.Mayan.Document.DocumentDetailModel() { Id = 1 }, + } }); // Act @@ -703,6 +738,7 @@ public async void UploadDocumentAsync_PropertyActivity_Success() DocumentTypeId = 4, File = this._helper.GetFormFile("Lorem Ipsum"), DocumentStatusCode = "DocumentStatus", + DocumentId = 1, }; await service.UploadPropertyActivityDocumentAsync(1, uploadRequest); @@ -790,6 +826,12 @@ public async void UploadDocumentAsync_Disposition_Success() { Id = 1, }, + DocumentExternalResponse = new() + { + Message = "Mayan test error", + Status = ExternalResponseStatus.Success, + Payload = new Models.Mayan.Document.DocumentDetailModel() { Id = 1 }, + } }); dispositionFileDocumentRepository.Setup(x => x.AddDispositionDocument(It.IsAny())) @@ -802,6 +844,7 @@ public async void UploadDocumentAsync_Disposition_Success() DocumentTypeId = 4, File = this._helper.GetFormFile("Lorem Ipsum"), DocumentStatusCode = "DocumentStatus", + DocumentId = 1, }; var result = await service.UploadDispositionDocumentAsync(100, uploadRequest); diff --git a/source/backend/tests/unit/api/Services/DocumentQueueServiceTest.cs b/source/backend/tests/unit/api/Services/DocumentQueueServiceTest.cs new file mode 100644 index 0000000000..b97c52c7cb --- /dev/null +++ b/source/backend/tests/unit/api/Services/DocumentQueueServiceTest.cs @@ -0,0 +1,681 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Security.Claims; +using System.Text.Json; +using System.Threading.Tasks; +using Azure; +using FluentAssertions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Pims.Api.Models.CodeTypes; +using Pims.Api.Models.Concepts.Document; +using Pims.Api.Models.Mayan.Document; +using Pims.Api.Models.Requests.Document.Upload; +using Pims.Api.Models.Requests.Http; +using Pims.Api.Services; +using Pims.Core.Api.Exceptions; +using Pims.Core.Exceptions; +using Pims.Core.Http.Configuration; +using Pims.Core.Security; +using Pims.Core.Test; +using Pims.Dal.Entities; +using Pims.Dal.Entities.Models; +using Pims.Dal.Repositories; +using Xunit; + +namespace Pims.Api.Test.Services +{ + [Trait("category", "unit")] + [Trait("category", "api")] + [Trait("group", "documentQueues")] + [ExcludeFromCodeCoverage] + public class DocumentQueueServiceTest + { + private TestHelper _helper; + + public DocumentQueueServiceTest() + { + this._helper = new TestHelper(); + } + + private DocumentQueueService CreateDocumentQueueServiceWithPermissions(params Permissions[] permissions) + { + var user = PrincipalHelper.CreateForPermission(permissions); + this._helper.CreatePimsContext(user, true); + var builder = new ConfigurationBuilder(); + return this._helper.Create(builder.Build()); + } + + [Fact] + public void SearchDocumentQueue_Success() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var filter = new DocumentQueueFilter(); + var documentQueues = new List { new PimsDocumentQueue() }; + var documentQueueRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(m => m.GetAllByFilter(filter)).Returns(documentQueues); + + // Act + var result = service.SearchDocumentQueue(filter); + + // Assert + result.Should().BeEquivalentTo(documentQueues); + documentQueueRepositoryMock.Verify(m => m.GetAllByFilter(filter), Times.Once); + } + + [Fact] + public void SearchDocumentQueue_InvalidPermissions_ThrowsNotAuthorizedException() + { + // Arrange + var filter = new DocumentQueueFilter(); + var service = CreateDocumentQueueServiceWithPermissions(); + + // Act + Action act = () => service.SearchDocumentQueue(filter); + + // Assert + act.Should().Throw(); + } + + [Fact] + public void Update_Success() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1 }; + var documentQueueRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(m => m.Update(documentQueue, false)); + documentQueueRepositoryMock.Setup(m => m.CommitTransaction()); + + // Act + var result = service.Update(documentQueue); + + // Assert + result.Should().Be(documentQueue); + documentQueueRepositoryMock.Verify(m => m.Update(documentQueue, false), Times.Once); + documentQueueRepositoryMock.Verify(m => m.CommitTransaction(), Times.Once); + } + + [Fact] + public void Update_InvalidPermissions_ThrowsNotAuthorizedException() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(); + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1 }; + + // Act + Action act = () => service.Update(documentQueue); + + // Assert + act.Should().Throw(); + } + + [Fact] + public async Task PollForDocumentDocumentIdNull_ThrowsInvalidDataException() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = null }; + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + + // Act + Func act = async () => await service.PollForDocument(documentQueue); + + // Assert + await act.Should().ThrowAsync(); + documentRepositoryMock.Verify(m => m.TryGet(It.IsAny()), Times.Never); + documentQueueRepositoryMock.Verify(m => m.TryGetById(It.IsAny()), Times.Never); + documentServiceMock.Verify(m => m.GetStorageDocumentDetail(It.IsAny()), Times.Never); + } + + [Fact] + public async Task PollForDocument_NoDatabaseDocumentQueue() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1 }; + var documentRepositoryMock = this._helper.GetService>(); + + // Act + Func act = async () => await service.PollForDocument(documentQueue); + + // Assert + await act.Should().ThrowAsync(); + documentRepositoryMock.Verify(m => m.TryGet(It.IsAny()), Times.Never); + } + + [Fact] + public async Task PollForDocument_RelatedDocumentMayanIdNull_UpdatesStatusToPIMSError() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1 }; + var relatedDocument = new PimsDocument { MayanId = null }; + var databaseDocumentQueue = new PimsDocumentQueue { DocumentQueueId = 1 }; + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + + documentRepositoryMock.Setup(m => m.TryGet(documentQueue.DocumentId.Value)).Returns(relatedDocument); + documentQueueRepositoryMock.Setup(m => m.TryGetById(documentQueue.DocumentQueueId)).Returns(databaseDocumentQueue); + + // Act + var result = await service.PollForDocument(documentQueue); + + // Assert + result.Should().Be(databaseDocumentQueue); + documentQueueRepositoryMock.Verify(m => m.Update(databaseDocumentQueue, false), Times.Once); + documentQueueRepositoryMock.Verify(m => m.CommitTransaction(), Times.Once); + } + + [Fact] + public async Task PollForDocument_GetStorageDocumentDetailFails_UpdatesStatusToPIMSError() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1 }; + var relatedDocument = new PimsDocument { MayanId = 1 }; + var databaseDocumentQueue = new PimsDocumentQueue { DocumentQueueId = 1 }; + var documentDetailsResponse = new ExternalResponse { Status = ExternalResponseStatus.Error }; + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + + documentRepositoryMock.Setup(m => m.TryGet(documentQueue.DocumentId.Value)).Returns(relatedDocument); + documentQueueRepositoryMock.Setup(m => m.TryGetById(documentQueue.DocumentQueueId)).Returns(databaseDocumentQueue); + documentServiceMock.Setup(m => m.GetStorageDocumentDetail(relatedDocument.MayanId.Value)).ReturnsAsync(documentDetailsResponse); + + // Act + var result = await service.PollForDocument(documentQueue); + + // Assert + result.Should().Be(databaseDocumentQueue); + documentQueueRepositoryMock.Verify(m => m.Update(databaseDocumentQueue, false), Times.Once); + documentQueueRepositoryMock.Verify(m => m.CommitTransaction(), Times.Once); + } + + [Fact] + public async Task PollForDocument_FileLatestIdNull_LogsFileStillProcessing() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1 }; + var relatedDocument = new PimsDocument { MayanId = 1 }; + var databaseDocumentQueue = new PimsDocumentQueue { DocumentQueueId = 1 }; + var documentDetailModel = new DocumentDetailModel { FileLatest = null }; + var documentDetailsResponse = new ExternalResponse { Status = ExternalResponseStatus.Success, Payload = documentDetailModel }; + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + + documentRepositoryMock.Setup(m => m.TryGet(documentQueue.DocumentId.Value)).Returns(relatedDocument); + documentQueueRepositoryMock.Setup(m => m.TryGetById(documentQueue.DocumentQueueId)).Returns(databaseDocumentQueue); + documentServiceMock.Setup(m => m.GetStorageDocumentDetail(relatedDocument.MayanId.Value)).ReturnsAsync(documentDetailsResponse); + + // Act + var result = await service.PollForDocument(documentQueue); + + // Assert + result.Should().Be(databaseDocumentQueue); + documentQueueRepositoryMock.Verify(m => m.Update(databaseDocumentQueue, false), Times.Never); + documentQueueRepositoryMock.Verify(m => m.CommitTransaction(), Times.Never); + } + + [Fact] + public async Task PollForDocument_FileLatestIdNotNull_UpdatesStatusToSuccess() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1 }; + var relatedDocument = new PimsDocument { MayanId = 1 }; + var databaseDocumentQueue = new PimsDocumentQueue { DocumentQueueId = 1 }; + var documentDetailModel = new DocumentDetailModel { FileLatest = new FileLatestModel { Id = 1 } }; + var documentDetailsResponse = new ExternalResponse { Status = ExternalResponseStatus.Success, Payload = documentDetailModel }; + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + + documentRepositoryMock.Setup(m => m.TryGet(documentQueue.DocumentId.Value)).Returns(relatedDocument); + documentQueueRepositoryMock.Setup(m => m.TryGetById(documentQueue.DocumentQueueId)).Returns(databaseDocumentQueue); + documentServiceMock.Setup(m => m.GetStorageDocumentDetail(relatedDocument.MayanId.Value)).ReturnsAsync(documentDetailsResponse); + + // Act + var result = await service.PollForDocument(documentQueue); + + // Assert + result.Should().Be(databaseDocumentQueue); + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.SUCCESS.ToString()); + documentQueueRepositoryMock.Verify(m => m.Update(databaseDocumentQueue, true), Times.Once); + documentQueueRepositoryMock.Verify(m => m.CommitTransaction(), Times.Once); + } + + [Fact] + public async Task Upload_Success() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue + { + DocumentQueueId = 1, + DocumentId = 1, + DocumentQueueStatusTypeCode = DocumentQueueStatusTypes.PENDING.ToString(), + Document = new byte[] { 1, 2, 3 }, + DocProcessRetries = 0, + AcquisitionFileDocumentId = 1 + }; + + var relatedDocument = new PimsDocument + { + DocumentId = 1, + DocumentTypeId = 1, + FileName = "test.pdf", + DocumentStatusTypeCode = "STATUS", + MayanId = null + }; + + var documentType = new PimsDocumentTyp + { + DocumentTypeId = 1, + MayanId = 1 + }; + + var documentUploadResponse = new DocumentUploadResponse + { + DocumentExternalResponse = new ExternalResponse + { + Status = ExternalResponseStatus.Success, + Payload = new DocumentDetailModel + { + FileLatest = new FileLatestModel + { + Id = 1 + } + } + }, + MetadataExternalResponse = new List>() + }; + + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + var documentTypeRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(documentQueue); + documentRepositoryMock.Setup(x => x.TryGetDocumentRelationships(It.IsAny())).Returns(relatedDocument); + documentTypeRepositoryMock.Setup(x => x.GetById(It.IsAny())).Returns(documentType); + documentServiceMock.Setup(x => x.UploadDocumentAsync(It.IsAny())).ReturnsAsync(documentUploadResponse); + + // Act + var result = await service.Upload(documentQueue); + + // Assert + result.Should().NotBeNull(); + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.SUCCESS.ToString()); + documentQueueRepositoryMock.Verify(x => x.Update(It.IsAny(), It.IsAny()), Times.AtLeastOnce); + documentQueueRepositoryMock.Verify(x => x.CommitTransaction(), Times.AtLeastOnce); + documentServiceMock.Verify(x => x.UploadDocumentAsync(It.IsAny()), Times.Once); + } + + [Fact] + public async Task Upload_Retry_Success() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue + { + DocumentQueueId = 1, + DocumentId = 1, + DocumentQueueStatusTypeCode = DocumentQueueStatusTypes.PIMS_ERROR.ToString(), + Document = new byte[] { 1, 2, 3 }, + DocProcessRetries = 0, + AcquisitionFileDocumentId = 1 + }; + + var relatedDocument = new PimsDocument + { + DocumentId = 1, + DocumentTypeId = 1, + FileName = "test.pdf", + DocumentStatusTypeCode = "STATUS", + MayanId = null + }; + + var documentType = new PimsDocumentTyp + { + DocumentTypeId = 1, + MayanId = 1 + }; + + var documentUploadResponse = new DocumentUploadResponse + { + DocumentExternalResponse = new ExternalResponse + { + Status = ExternalResponseStatus.Success, + Payload = new DocumentDetailModel + { + FileLatest = new FileLatestModel + { + Id = 1 + } + } + }, + MetadataExternalResponse = new List>() + }; + + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + var documentTypeRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(documentQueue); + documentRepositoryMock.Setup(x => x.TryGetDocumentRelationships(It.IsAny())).Returns(relatedDocument); + documentTypeRepositoryMock.Setup(x => x.GetById(It.IsAny())).Returns(documentType); + documentServiceMock.Setup(x => x.UploadDocumentAsync(It.IsAny())).ReturnsAsync(documentUploadResponse); + + // Act + var result = await service.Upload(documentQueue); + + // Assert + result.Should().NotBeNull(); + result.DocProcessRetries.Should().Be(1); + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.SUCCESS.ToString()); + documentQueueRepositoryMock.Verify(x => x.Update(It.IsAny(), It.IsAny()), Times.AtLeastOnce); + documentQueueRepositoryMock.Verify(x => x.CommitTransaction(), Times.AtLeastOnce); + documentServiceMock.Verify(x => x.UploadDocumentAsync(It.IsAny()), Times.Once); + } + + [Fact] + public async Task Upload_ValidateQueuedDocumentFails_UpdatesStatusToPIMSError() + { + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + // Arrange + var documentQueue = new PimsDocumentQueue + { + DocumentQueueId = 1, + DocumentId = 1, + DocumentQueueStatusTypeCode = DocumentQueueStatusTypes.PENDING.ToString(), + Document = null, + DocProcessRetries = 0, + AcquisitionFileDocumentId = 1 + }; + + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(documentQueue); + + // Act + var result = await service.Upload(documentQueue); + + // Assert + result.Should().NotBeNull(); + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.PIMS_ERROR.ToString()); + documentQueueRepositoryMock.Verify(x => x.Update(It.IsAny(), It.IsAny()), Times.AtLeastOnce); + documentQueueRepositoryMock.Verify(x => x.CommitTransaction(), Times.AtLeastOnce); + documentServiceMock.Verify(x => x.UploadDocumentAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task UploadDocumentQueueNotFound_ThrowsKeyNotFoundException() + { + // Arrange + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1 }; + var documentQueueRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns((PimsDocumentQueue)null); + + // Act + Func act = async () => await service.Upload(documentQueue); + + // Assert + await act.Should().ThrowAsync(); + documentQueueRepositoryMock.Verify(x => x.TryGetById(It.IsAny()), Times.Once); + } + + [Fact] + public async Task Upload_ValidationFails_NoDocument() + { + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + // Arrange + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1 }; + + var documentQueueRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(documentQueue); + + // Act + var result = await service.Upload(documentQueue); + + // Assert + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.PIMS_ERROR.ToString()); + documentQueueRepositoryMock.Verify(x => x.TryGetById(It.IsAny()), Times.Once); + } + + [Fact] + public async Task Upload_ValidationFails_Status() + { + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + // Arrange + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1, Document = new byte[] { 1, 2, 3 }, DocumentQueueStatusTypeCode = DocumentQueueStatusTypes.SUCCESS.ToString() }; + + var documentQueueRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1, Document = new byte[] { 1, 2, 3 }, DocumentQueueStatusTypeCode = DocumentQueueStatusTypes.PIMS_ERROR.ToString() }); + + // Act + var result = await service.Upload(documentQueue); + + // Assert + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.PIMS_ERROR.ToString()); + documentQueueRepositoryMock.Verify(x => x.TryGetById(It.IsAny()), Times.Once); + } + + [Fact] + public async Task Upload_ValidationFails_Retries() + { + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + // Arrange + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1, Document = new byte[] { 1, 2, 3 }, DocProcessRetries = 10 }; + + var documentQueueRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1, Document = new byte[] { 1, 2, 3 }, DocProcessRetries = 0 }); + + // Act + var result = await service.Upload(documentQueue); + + // Assert + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.PIMS_ERROR.ToString()); + documentQueueRepositoryMock.Verify(x => x.TryGetById(It.IsAny()), Times.Once); + } + + [Fact] + public async Task Upload_RelatedDocument_MayanId() + { + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + // Arrange + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1, Document = new byte[] { 1, 2, 3 } }; + + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(documentQueue); + documentRepositoryMock.Setup(x => x.TryGetDocumentRelationships(It.IsAny())).Returns(new PimsDocument() { DocumentTypeId = 1, MayanId = 1 }); + + // Act + var result = await service.Upload(documentQueue); + + // Assert + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.SUCCESS.ToString()); + documentQueueRepositoryMock.Verify(x => x.TryGetById(It.IsAny()), Times.Once); + documentRepositoryMock.Verify(x => x.TryGetDocumentRelationships(It.IsAny()), Times.Once); + } + + [Fact] + public async Task UploadDocumentTypeNotFound_ThrowsKeyNotFoundException() + { + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + // Arrange + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1, Document = new byte[] { 1, 2, 3 } }; + var relatedDocument = new PimsDocument { DocumentId = 1, DocumentTypeId = 1 }; + + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + var documentTypeRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(documentQueue); + documentRepositoryMock.Setup(x => x.TryGetDocumentRelationships(It.IsAny())).Returns(relatedDocument); + documentTypeRepositoryMock.Setup(x => x.GetById(It.IsAny())).Throws(); + + // Act + var response = await service.Upload(documentQueue); + + // Assert + response.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.PIMS_ERROR.ToString()); + documentQueueRepositoryMock.Verify(x => x.TryGetById(It.IsAny()), Times.Once); + documentRepositoryMock.Verify(x => x.TryGetDocumentRelationships(It.IsAny()), Times.Once); + documentTypeRepositoryMock.Verify(x => x.GetById(It.IsAny()), Times.Once); + } + + [Fact] + public async Task UploadDocument_ThrowsJsonException() + { + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + // Arrange + var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1, Document = new byte[] { 1, 2, 3 } }; + var relatedDocument = new PimsDocument { DocumentId = 1, DocumentTypeId = 1 }; + + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + var documentTypeRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(documentQueue); + documentRepositoryMock.Setup(x => x.TryGetDocumentRelationships(It.IsAny())).Returns(relatedDocument); + documentTypeRepositoryMock.Setup(x => x.GetById(It.IsAny())).Throws(); + documentServiceMock.Setup(x => x.UploadDocumentAsync(It.IsAny())).ThrowsAsync(new JsonException("test error")); + + // Act + var response = await service.Upload(documentQueue); + + // Assert + response.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.PIMS_ERROR.ToString()); + documentQueueRepositoryMock.Verify(x => x.TryGetById(It.IsAny()), Times.Once); + documentRepositoryMock.Verify(x => x.TryGetDocumentRelationships(It.IsAny()), Times.Once); + documentTypeRepositoryMock.Verify(x => x.GetById(It.IsAny()), Times.Once); + } + + [Fact] + public async Task Upload_DocumentTypeIdNull() + { + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + // Arrange + var documentQueue = new PimsDocumentQueue + { + DocumentQueueId = 1, + DocumentId = 1, + DocumentQueueStatusTypeCode = DocumentQueueStatusTypes.PENDING.ToString(), + Document = new byte[] { 1, 2, 3 }, + DocProcessRetries = 0, + AcquisitionFileDocumentId = 1 + }; + + var relatedDocument = new PimsDocument + { + DocumentId = 1, + DocumentTypeId = 1, + FileName = "test.pdf", + DocumentStatusTypeCode = "STATUS", + MayanId = null + }; + + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(documentQueue); + documentRepositoryMock.Setup(x => x.TryGetDocumentRelationships(It.IsAny())).Returns((PimsDocument)null); + + // Act + var result = await service.Upload(documentQueue); + + // Assert + result.Should().NotBeNull(); + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.PIMS_ERROR.ToString()); + documentQueueRepositoryMock.Verify(x => x.Update(It.IsAny(), It.IsAny()), Times.AtLeastOnce); + documentQueueRepositoryMock.Verify(x => x.CommitTransaction(), Times.AtLeastOnce); + } + + [Fact] + public async Task Upload_UploadDocumentFails_UpdatesStatusToMayanError() + { + var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin); + // Arrange + var documentQueue = new PimsDocumentQueue + { + DocumentQueueId = 1, + DocumentId = 1, + DocumentQueueStatusTypeCode = DocumentQueueStatusTypes.PENDING.ToString(), + Document = new byte[] { 1, 2, 3 }, + DocProcessRetries = 0, + AcquisitionFileDocumentId = 1 + }; + + var relatedDocument = new PimsDocument + { + DocumentId = 1, + DocumentTypeId = 1, + FileName = "test.pdf", + DocumentStatusTypeCode = "STATUS", + MayanId = null + }; + + var documentType = new PimsDocumentTyp + { + DocumentTypeId = 1, + MayanId = 1 + }; + + var documentUploadResponse = new DocumentUploadResponse + { + DocumentExternalResponse = new ExternalResponse + { + Status = ExternalResponseStatus.Error + }, + MetadataExternalResponse = new List>() + }; + + var documentRepositoryMock = this._helper.GetService>(); + var documentQueueRepositoryMock = this._helper.GetService>(); + var documentServiceMock = this._helper.GetService>(); + var documentTypeRepositoryMock = this._helper.GetService>(); + + documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny())).Returns(documentQueue); + documentRepositoryMock.Setup(x => x.TryGetDocumentRelationships(It.IsAny())).Returns(relatedDocument); + documentTypeRepositoryMock.Setup(x => x.GetById(It.IsAny())).Returns(documentType); + documentServiceMock.Setup(x => x.UploadDocumentAsync(It.IsAny())).ReturnsAsync(documentUploadResponse); + + // Act + var result = await service.Upload(documentQueue); + + // Assert + result.Should().NotBeNull(); + result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.MAYAN_ERROR.ToString()); + documentQueueRepositoryMock.Verify(x => x.Update(It.IsAny(), It.IsAny()), Times.AtLeastOnce); + documentQueueRepositoryMock.Verify(x => x.CommitTransaction(), Times.AtLeastOnce); + documentServiceMock.Verify(x => x.UploadDocumentAsync(It.IsAny()), Times.Once); + } + } +} diff --git a/source/backend/tests/unit/api/Services/DocumentServiceTest.cs b/source/backend/tests/unit/api/Services/DocumentServiceTest.cs index b43525fab6..ce278d5de5 100644 --- a/source/backend/tests/unit/api/Services/DocumentServiceTest.cs +++ b/source/backend/tests/unit/api/Services/DocumentServiceTest.cs @@ -198,7 +198,6 @@ public async void UploadDocumentAsync_UploadRequest_Sucess() // Assert avService.Verify(x => x.ScanAsync(It.IsAny()), Times.Once); documentStorageRepository.Verify(x => x.TryUploadDocumentAsync(It.IsAny(), It.IsAny()), Times.Once); - documentStorageRepository.Verify(x => x.TryCreateDocumentMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } [Fact] diff --git a/source/backend/tests/unit/api/Services/FormDocumentServiceTest.cs b/source/backend/tests/unit/api/Services/FormDocumentServiceTest.cs index 7d1abef4b9..20cb089ebc 100644 --- a/source/backend/tests/unit/api/Services/FormDocumentServiceTest.cs +++ b/source/backend/tests/unit/api/Services/FormDocumentServiceTest.cs @@ -130,7 +130,7 @@ public void UploadFormDocumentTemplateAsync_Success() var result = service.UploadFormDocumentTemplateAsync(testTypeCode, testUploadRequest); // Assert - documentService.Verify(x => x.UploadDocumentAsync(testUploadRequest), Times.Once); + documentService.Verify(x => x.UploadDocumentSync(testUploadRequest), Times.Once); } [Fact] @@ -178,7 +178,7 @@ public void UploadFormDocumentTemplateAsync_DeleteIfPrevious() // Assert documentRepositoryMock.Verify(x => x.DocumentRelationshipCount(testDocumentId), Times.Once); documentServiceMock.Verify(x => x.DeleteDocumentAsync(testExistingDocument), Times.Once); - documentServiceMock.Verify(x => x.UploadDocumentAsync(testUploadRequest), Times.Once); + documentServiceMock.Verify(x => x.UploadDocumentSync(testUploadRequest), Times.Once); } [Fact] diff --git a/source/backend/tests/unit/dal/Repositories/DocumentQueueRepositoryTest.cs b/source/backend/tests/unit/dal/Repositories/DocumentQueueRepositoryTest.cs new file mode 100644 index 0000000000..05076834e0 --- /dev/null +++ b/source/backend/tests/unit/dal/Repositories/DocumentQueueRepositoryTest.cs @@ -0,0 +1,253 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using FluentAssertions; +using Pims.Core.Test; +using Pims.Dal.Entities; +using Pims.Dal.Exceptions; +using Pims.Dal.Repositories; +using Pims.Core.Security; +using Xunit; +using Pims.Core.Exceptions; +using Pims.Dal.Entities.Models; +using System.Security; +using Pims.Api.Models.CodeTypes; + +namespace Pims.Dal.Test.Repositories +{ + [Trait("category", "unit")] + [Trait("category", "dal")] + [Trait("group", "documentQueues")] + [ExcludeFromCodeCoverage] + public class DocumentQueueRepositoryTest + { + #region Constructors + public DocumentQueueRepositoryTest() { } + #endregion + + #region Tests + + #region TryGetById + [Fact] + public void TryGetById_Success() + { + // Arrange + var helper = new TestHelper(); + var user = PrincipalHelper.CreateForPermission(Permissions.DocumentView); + + var documentQueue = EntityHelper.CreateDocumentQueue(); + + var context = helper.CreatePimsContext(user, true).AddAndSaveChanges(documentQueue); + var repository = helper.CreateRepository(user); + + // Act + var result = repository.TryGetById(1); + + // Assert + result.Should().NotBeNull(); + result.Should().BeAssignableTo(); + } + #endregion + + #region Update + [Fact] + public void Update_Success() + { + // Arrange + var helper = new TestHelper(); + var user = PrincipalHelper.CreateForPermission(Permissions.DocumentEdit); + + var documentQueue = EntityHelper.CreateDocumentQueue(); + + var context = helper.CreatePimsContext(user, true).AddAndSaveChanges(documentQueue); + var repository = helper.CreateRepository(user); + + // Act + documentQueue.DocumentQueueStatusTypeCode = "Updated"; + var result = repository.Update(documentQueue); + context.CommitTransaction(); + + // Assert + context.PimsDocumentQueues.Should().HaveCount(1); + context.PimsDocumentQueues.FirstOrDefault().Document.Should().NotBeNull(); + context.PimsDocumentQueues.FirstOrDefault().DocumentQueueStatusTypeCode.Should().Be("Updated"); + } + + [Fact] + public void Update_Success_RemoveDocument() + { + // Arrange + var helper = new TestHelper(); + var user = PrincipalHelper.CreateForPermission(Permissions.DocumentEdit); + + var documentQueue = EntityHelper.CreateDocumentQueue(); + + var context = helper.CreatePimsContext(user, true).AddAndSaveChanges(documentQueue); + var repository = helper.CreateRepository(user); + + // Act + documentQueue.DocumentQueueStatusTypeCode = "Updated"; + documentQueue.Document = null; + var result = repository.Update(documentQueue, removeDocument: true); + context.CommitTransaction(); + + // Assert + context.PimsDocumentQueues.Should().HaveCount(1); + context.PimsDocumentQueues.FirstOrDefault().Document.Should().BeNull(); + context.PimsDocumentQueues.FirstOrDefault().DocumentQueueStatusTypeCode.Should().Be("Updated"); + } + + [Fact] + public void Update_Null() + { + // Arrange + var helper = new TestHelper(); + var user = PrincipalHelper.CreateForPermission(Permissions.DocumentView); + + var repository = helper.CreateRepository(user); + + // Act + Action act = () => repository.Update(null); + + // Assert + act.Should().Throw(); + } + #endregion + + #region Delete + [Fact] + public void Delete_Success() + { + // Arrange + var helper = new TestHelper(); + var user = PrincipalHelper.CreateForPermission(Permissions.DocumentView); + + var documentQueue = EntityHelper.CreateDocumentQueue(); + + var context = helper.CreatePimsContext(user, true).AddAndSaveChanges(documentQueue); + var repository = helper.CreateRepository(user); + + // Act + context.ChangeTracker.Clear(); + var result = repository.Delete(documentQueue); + context.CommitTransaction(); + + // Assert + context.PimsDocumentQueues.Should().BeEmpty(); + } + + [Fact] + public void Delete_Null() + { + // Arrange + var helper = new TestHelper(); + var user = PrincipalHelper.CreateForPermission(Permissions.DocumentView); + + var repository = helper.CreateRepository(user); + + // Act + Action act = () => repository.Delete(null); + + // Assert + act.Should().Throw(); + } + #endregion + + #region GetAllByFilter + [Theory] + [InlineData(2, null, 2, null, null, null, null, new int[] { 1, 2 })] + [InlineData(1, null, 1, null, null, null, null, new int[] { 1 })] + [InlineData(1, DocumentQueueStatusTypes.PENDING, 1, null, null, null, null, new int[] { 1 })] + [InlineData(1, DocumentQueueStatusTypes.SUCCESS, 1, null, null, null, null, new int[] { 2 })] + [InlineData(1, DocumentQueueStatusTypes.PROCESSING, 0, null, null, null, null, null)] + [InlineData(0, null, 0, null, null, null, null, null)] + [InlineData(1, null, 1, "2023-01-01", null, null, null, new int[] { 2 })] + [InlineData(1, null, 1, null, "2023-12-31", null, null, new int[] { 1 })] + [InlineData(1, null, 1, null, null, 3, null, new int[] { 1 })] + [InlineData(1, null, 1, null, null, null, "PAIMS", new int[] { 1 })] + public void GetAllByFilter_Success(int quantity, string status, int expectedCount, string startDate, string endDate, int? maxRetries, string dataSourceTypeCode, int[] expectedDocumentQueueIds) + { + // Arrange + var helper = new TestHelper(); + var user = PrincipalHelper.CreateForPermission(Permissions.DocumentView); + + var documentQueue1 = EntityHelper.CreateDocumentQueue(1, DocumentQueueStatusTypes.PENDING.ToString(), "PAIMS"); + documentQueue1.DocProcessStartDt = DateTime.Parse("2022-01-01"); + documentQueue1.DocProcessEndDt = DateTime.Parse("2023-12-31"); + + var documentQueue2 = EntityHelper.CreateDocumentQueue(2, DocumentQueueStatusTypes.SUCCESS.ToString()); + documentQueue2.DocProcessStartDt = DateTime.Parse("2023-01-01"); + documentQueue2.DocProcessEndDt = DateTime.Parse("2024-12-31"); + documentQueue2.DocProcessRetries = 4; + + var context = helper.CreatePimsContext(user, true).AddAndSaveChanges(documentQueue1, documentQueue2); + var repository = helper.CreateRepository(user); + + var filter = new DocumentQueueFilter + { + Quantity = quantity, + DocumentQueueStatusTypeCodes = status != null ? new string[] { status } : null, + DocProcessStartDate = startDate != null ? DateTime.Parse(startDate) : (DateTime?)null, + DocProcessEndDate = endDate != null ? DateTime.Parse(endDate) : (DateTime?)null, + MaxDocProcessRetries = maxRetries, + DataSourceTypeCode = dataSourceTypeCode + }; + + // Act + var result = repository.GetAllByFilter(filter); + + // Assert + result.Should().HaveCount(expectedCount); + if (expectedCount > 0) + { + result.Should().OnlyContain(dq => expectedDocumentQueueIds.Any(edq => edq == dq.DocumentQueueId)); + } + } + + [Fact] + public void GetAllByFilter_NoResults() + { + // Arrange + var helper = new TestHelper(); + var user = PrincipalHelper.CreateForPermission(Permissions.DocumentView); + + var context = helper.CreatePimsContext(user, true); + var repository = helper.CreateRepository(user); + + var filter = new DocumentQueueFilter { Quantity = 1 }; + + // Act + var result = repository.GetAllByFilter(filter); + + // Assert + result.Should().BeEmpty(); + } + #endregion + + #region DocumentQueueCount + [Fact] + public void DocumentQueueCount_Success() + { + // Arrange + var helper = new TestHelper(); + var user = PrincipalHelper.CreateForPermission(Permissions.DocumentView); + + var documentQueue = EntityHelper.CreateDocumentQueue(); + + var context = helper.CreatePimsContext(user, true).AddAndSaveChanges(documentQueue); + var repository = helper.CreateRepository(user); + + var statusType = new PimsDocumentQueueStatusType { DocumentQueueStatusTypeCode = documentQueue.DocumentQueueStatusTypeCode }; + + // Act + var result = repository.DocumentQueueCount(statusType); + + // Assert + result.Should().Be(1); + } + #endregion + + #endregion + } +} diff --git a/tools/mayan_sync_helper/json_output/mayan_sync.json b/tools/mayan_sync_helper/json_output/mayan_sync.json index 346d0c3c3b..088778cc97 100644 --- a/tools/mayan_sync_helper/json_output/mayan_sync.json +++ b/tools/mayan_sync_helper/json_output/mayan_sync.json @@ -1,1473 +1,1260 @@ { - "document_types": [ - { - "categories": [ - "ACQUIRE", - "MANAGEMENT" - ], - "display_order": 0, - "label": "Affidavit of service", - "purpose": "The Affidavit of Services and correspondence with the Process Server ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "AFFISERV" - }, - { - "categories": [ - "ACQUIRE" - ], - "display_order": 1, - "label": "Agricultural Land Commission (ALC)", - "purpose": "For ALC Applications, Orders, and Resolutions ", - "metadata_types": [ - { - "name": "APPLICATION_NUMBER", - "required": false - }, - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "ALC" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT" - ], - "display_order": 25, - "label": "Form 5 - Approval of expropriation", - "purpose": "The Approval of Expropriation (Form 5) for Expropriations", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "APPREXPR" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT" - ], - "display_order": 2, - "label": "Appraisals/Reviews", - "purpose": "Appraisals property in draft / final state", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "APPRREVI" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 4, - "label": "BC assessment search", - "purpose": "BC Assessment search", - "metadata_types": [ - { - "name": "CIVIC_ADDRESS", - "required": false - }, - { - "name": "JURISDICTION", - "required": false - }, - { - "name": "ROLL_NUMBER", - "required": false - }, - { - "name": "YEAR", - "required": false - } - ], - "name": "BCASSE" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 5, - "label": "Briefing notes", - "purpose": "Draft / final versions of the briefing notes", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "BRIENOTE" - }, - { - "categories": [ - "ACQUIRE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 7, - "label": "Canada lands survey", - "purpose": "Surveys registered in the Canada Land System.", - "metadata_types": [ - { - "name": "CANADA_LAND_SURVEY_NUMBER", - "required": false - }, - { - "name": "INDIAN_RESERVE_OR_NATIONAL_PARK", - "required": false - } - ], - "name": "CANALAND" - }, - { - "categories": [ - "MANAGEMENT" - ], - "display_order": 6, - "label": "CDOGS template", - "purpose": "Templates used for document generation in PIMS", - "metadata_types": [], - "name": "CDOGTEMP" - }, - { - "categories": [ - "DISPOSE", - "MANAGEMENT" - ], - "display_order": 8, - "label": "Certificate of Compliance", - "purpose": "The Certificate of Compliance (CoC)", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "CERTCOMPL" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT" - ], - "display_order": 9, - "label": "Certificate of Insurance (H0111)", - "purpose": "The Certificate of Insurance H.0111", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "CERTINSU" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT" - ], - "display_order": 11, - "label": "Compensation cheque", - "purpose": "The compensation cheque ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "COMPCHEQ" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT" - ], - "display_order": 12, - "label": "Compensation requisition (H-120)", - "purpose": "The H.120 Compensation Requisitions ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "COMPREQU" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 10, - "label": "Company search", - "purpose": "The company search", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "COMPSEAR" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 13, - "label": "Condition of entry (H0443)", - "purpose": "H.0443 Conditions of Entry ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "CONDENTR" - }, - { - "categories": [ - "DISPOSE", - "LEASLIC", - "MANAGEMENT" - ], - "display_order": 14, - "label": "Conveyance closing documents (ex: PTT forms, Form A transfer etc.)", - "purpose": "The conveyance closing documents which might include PTT, Form A Transfer, or Statement of Adjustments. ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "CONVCLOS" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 15, - "label": "Correspondence", - "purpose": "Attach letters, memos, emails (not record of negotiation, or legal correspondence)", - "metadata_types": [ - { - "name": "CIVIC_ADDRESS", - "required": false - }, - { - "name": "DATE", - "required": false - }, - { - "name": "OWNER", - "required": false - }, - { - "name": "PROPERTY_IDENTIFIER", - "required": false - }, - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "CORR" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 16, - "label": "Crown grant", - "purpose": "The crown grant application and final crown grant", - "metadata_types": [ - { - "name": "CROWN_GRANT_NUMBER", - "required": false - } - ], - "name": "CROWGRAN" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 17, - "label": "District road register", - "purpose": "The District road register ", - "metadata_types": [ - { - "name": "ELECTORAL_DISTRICT", - "required": false - }, - { - "name": "HIGHWAY_DISTRICT", - "required": false - }, - { - "name": "ROAD_NAME", - "required": false - } - ], - "name": "DISTREGI" - }, - { - "categories": [ - "DISPOSE", - "MANAGEMENT" - ], - "display_order": 18, - "label": "Enhanced Referral Records", - "purpose": "The Referral Record for both incoming and outgoing referrals.", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "ENHREFREC" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 19, - "label": "Field notes", - "purpose": "The survey field notes ", - "metadata_types": [ - { - "name": "DISTRICT_LOT_NUMBER", - "required": false - }, - { - "name": "FIELD_BOOK_NUMBER_YEAR", - "required": false - }, - { - "name": "LAND_DISTRICT", - "required": false - } - ], - "name": "FIELNOTE" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 20, - "label": "Financial records (invoices, journal vouchers, received cheques etc.)", - "purpose": "Financial records which could include invoices, JV\u2019s, contracts ect. ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "FINRECRD" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 22, - "label": "First nations consultation", - "purpose": "Record of engagement with first nations (ex: First Nations Consultation Log)", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "FIRSNTAI" - }, - { - "categories": [ - "DISPOSE", - "MANAGEMENT" - ], - "display_order": 21, - "label": "First Nations Strength of Claim Report", - "purpose": "The First Nation Strength of Claim report", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "FNSCR" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 24, - "label": "Form 12", - "purpose": "Form 12 - Certificate As To Highway In Statutory Right Of Way Plan, pre and post registration", - "metadata_types": [ - { - "name": "GAZETTE_DATE", - "required": false - }, - { - "name": "LEGAL_SURVEY_PLAN_NUMBER", - "required": false - }, - { - "name": "MOTI_PLAN_NUMBER", - "required": false - }, - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "FORM12" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 29, - "label": "Gazette", - "purpose": "Gazette notices \u2013 all types of gazettes (establishing, closing, widening, erratum\u2019s) ", - "metadata_types": [ - { - "name": "GAZETTE_DATE", - "required": false - }, - { - "name": "GAZETTE_PAGE_NUMBER", - "required": false - }, - { - "name": "GAZETTE_PUBLISHED_DATE", - "required": false - }, - { - "name": "GAZETTE_TYPE", - "required": false - }, - { - "name": "LEGAL_SURVEY_PLAN_NUMBER", - "required": false - }, - { - "name": "LTSA_SCHEDULE_FILING_NUMBER", - "required": false - }, - { - "name": "MOTI_PLAN_NUMBER", - "required": false - }, - { - "name": "ROAD_NAME", - "required": false - } - ], - "name": "GAZE" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 30, - "label": "Historical file", - "purpose": "All types of scanned documents from a historical file", - "metadata_types": [ - { - "name": "END_DATE", - "required": false - }, - { - "name": "FILE_NUMBER", - "required": false - }, - { - "name": "PHYSICAL_LOCATION", - "required": false - }, - { - "name": "SECTION_NUMBER", - "required": false - }, - { - "name": "START_DATE", - "required": false - } - ], - "name": "HISTFILE" - }, - { - "categories": [ - "ACQUIRE" - ], - "display_order": 32, - "label": "Land Act Tenure/Reserves", - "purpose": "Applications and final Land Act Tenure Reserves (Sections 15/16/17)", - "metadata_types": [ - { - "name": "REFAG_DOC_NUMBER", - "required": false - }, - { - "name": "REFAG_LANDFILE_NUMBER", - "required": false - }, - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "LANDACTTENRES" - }, - { - "categories": [ - "ACQUIRE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 33, - "label": "Lease / License agreement", - "purpose": "Draft or Final Lease/License agreement, amending agreement or extension", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "LEASLICE" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 34, - "label": "Legal correspondence (ex: to AG/external lawyers)", - "purpose": "Emails or letters with internal and external lawyers ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "LEGACORR" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 35, - "label": "Legal survey plan", - "purpose": "Attach copies of legal survey plans, draft or registered versions", - "metadata_types": [ - { - "name": "LEGAL_SURVEY_PLAN_NUMBER", - "required": false - }, - { - "name": "MOTI_PLAN_NUMBER", - "required": false - }, - { - "name": "PLAN_TYPE", - "required": false - } - ], - "name": "LEGASURV" - }, - { - "categories": [ - "LEASLIC", - "MANAGEMENT" - ], - "display_order": 3, - "label": "Approval/sign-off", - "purpose": "The record of correspondence (email) approving Lease/Licence", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "LICEAPPR" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 31, - "label": "LTSA documents and plans (except title search)", - "purpose": "Attach LTSA documents ie: copies of mortgages, SRW\u2019s, Easements, Pending Litigation (except title searches and legal plans)", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "LTSADOCU" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 36, - "label": "Ministerial order", - "purpose": "Ministerial Orders", - "metadata_types": [ - { - "name": "DATE_SIGNED", - "required": false - }, - { - "name": "MOTI_FILE_NUMBER", - "required": false - }, - { - "name": "MO_NUMBER", - "required": false - }, - { - "name": "PROPERTY_IDENTIFIER", - "required": false - }, - { - "name": "ROAD_NAME", - "required": false - } - ], - "name": "MINIORDE" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 37, - "label": "Miscellaneous notes (LTSA)", - "purpose": "Miscellaneous Notes from the property title", - "metadata_types": [ - { - "name": "PID", - "required": false - } - ], - "name": "MISCNOTE" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 38, - "label": "MoTI plan", - "purpose": "Moti Plans, plans not found in LTSA (Road Surveys)", - "metadata_types": [ - { - "name": "LEGAL_SURVEY_PLAN_NUMBER", - "required": false - }, - { - "name": "MOTI_FILE_NUMBER", - "required": false - }, - { - "name": "MOTI_PLAN_NUMBER", - "required": false - }, - { - "name": "PUBLISHED_DATE", - "required": false - }, - { - "name": "RELATED_GAZETTE", - "required": false - } - ], - "name": "MOTIPLAN" - }, - { - "categories": [ - "ACQUIRE" - ], - "display_order": 26, - "label": "Form 7 - Notice of abandonement", - "purpose": "", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "NOTIABANDON" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT" - ], - "display_order": 27, - "label": "Form 8 - Notice of advanced payment ", - "purpose": "The Notice of Advance Payment (Form 8) for Expropriation", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "NOTIADVA" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 39, - "label": "Notice of claims/Litigation documents", - "purpose": "The Notice of Claims/Litigation documents ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "NOTICLAI" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT" - ], - "display_order": 23, - "label": "Form 1 - Notice of expropriation ", - "purpose": "The Notice of Expropriation (Form 1)", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "NOTIEXPR" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 40, - "label": "Notice of possible entry (H0224)", - "purpose": "The Notice of Possible Entry (H.0224) ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "NOTIPOSS" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 41, - "label": "Order in Council (OIC)", - "purpose": "The Order in Council", - "metadata_types": [ - { - "name": "OIC_NUMBER", - "required": false - }, - { - "name": "OIC_ROUTE_NUMBER", - "required": false - }, - { - "name": "OIC_TYPE", - "required": false - }, - { - "name": "ROAD_NAME", - "required": false - }, - { - "name": "YEAR", - "required": false - } - ], - "name": "OIC" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 42, - "label": "Other", - "purpose": "Uncategorized / Miscellaneous document", - "metadata_types": [ - { - "name": "PIN", - "required": false - }, - { - "name": "PROPERTY_IDENTIFIER", - "required": false - }, - { - "name": "ROAD_NAME", - "required": false - }, - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "OTHER" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT" - ], - "display_order": 43, - "label": "Other Land Agreement (Indemnity Letter, Letter of Intended Use, Assumption Agreement, etc)", - "purpose": "Other forms of Land Agreements such as Indemnity Letters, Letter of Intended Use, Assumption Agreements etc. ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "OTHERLANDAGREEMENT" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 44, - "label": "Owner agreement/offer", - "purpose": "Attach copies of draft or final Offer to Purchase agreements (H.179s) or external Owner agreements for Offer and Purchase", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "OWNEAGRE" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 45, - "label": "PA plans / Design drawings", - "purpose": "Property acquisition drawings, design drawings, engineered drawings in draft and final versions ", - "metadata_types": [ - { - "name": "PLAN_NUMBER", - "required": false - }, - { - "name": "PLAN_REVISION", - "required": false - }, - { - "name": "PROJECT_NAME", - "required": false - }, - { - "name": "PROJECT_NUMBER", - "required": false - } - ], - "name": "PAPLAN" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT", - "RESEARCH" - ], - "display_order": 46, - "label": "Photos / Images / Video", - "purpose": "Attach Photos, images or videos relevant to file/property", - "metadata_types": [ - { - "name": "CIVIC_ADDRESS", - "required": false - }, - { - "name": "DATE", - "required": false - }, - { - "name": "OWNER", - "required": false - }, - { - "name": "PROPERTY_IDENTIFIER", - "required": false - }, - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "PHOTIMAG" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 47, - "label": "Privy council", - "purpose": "The Privy Council ", - "metadata_types": [ - { - "name": "YEAR_PRIVY_COUNCIL_NUMBER", - "required": false - } - ], - "name": "PRIVCOUN" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 48, - "label": "Professional reports (ex: engineering/environmental etc.)", - "purpose": "Professional/Expert Report in draft or final (Ie: Geotechnical, Environmental, Archeology, Engineering Reports) ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "PROFREPO" - }, - { - "categories": [ - "DISPOSE", - "MANAGEMENT" - ], - "display_order": 49, - "label": "Purchase and Sale Agreement", - "purpose": "The Purchase and Sale Agreement for a road closure file (H.179RC)", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "PURSALEAGRE" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT" - ], - "display_order": 50, - "label": "Record of negotiation", - "purpose": "The Record of Negotiations ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "RECONEGO" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT" - ], - "display_order": 51, - "label": "Release of claims", - "purpose": "The Release of Claims document, Draft and Final versions", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "RELECLAI" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT" - ], - "display_order": 52, - "label": "Spending authority approval (SAA)", - "purpose": "The Spending Authority Approval (SAA) document, Draft and Final versions ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "SPENAUTH" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "PROJECT" - ], - "display_order": 53, - "label": "Surplus property declaration", - "purpose": "The Surplus Property Declarations document in Draft or Final indicating surplus land", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "SURPPROP" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 54, - "label": "Tax notices and assessments", - "purpose": "Tax notices or Tax assessments", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "TAXNOTI" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT" - ], - "display_order": 55, - "label": "Temporary license for construction access (H0074)", - "purpose": "The Temporary License for Construction Access Agreement (TLCA) Draft or Final version", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "TEMPLICE" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "LEASLIC", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 56, - "label": "Title search / Historical title", - "purpose": "The Title Search from LTSA which could be a current or historical search ", - "metadata_types": [ - { - "name": "OWNER", - "required": false - }, - { - "name": "PID", - "required": false - }, - { - "name": "TITLE_NUMBER", - "required": false - } - ], - "name": "TITLSEAR" - }, - { - "categories": [ - "ACQUIRE", - "DISPOSE", - "MANAGEMENT", - "RESEARCH" - ], - "display_order": 57, - "label": "Transfer of administration", - "purpose": "The Transfer of Administration (TAC) document \u2013 Draft or Final version ", - "metadata_types": [ - { - "name": "DATE_SIGNED", - "required": false - }, - { - "name": "MOTI_FILE_NUMBER", - "required": false - }, - { - "name": "PROPERTY_IDENTIFIER", - "required": false - }, - { - "name": "ROAD_NAME", - "required": false - }, - { - "name": "TRANSFER_NUMBER", - "required": false - } - ], - "name": "TRANADMI" - }, - { - "categories": [ - "ACQUIRE", - "MANAGEMENT" - ], - "display_order": 28, - "label": "Form 9 - Vesting notice (Form 9)", - "purpose": "The Vesting Notice (Form 9) for Expropriations ", - "metadata_types": [ - { - "name": "SHORT_DESCRIPTOR", - "required": false - } - ], - "name": "VESTNOTI" - } - ], - "metadata_types": [ - { - "label": "Application #", - "name": "APPLICATION_NUMBER" - }, - { - "label": "Canada land survey #", - "name": "CANADA_LAND_SURVEY_NUMBER" - }, - { - "label": "Civic address", - "name": "CIVIC_ADDRESS" - }, - { - "label": "Crown grant #", - "name": "CROWN_GRANT_NUMBER" - }, - { - "label": "Date", - "name": "DATE" - }, - { - "label": "Date signed", - "name": "DATE_SIGNED" - }, - { - "label": "District lot #", - "name": "DISTRICT_LOT_NUMBER" - }, - { - "label": "Electoral district", - "name": "ELECTORAL_DISTRICT" - }, - { - "label": "End date", - "name": "END_DATE" - }, - { - "label": "Field book #/Year", - "name": "FIELD_BOOK_NUMBER_YEAR" - }, - { - "label": "File #", - "name": "FILE_NUMBER" - }, - { - "label": "Gazette date", - "name": "GAZETTE_DATE" - }, - { - "label": "Gazette page #", - "name": "GAZETTE_PAGE_NUMBER" - }, - { - "label": "Gazette published date", - "name": "GAZETTE_PUBLISHED_DATE" - }, - { - "label": "Gazette type", - "name": "GAZETTE_TYPE" - }, - { - "label": "Highway district", - "name": "HIGHWAY_DISTRICT" - }, - { - "label": "Indian reserve or national park", - "name": "INDIAN_RESERVE_OR_NATIONAL_PARK" - }, - { - "label": "Jurisdiction", - "name": "JURISDICTION" - }, - { - "label": "Land district", - "name": "LAND_DISTRICT" - }, - { - "label": "Legal survey plan #", - "name": "LEGAL_SURVEY_PLAN_NUMBER" - }, - { - "label": "LTSA schedule filing #", - "name": "LTSA_SCHEDULE_FILING_NUMBER" - }, - { - "label": "MoTI file #", - "name": "MOTI_FILE_NUMBER" - }, - { - "label": "MoTI plan #", - "name": "MOTI_PLAN_NUMBER" - }, - { - "label": "MO #", - "name": "MO_NUMBER" - }, - { - "label": "OIC #", - "name": "OIC_NUMBER" - }, - { - "label": "OIC route #", - "name": "OIC_ROUTE_NUMBER" - }, - { - "label": "OIC type", - "name": "OIC_TYPE" - }, - { - "label": "Owner", - "name": "OWNER" - }, - { - "label": "Physical location", - "name": "PHYSICAL_LOCATION" - }, - { - "label": "PID", - "name": "PID" - }, - { - "label": "PIN", - "name": "PIN" - }, - { - "label": "Plan #", - "name": "PLAN_NUMBER" - }, - { - "label": "Plan revision", - "name": "PLAN_REVISION" - }, - { - "label": "Plan type", - "name": "PLAN_TYPE" - }, - { - "label": "Project name", - "name": "PROJECT_NAME" - }, - { - "label": "Project #", - "name": "PROJECT_NUMBER" - }, - { - "label": "Property identifier (PID or PIN or road name)", - "name": "PROPERTY_IDENTIFIER" - }, - { - "label": "Published date", - "name": "PUBLISHED_DATE" - }, - { - "label": "Reference/Agency Document #", - "name": "REFAG_DOC_NUMBER" - }, - { - "label": "Reference/Agency Lands file #", - "name": "REFAG_LANDFILE_NUMBER" - }, - { - "label": "Related gazette", - "name": "RELATED_GAZETTE" - }, - { - "label": "Road name", - "name": "ROAD_NAME" - }, - { - "label": "Road name", - "name": "ROAD_NAME" - }, - { - "label": "Roll #", - "name": "ROLL_NUMBER" - }, - { - "label": "Section #", - "name": "SECTION_NUMBER" - }, - { - "label": "Short descriptor", - "name": "SHORT_DESCRIPTOR" - }, - { - "label": "Start date", - "name": "START_DATE" - }, - { - "label": "Title #", - "name": "TITLE_NUMBER" - }, - { - "label": "Transfer #", - "name": "TRANSFER_NUMBER" - }, - { - "label": "Year", - "name": "YEAR" - }, - { - "label": "Year - privy council #", - "name": "YEAR_PRIVY_COUNCIL_NUMBER" - } - ], - "remove_lingering_document_types": true, - "remove_lingering_metadata_types": true -} \ No newline at end of file + "document_types": [ + { + "categories": ["ACQUIRE", "MANAGEMENT"], + "display_order": 0, + "label": "Affidavit of service", + "purpose": "The Affidavit of Services and correspondence with the Process Server ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "AFFISERV" + }, + { + "categories": ["ACQUIRE"], + "display_order": 1, + "label": "Agricultural Land Commission (ALC)", + "purpose": "For ALC Applications, Orders, and Resolutions ", + "metadata_types": [ + { + "name": "APPLICATION_NUMBER", + "required": false + }, + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "ALC" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT"], + "display_order": 25, + "label": "Form 5 - Approval of expropriation", + "purpose": "The Approval of Expropriation (Form 5) for Expropriations", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "APPREXPR" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT"], + "display_order": 2, + "label": "Appraisals/Reviews", + "purpose": "Appraisals property in draft / final state", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "APPRREVI" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 4, + "label": "BC assessment search", + "purpose": "BC Assessment search", + "metadata_types": [ + { + "name": "CIVIC_ADDRESS", + "required": false + }, + { + "name": "JURISDICTION", + "required": false + }, + { + "name": "ROLL_NUMBER", + "required": false + }, + { + "name": "YEAR", + "required": false + } + ], + "name": "BCASSE" + }, + { + "categories": [ + "ACQUIRE", + "DISPOSE", + "LEASLIC", + "MANAGEMENT", + "PROJECT", + "RESEARCH" + ], + "display_order": 5, + "label": "Briefing notes", + "purpose": "Draft / final versions of the briefing notes", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "BRIENOTE" + }, + { + "categories": ["ACQUIRE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 7, + "label": "Canada lands survey", + "purpose": "Surveys registered in the Canada Land System.", + "metadata_types": [ + { + "name": "CANADA_LAND_SURVEY_NUMBER", + "required": false + }, + { + "name": "INDIAN_RESERVE_OR_NATIONAL_PARK", + "required": false + } + ], + "name": "CANALAND" + }, + { + "categories": ["MANAGEMENT"], + "display_order": 6, + "label": "CDOGS template", + "purpose": "Templates used for document generation in PIMS", + "metadata_types": [], + "name": "CDOGTEMP" + }, + { + "categories": ["DISPOSE", "MANAGEMENT"], + "display_order": 8, + "label": "Certificate of Compliance", + "purpose": "The Certificate of Compliance (CoC)", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "CERTCOMPL" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT"], + "display_order": 9, + "label": "Certificate of Insurance (H0111)", + "purpose": "The Certificate of Insurance H.0111", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "CERTINSU" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT"], + "display_order": 11, + "label": "Compensation cheque", + "purpose": "The compensation cheque ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "COMPCHEQ" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT"], + "display_order": 12, + "label": "Compensation requisition (H-120)", + "purpose": "The H.120 Compensation Requisitions ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "COMPREQU" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 10, + "label": "Company search", + "purpose": "The company search", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "COMPSEAR" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT", "RESEARCH"], + "display_order": 13, + "label": "Condition of entry (H0443)", + "purpose": "H.0443 Conditions of Entry ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "CONDENTR" + }, + { + "categories": ["DISPOSE", "LEASLIC", "MANAGEMENT"], + "display_order": 14, + "label": "Conveyance closing documents (ex: PTT forms, Form A transfer etc.)", + "purpose": "The conveyance closing documents which might include PTT, Form A Transfer, or Statement of Adjustments. ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "CONVCLOS" + }, + { + "categories": [ + "ACQUIRE", + "DISPOSE", + "LEASLIC", + "MANAGEMENT", + "PROJECT", + "RESEARCH" + ], + "display_order": 15, + "label": "Correspondence", + "purpose": "Attach letters, memos, emails (not record of negotiation, or legal correspondence)", + "metadata_types": [ + { + "name": "CIVIC_ADDRESS", + "required": false + }, + { + "name": "DATE", + "required": false + }, + { + "name": "OWNER", + "required": false + }, + { + "name": "PROPERTY_IDENTIFIER", + "required": false + }, + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "CORR" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT", "RESEARCH"], + "display_order": 16, + "label": "Crown grant", + "purpose": "The crown grant application and final crown grant", + "metadata_types": [ + { + "name": "CROWN_GRANT_NUMBER", + "required": false + } + ], + "name": "CROWGRAN" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT", "RESEARCH"], + "display_order": 17, + "label": "District road register", + "purpose": "The District road register ", + "metadata_types": [ + { + "name": "ELECTORAL_DISTRICT", + "required": false + }, + { + "name": "HIGHWAY_DISTRICT", + "required": false + }, + { + "name": "ROAD_NAME", + "required": false + } + ], + "name": "DISTREGI" + }, + { + "categories": ["DISPOSE", "MANAGEMENT"], + "display_order": 18, + "label": "Enhanced Referral Records", + "purpose": "The Referral Record for both incoming and outgoing referrals.", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "ENHREFREC" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT", "RESEARCH"], + "display_order": 19, + "label": "Field notes", + "purpose": "The survey field notes ", + "metadata_types": [ + { + "name": "DISTRICT_LOT_NUMBER", + "required": false + }, + { + "name": "FIELD_BOOK_NUMBER_YEAR", + "required": false + }, + { + "name": "LAND_DISTRICT", + "required": false + } + ], + "name": "FIELNOTE" + }, + { + "categories": [ + "ACQUIRE", + "DISPOSE", + "LEASLIC", + "MANAGEMENT", + "PROJECT", + "RESEARCH" + ], + "display_order": 20, + "label": "Financial records (invoices, journal vouchers, received cheques etc.)", + "purpose": "Financial records which could include invoices, JV\u2019s, contracts ect. ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "FINRECRD" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 22, + "label": "First nations consultation", + "purpose": "Record of engagement with first nations (ex: First Nations Consultation Log)", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "FIRSNTAI" + }, + { + "categories": ["DISPOSE", "MANAGEMENT"], + "display_order": 21, + "label": "First Nations Strength of Claim Report", + "purpose": "The First Nation Strength of Claim report", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "FNSCR" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 24, + "label": "Form 12", + "purpose": "Form 12 - Certificate As To Highway In Statutory Right Of Way Plan, pre and post registration", + "metadata_types": [ + { + "name": "GAZETTE_DATE", + "required": false + }, + { + "name": "LEGAL_SURVEY_PLAN_NUMBER", + "required": false + }, + { + "name": "MOTI_PLAN_NUMBER", + "required": false + }, + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "FORM12" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 29, + "label": "Gazette", + "purpose": "Gazette notices \u2013 all types of gazettes (establishing, closing, widening, erratum\u2019s) ", + "metadata_types": [ + { + "name": "GAZETTE_DATE", + "required": false + }, + { + "name": "GAZETTE_PAGE_NUMBER", + "required": false + }, + { + "name": "GAZETTE_PUBLISHED_DATE", + "required": false + }, + { + "name": "GAZETTE_TYPE", + "required": false + }, + { + "name": "LEGAL_SURVEY_PLAN_NUMBER", + "required": false + }, + { + "name": "LTSA_SCHEDULE_FILING_NUMBER", + "required": false + }, + { + "name": "MOTI_PLAN_NUMBER", + "required": false + }, + { + "name": "ROAD_NAME", + "required": false + } + ], + "name": "GAZE" + }, + { + "categories": [ + "ACQUIRE", + "DISPOSE", + "LEASLIC", + "MANAGEMENT", + "PROJECT", + "RESEARCH" + ], + "display_order": 30, + "label": "Historical file", + "purpose": "All types of scanned documents from a historical file", + "metadata_types": [ + { + "name": "END_DATE", + "required": false + }, + { + "name": "FILE_NUMBER", + "required": false + }, + { + "name": "PHYSICAL_LOCATION", + "required": false + }, + { + "name": "SECTION_NUMBER", + "required": false + }, + { + "name": "START_DATE", + "required": false + } + ], + "name": "HISTFILE" + }, + { + "categories": ["ACQUIRE"], + "display_order": 32, + "label": "Land Act Tenure/Reserves", + "purpose": "Applications and final Land Act Tenure Reserves (Sections 15/16/17)", + "metadata_types": [ + { + "name": "REFAG_DOC_NUMBER", + "required": false + }, + { + "name": "REFAG_LANDFILE_NUMBER", + "required": false + }, + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "LANDACTTENRES" + }, + { + "categories": ["ACQUIRE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 33, + "label": "Lease / License agreement", + "purpose": "Draft or Final Lease/License agreement, amending agreement or extension", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "LEASLICE" + }, + { + "categories": [ + "ACQUIRE", + "DISPOSE", + "LEASLIC", + "MANAGEMENT", + "PROJECT", + "RESEARCH" + ], + "display_order": 34, + "label": "Legal correspondence (ex: to AG/external lawyers)", + "purpose": "Emails or letters with internal and external lawyers ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "LEGACORR" + }, + { + "categories": [ + "ACQUIRE", + "DISPOSE", + "LEASLIC", + "MANAGEMENT", + "PROJECT", + "RESEARCH" + ], + "display_order": 35, + "label": "Legal survey plan", + "purpose": "Attach copies of legal survey plans, draft or registered versions", + "metadata_types": [ + { + "name": "LEGAL_SURVEY_PLAN_NUMBER", + "required": false + }, + { + "name": "MOTI_PLAN_NUMBER", + "required": false + }, + { + "name": "PLAN_TYPE", + "required": false + } + ], + "name": "LEGASURV" + }, + { + "categories": ["LEASLIC", "MANAGEMENT"], + "display_order": 3, + "label": "Approval/sign-off", + "purpose": "The record of correspondence (email) approving Lease/Licence", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "LICEAPPR" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 31, + "label": "LTSA documents and plans (except title search)", + "purpose": "Attach LTSA documents ie: copies of mortgages, SRW\u2019s, Easements, Pending Litigation (except title searches and legal plans)", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "LTSADOCU" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT", "RESEARCH"], + "display_order": 36, + "label": "Ministerial order", + "purpose": "Ministerial Orders", + "metadata_types": [ + { + "name": "DATE_SIGNED", + "required": false + }, + { + "name": "MOTI_FILE_NUMBER", + "required": false + }, + { + "name": "MO_NUMBER", + "required": false + }, + { + "name": "PROPERTY_IDENTIFIER", + "required": false + }, + { + "name": "ROAD_NAME", + "required": false + } + ], + "name": "MINIORDE" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 37, + "label": "Miscellaneous notes (LTSA)", + "purpose": "Miscellaneous Notes from the property title", + "metadata_types": [ + { + "name": "PID", + "required": false + } + ], + "name": "MISCNOTE" + }, + { + "categories": [ + "ACQUIRE", + "DISPOSE", + "LEASLIC", + "MANAGEMENT", + "PROJECT", + "RESEARCH" + ], + "display_order": 38, + "label": "MoTI plan", + "purpose": "Moti Plans, plans not found in LTSA (Road Surveys)", + "metadata_types": [ + { + "name": "LEGAL_SURVEY_PLAN_NUMBER", + "required": false + }, + { + "name": "MOTI_FILE_NUMBER", + "required": false + }, + { + "name": "MOTI_PLAN_NUMBER", + "required": false + }, + { + "name": "PUBLISHED_DATE", + "required": false + }, + { + "name": "RELATED_GAZETTE", + "required": false + } + ], + "name": "MOTIPLAN" + }, + { + "categories": ["ACQUIRE"], + "display_order": 26, + "label": "Form 7 - Notice of abandonement", + "purpose": "", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "NOTIABANDON" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT"], + "display_order": 27, + "label": "Form 8 - Notice of advanced payment ", + "purpose": "The Notice of Advance Payment (Form 8) for Expropriation", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "NOTIADVA" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT", "RESEARCH"], + "display_order": 39, + "label": "Notice of claims/Litigation documents", + "purpose": "The Notice of Claims/Litigation documents ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "NOTICLAI" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT"], + "display_order": 23, + "label": "Form 1 - Notice of expropriation ", + "purpose": "The Notice of Expropriation (Form 1)", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "NOTIEXPR" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT", "RESEARCH"], + "display_order": 40, + "label": "Notice of possible entry (H0224)", + "purpose": "The Notice of Possible Entry (H.0224) ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "NOTIPOSS" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT", "RESEARCH"], + "display_order": 41, + "label": "Order in Council (OIC)", + "purpose": "The Order in Council", + "metadata_types": [ + { + "name": "OIC_NUMBER", + "required": false + }, + { + "name": "OIC_ROUTE_NUMBER", + "required": false + }, + { + "name": "OIC_TYPE", + "required": false + }, + { + "name": "ROAD_NAME", + "required": false + }, + { + "name": "YEAR", + "required": false + } + ], + "name": "OIC" + }, + { + "categories": [ + "ACQUIRE", + "DISPOSE", + "LEASLIC", + "MANAGEMENT", + "PROJECT", + "RESEARCH" + ], + "display_order": 42, + "label": "Other", + "purpose": "Uncategorized / Miscellaneous document", + "metadata_types": [ + { + "name": "PIN", + "required": false + }, + { + "name": "PROPERTY_IDENTIFIER", + "required": false + }, + { + "name": "ROAD_NAME", + "required": false + }, + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "OTHER" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT"], + "display_order": 43, + "label": "Other Land Agreement (Indemnity Letter, Letter of Intended Use, Assumption Agreement, etc)", + "purpose": "Other forms of Land Agreements such as Indemnity Letters, Letter of Intended Use, Assumption Agreements etc. ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "OTHERLANDAGREEMENT" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT", "RESEARCH"], + "display_order": 44, + "label": "Owner agreement/offer", + "purpose": "Attach copies of draft or final Offer to Purchase agreements (H.179s) or external Owner agreements for Offer and Purchase", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "OWNEAGRE" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT", "PROJECT", "RESEARCH"], + "display_order": 45, + "label": "PA plans / Design drawings", + "purpose": "Property acquisition drawings, design drawings, engineered drawings in draft and final versions ", + "metadata_types": [ + { + "name": "PLAN_NUMBER", + "required": false + }, + { + "name": "PLAN_REVISION", + "required": false + }, + { + "name": "PROJECT_NAME", + "required": false + }, + { + "name": "PROJECT_NUMBER", + "required": false + } + ], + "name": "PAPLAN" + }, + { + "categories": [ + "ACQUIRE", + "DISPOSE", + "LEASLIC", + "MANAGEMENT", + "PROJECT", + "RESEARCH" + ], + "display_order": 46, + "label": "Photos / Images / Video", + "purpose": "Attach Photos, images or videos relevant to file/property", + "metadata_types": [ + { + "name": "CIVIC_ADDRESS", + "required": false + }, + { + "name": "DATE", + "required": false + }, + { + "name": "OWNER", + "required": false + }, + { + "name": "PROPERTY_IDENTIFIER", + "required": false + }, + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "PHOTIMAG" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT", "RESEARCH"], + "display_order": 47, + "label": "Privy council", + "purpose": "The Privy Council ", + "metadata_types": [ + { + "name": "YEAR_PRIVY_COUNCIL_NUMBER", + "required": false + } + ], + "name": "PRIVCOUN" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 48, + "label": "Professional reports (ex: engineering/environmental etc.)", + "purpose": "Professional/Expert Report in draft or final (Ie: Geotechnical, Environmental, Archeology, Engineering Reports) ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "PROFREPO" + }, + { + "categories": ["DISPOSE", "MANAGEMENT"], + "display_order": 49, + "label": "Purchase and Sale Agreement", + "purpose": "The Purchase and Sale Agreement for a road closure file (H.179RC)", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "PURSALEAGRE" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT"], + "display_order": 50, + "label": "Record of negotiation", + "purpose": "The Record of Negotiations ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "RECONEGO" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT"], + "display_order": 51, + "label": "Release of claims", + "purpose": "The Release of Claims document, Draft and Final versions", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "RELECLAI" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "PROJECT"], + "display_order": 52, + "label": "Spending authority approval (SAA)", + "purpose": "The Spending Authority Approval (SAA) document, Draft and Final versions ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "SPENAUTH" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "PROJECT"], + "display_order": 53, + "label": "Surplus property declaration", + "purpose": "The Surplus Property Declarations document in Draft or Final indicating surplus land", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "SURPPROP" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 54, + "label": "Tax notices and assessments", + "purpose": "Tax notices or Tax assessments", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "TAXNOTI" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT"], + "display_order": 55, + "label": "Temporary license for construction access (H0074)", + "purpose": "The Temporary License for Construction Access Agreement (TLCA) Draft or Final version", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "TEMPLICE" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "LEASLIC", "MANAGEMENT", "RESEARCH"], + "display_order": 56, + "label": "Title search / Historical title", + "purpose": "The Title Search from LTSA which could be a current or historical search ", + "metadata_types": [ + { + "name": "OWNER", + "required": false + }, + { + "name": "PID", + "required": false + }, + { + "name": "TITLE_NUMBER", + "required": false + } + ], + "name": "TITLSEAR" + }, + { + "categories": ["ACQUIRE", "DISPOSE", "MANAGEMENT", "RESEARCH"], + "display_order": 57, + "label": "Transfer of administration", + "purpose": "The Transfer of Administration (TAC) document \u2013 Draft or Final version ", + "metadata_types": [ + { + "name": "DATE_SIGNED", + "required": false + }, + { + "name": "MOTI_FILE_NUMBER", + "required": false + }, + { + "name": "PROPERTY_IDENTIFIER", + "required": false + }, + { + "name": "ROAD_NAME", + "required": false + }, + { + "name": "TRANSFER_NUMBER", + "required": false + } + ], + "name": "TRANADMI" + }, + { + "categories": ["ACQUIRE", "MANAGEMENT"], + "display_order": 28, + "label": "Form 9 - Vesting notice (Form 9)", + "purpose": "The Vesting Notice (Form 9) for Expropriations ", + "metadata_types": [ + { + "name": "SHORT_DESCRIPTOR", + "required": false + } + ], + "name": "VESTNOTI" + } + ], + "metadata_types": [ + { + "label": "Application #", + "name": "APPLICATION_NUMBER" + }, + { + "label": "Canada land survey #", + "name": "CANADA_LAND_SURVEY_NUMBER" + }, + { + "label": "Civic address", + "name": "CIVIC_ADDRESS" + }, + { + "label": "Crown grant #", + "name": "CROWN_GRANT_NUMBER" + }, + { + "label": "Date", + "name": "DATE" + }, + { + "label": "Date signed", + "name": "DATE_SIGNED" + }, + { + "label": "District lot #", + "name": "DISTRICT_LOT_NUMBER" + }, + { + "label": "Electoral district", + "name": "ELECTORAL_DISTRICT" + }, + { + "label": "End date", + "name": "END_DATE" + }, + { + "label": "Field book #/Year", + "name": "FIELD_BOOK_NUMBER_YEAR" + }, + { + "label": "File #", + "name": "FILE_NUMBER" + }, + { + "label": "Gazette date", + "name": "GAZETTE_DATE" + }, + { + "label": "Gazette page #", + "name": "GAZETTE_PAGE_NUMBER" + }, + { + "label": "Gazette published date", + "name": "GAZETTE_PUBLISHED_DATE" + }, + { + "label": "Gazette type", + "name": "GAZETTE_TYPE" + }, + { + "label": "Highway district", + "name": "HIGHWAY_DISTRICT" + }, + { + "label": "Indian reserve or national park", + "name": "INDIAN_RESERVE_OR_NATIONAL_PARK" + }, + { + "label": "Jurisdiction", + "name": "JURISDICTION" + }, + { + "label": "Land district", + "name": "LAND_DISTRICT" + }, + { + "label": "Legal survey plan #", + "name": "LEGAL_SURVEY_PLAN_NUMBER" + }, + { + "label": "LTSA schedule filing #", + "name": "LTSA_SCHEDULE_FILING_NUMBER" + }, + { + "label": "MoTI file #", + "name": "MOTI_FILE_NUMBER" + }, + { + "label": "MoTI plan #", + "name": "MOTI_PLAN_NUMBER" + }, + { + "label": "MO #", + "name": "MO_NUMBER" + }, + { + "label": "OIC #", + "name": "OIC_NUMBER" + }, + { + "label": "OIC route #", + "name": "OIC_ROUTE_NUMBER" + }, + { + "label": "OIC type", + "name": "OIC_TYPE" + }, + { + "label": "Owner", + "name": "OWNER" + }, + { + "label": "Physical location", + "name": "PHYSICAL_LOCATION" + }, + { + "label": "PID", + "name": "PID" + }, + { + "label": "PIN", + "name": "PIN" + }, + { + "label": "Plan #", + "name": "PLAN_NUMBER" + }, + { + "label": "Plan revision", + "name": "PLAN_REVISION" + }, + { + "label": "Plan type", + "name": "PLAN_TYPE" + }, + { + "label": "Project name", + "name": "PROJECT_NAME" + }, + { + "label": "Project #", + "name": "PROJECT_NUMBER" + }, + { + "label": "Property identifier (PID or PIN or road name)", + "name": "PROPERTY_IDENTIFIER" + }, + { + "label": "Published date", + "name": "PUBLISHED_DATE" + }, + { + "label": "Reference/Agency Document #", + "name": "REFAG_DOC_NUMBER" + }, + { + "label": "Reference/Agency Lands file #", + "name": "REFAG_LANDFILE_NUMBER" + }, + { + "label": "Related gazette", + "name": "RELATED_GAZETTE" + }, + { + "label": "Road name", + "name": "ROAD_NAME" + }, + { + "label": "Road name", + "name": "ROAD_NAME" + }, + { + "label": "Roll #", + "name": "ROLL_NUMBER" + }, + { + "label": "Section #", + "name": "SECTION_NUMBER" + }, + { + "label": "Short descriptor", + "name": "SHORT_DESCRIPTOR" + }, + { + "label": "Start date", + "name": "START_DATE" + }, + { + "label": "Title #", + "name": "TITLE_NUMBER" + }, + { + "label": "Transfer #", + "name": "TRANSFER_NUMBER" + }, + { + "label": "Year", + "name": "YEAR" + }, + { + "label": "Year - privy council #", + "name": "YEAR_PRIVY_COUNCIL_NUMBER" + } + ], + "remove_lingering_document_types": true, + "remove_lingering_metadata_types": true +}