Skip to content

Commit

Permalink
psp-9500 Scheduler microservice (#4483)
Browse files Browse the repository at this point in the history
* split off geoserver functionality into a microservice - update docker and ci/cd to support.

* psp-9500 scheduler microservice.

* change service scope

* Readme updates.

* github action corrections.

---------

Co-authored-by: Smith <[email protected]>
  • Loading branch information
devinleighsmith and Smith authored Dec 5, 2024
1 parent 623b443 commit 2ebb2e6
Show file tree
Hide file tree
Showing 100 changed files with 2,587 additions and 114 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/api-dotnetcore.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
if: ${{ needs.check-changes.outputs.backend == 'true' }}
strategy:
matrix:
services: [{directory: ./source/backend/api, solution: 'Pims.sln'}, {directory: ./source/backend/proxy, solution: 'Proxy.sln'}]
services: [{directory: ./source/backend/api, solution: 'Pims.sln'}, {directory: ./source/backend/proxy, solution: 'Proxy.sln'}, {directory: ./source/backend/scheduler, solution: 'Scheduler.sln'}]
env:
working-directory: ${{ matrix.services.directory }}
solution-name: ${{ matrix.services.solution }}
Expand Down
13 changes: 8 additions & 5 deletions .github/workflows/ci-cd-pims-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,10 @@ env:
DEPLOYMENT_NAMESPACE: "3cd915-dev"

on:
pull_request_target:
branches: [dev]
types: [closed]
workflow_dispatch:

jobs:
ci-cd-start-notification:
if: github.event.pull_request.merged == true
name: CI-CD Start Notification to Teams Channel
runs-on: ubuntu-latest
steps:
Expand Down Expand Up @@ -83,6 +80,7 @@ jobs:
run: |
./openshift/4.0/player.sh build api -apply
./openshift/4.0/player.sh build proxy -apply
./openshift/4.0/player.sh build scheduler -apply
deploy:
name: Deploy to OpenShift
Expand Down Expand Up @@ -116,6 +114,12 @@ jobs:
oc tag pims-proxy:latest-$DESTINATION pims-proxy:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-proxy-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-proxy-$DESTINATION
- name: Deploy scheduler microservice
shell: bash
run: |
oc tag pims-scheduler:latest-$DESTINATION pims-scheduler:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-scheduler-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-scheduler-$DESTINATION
- name: Deploy mayan
shell: bash
run: |
Expand Down Expand Up @@ -193,7 +197,6 @@ jobs:
name: CI-CD End Notification to Teams Channel
runs-on: ubuntu-latest
needs: keycloak-sync
if: ${{ always() && github.event.pull_request.merged == true }}
steps:
- name: check workflow status
uses: martialonline/workflow-status@v4
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/deploy-prod-start.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ jobs:
run: |
[[ -z ${{github.event.inputs.OVERRIDE_VERSION}} ]] && RELEASE_VERSION=${{steps.previoustag.outputs.tag}}-master || RELEASE_VERSION=${{github.event.inputs.OVERRIDE_VERSION}}-master
RELEASE_TAG=$RELEASE_VERSION ./openshift/4.0/player.sh deploy api $DESTINATION -apply
RELEASE_TAG=$RELEASE_VERSION ./openshift/4.0/player.sh deploy proxy $DESTINATION -apply
RELEASE_TAG=$RELEASE_VERSION ./openshift/4.0/player.sh deploy scheduler $DESTINATION -apply
RELEASE_TAG=$RELEASE_VERSION ./openshift/4.0/player.sh deploy app $DESTINATION -apply
oc tag mayan-bcgov:$RELEASE_VERSION mayan-bcgov:master
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/retag-dev-to-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ jobs:
oc tag pims-proxy:$RELEASE_TAG pims-proxy:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-proxy-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-proxy-$DESTINATION
- name: Deploy scheduler microservice
shell: bash
run: |
oc tag pims-scheduler:$RELEASE_TAG pims-scheduler:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-scheduler-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-scheduler-$DESTINATION
- name: Deploy mayan
shell: bash
run: |
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/retag-test-to-uat-argo.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ jobs:
oc tag pims-proxy:$RELEASE_TAG pims-proxy:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-proxy-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-proxy-$DESTINATION
- name: Deploy scheduler microservice
shell: bash
run: |
oc tag pims-scheduler:$RELEASE_TAG pims-scheduler:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-scheduler-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-scheduler-$DESTINATION
- name: Deploy mayan
shell: bash
run: |
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/retag-test-to-uat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ jobs:
- name: call scripts to deploy api and frontend
run: |
./openshift/4.0/player.sh deploy api $DESTINATION -apply
./openshift/4.0/player.sh deploy proxy $DESTINATION -apply
./openshift/4.0/player.sh deploy scheduler $DESTINATION -apply
./openshift/4.0/player.sh deploy app $DESTINATION -apply
oc tag mayan-bcgov:test mayan-bcgov:$DESTINATION
# the proxy can only be deployed via DEPLOYMENTS (ArgoCD way)
Expand Down Expand Up @@ -162,6 +164,7 @@ jobs:
oc tag pims-app:uat pims-app:v${VERSION}-master
oc tag pims-api:uat pims-api:v${VERSION}-master
oc tag pims-proxy:uat pims-proxy:v${VERSION}-master
oc tag pims-scheduler:uat pims-scheduler:v${VERSION}-master
oc tag mayan-bcgov:uat mayan-bcgov:v${VERSION}-master
ci-cd-end-notification:
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/uat_hotfix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ jobs:
run: |
./openshift/4.0/player.sh build api -apply
./openshift/4.0/player.sh build proxy -apply
./openshift/4.0/player.sh build scheduler -apply
deploy:
name: Deploy frontend and api to OpenShift
Expand All @@ -100,6 +101,7 @@ jobs:
run: |
./openshift/4.0/player.sh deploy api $DESTINATION -apply
./openshift/4.0/player.sh deploy proxy $DESTINATION -apply
./openshift/4.0/player.sh deploy scheduler $DESTINATION -apply
./openshift/4.0/player.sh deploy app $DESTINATION -apply
# the command:
Expand Down Expand Up @@ -188,6 +190,8 @@ jobs:
VERSION=$(make version)
oc tag pims-app:uat pims-app:v${VERSION}-master
oc tag pims-api:uat pims-api:v${VERSION}-master
oc tag pims-api:uat pims-proxy:v${VERSION}-master
oc tag pims-api:uat pims-scheduler:v${VERSION}-master
ci-cd-end-notification:
if: always()
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/uat_hotfix_argo.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ jobs:
run: |
./openshift/4.0/player.sh build api -apply
./openshift/4.0/player.sh build proxy -apply
./openshift/4.0/player.sh build scheduler -apply
deploy:
name: Deploy to OpenShift
Expand Down Expand Up @@ -115,6 +116,12 @@ jobs:
oc tag pims-proxy:latest-$DESTINATION pims-proxy:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-proxy-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-proxy-$DESTINATION
- name: Deploy scheduler microservice
shell: bash
run: |
oc tag pims-scheduler:latest-$DESTINATION pims-scheduler:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-scheduler-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-scheduler-$DESTINATION
# the command:
# 1) creates an openshift job with generated name to avoid name conflict, substituting the variables in the template.
Expand Down Expand Up @@ -203,6 +210,7 @@ jobs:
oc tag pims-app:uat pims-app:v${VERSION}-master
oc tag pims-api:uat pims-api:v${VERSION}-master
oc tag pims-proxy:uat pims-proxy:v${VERSION}-master
oc tag pims-scheduler:uat pims-scheduler:v${VERSION}-master
oc tag mayan-bcgov:uat mayan-bcgov:v${VERSION}-master
ci-cd-end-notification:
Expand Down
5 changes: 4 additions & 1 deletion .github/workflows/uat_pre_release_hotfix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ jobs:
run: |
OC_JOB_NAME=${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh build api -apply
OC_JOB_NAME=${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh build proxy -apply
OC_JOB_NAME=${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh build scheduler -apply
deploy:
name: Deploy frontend and api to OpenShift
Expand All @@ -126,6 +127,7 @@ jobs:
run: |
RELEASE_TAG=latest-${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh deploy api $DESTINATION -apply
RELEASE_TAG=latest-${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh deploy proxy $DESTINATION -apply
RELEASE_TAG=latest-${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh deploy scheduler $DESTINATION -apply
RELEASE_TAG=latest-${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh deploy app $DESTINATION -apply
# the command:
Expand Down Expand Up @@ -214,7 +216,8 @@ jobs:
VERSION=$(make version)
oc tag pims-app:uat pims-app:v${VERSION}-master
oc tag pims-api:uat pims-api:v${VERSION}-master
oc tag pims-proxy:uat pims-api:v${VERSION}-master
oc tag pims-proxy:uat pims-proxy:v${VERSION}-master
oc tag pims-scheduler:uat pims-scheduler:v${VERSION}-master
ci-cd-end-notification:
if: always()
Expand Down
11 changes: 10 additions & 1 deletion .github/workflows/uat_pre_release_hotfix_argo.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ jobs:
run: |
OC_JOB_NAME=${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh build api -apply
OC_JOB_NAME=${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh build proxy -apply
OC_JOB_NAME=${{github.event.inputs.HOTFIX_BRANCH}} && ./openshift/4.0/player.sh build scheduler -apply
deploy:
name: Deploy to OpenShift
Expand Down Expand Up @@ -144,6 +145,13 @@ jobs:
oc tag pims-proxy:$RELEASE_TAG pims-proxy:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-proxy-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-proxy-$DESTINATION
- name: Deploy scheduler microservice
shell: bash
run: |
export RELEASE_TAG=latest-${{github.event.inputs.HOTFIX_BRANCH}}
oc tag pims-scheduler:$RELEASE_TAG pims-scheduler:$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout restart deployment/pims-scheduler-$DESTINATION
oc -n $DEPLOYMENT_NAMESPACE rollout status --timeout=600s deployment/pims-scheduler-$DESTINATION
# the command:
# 1) creates an openshift job with generated name to avoid name conflict, substituting the variables in the template.
Expand Down Expand Up @@ -231,7 +239,8 @@ jobs:
VERSION=$(make version)
oc tag pims-app:uat pims-app:v${VERSION}-master
oc tag pims-api:uat pims-api:v${VERSION}-master
oc tag pims-proxy:uat pims-api:v${VERSION}-master
oc tag pims-proxy:uat pims-proxy:v${VERSION}-master
oc tag pims-scheduler:uat pims-scheduler:v${VERSION}-master
ci-cd-end-notification:
if: always()
Expand Down
40 changes: 40 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,44 @@ services:
networks:
- psp

####################### Scheduler #######################
scheduler:
profiles:
- all
- psp
restart: on-failure:1
container_name: psp-scheduler
build:
context: source/backend
dockerfile: Dockerfile.scheduler
args:
BUILD_CONFIGURATION: Debug
env_file: source/backend/scheduler/.env
environment:
- ASPNETCORE_ENVIRONMENT=Docker
ports:
- ${SCHEDULER_HTTPS_PORT:-8058}:443
- ${SCHEDULER_HTTP_PORT:-8057}:8080
depends_on:
- scheduler-redis
networks:
- psp

####################### Redis #######################
scheduler-redis:
image: redis:6.2-alpine
profiles:
- all
- psp
restart: on-failure:1
ports:
- 6379:6379
networks:
- psp
command: redis-server --save 20 1 --loglevel warning
volumes:
- redis:/data

####################### Frontend #######################
frontend:
profiles:
Expand Down Expand Up @@ -199,3 +237,5 @@ volumes:
name: psp-api-db-rhel-data
frontend-node-cache:
name: psp-frontend-node-cache
redis-scheduler:
driver: local
33 changes: 33 additions & 0 deletions source/backend/Dockerfile.scheduler
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
ARG BUILD_CONFIGURATION=Release
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
WORKDIR /app
EXPOSE 5001 5000

# Copy csproj and restore as distinct layers
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
WORKDIR /src
COPY scheduler scheduler/
COPY scheduler/Directory.Build.props ./
COPY core core/
COPY apimodels apimodels/
COPY entities entities/
COPY core.api core.api/
COPY keycloak keycloak/
COPY scheduler/*.csproj scheduler/

RUN dotnet restore scheduler/Scheduler.sln
ENV PATH="$PATH:/root/.dotnet/tools"
# Copy everything else and build
WORKDIR /src/scheduler
RUN dotnet build "Pims.Scheduler.csproj" -c "$BUILD_CONFIGURATION" -o /app/build

FROM build AS publish
RUN dotnet publish "Pims.Scheduler.csproj" -c "$BUILD_CONFIGURATION" -o /app/publish

# Runtime image
FROM base AS final
WORKDIR /app
COPY --from=publish /app/publish .
COPY entrypoint.scheduler.sh .
RUN chmod +x /app/entrypoint.scheduler.sh
ENTRYPOINT ["/app/entrypoint.scheduler.sh"]
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
using MapsterMapper;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Pims.Core.Api.Exceptions;
using Pims.Api.Models.Concepts.DispositionFile;
using Pims.Api.Models.Concepts.File;
using Pims.Core.Api.Policies;
using Pims.Api.Services;
using Pims.Core.Api.Exceptions;
using Pims.Core.Api.Policies;
using Pims.Core.Json;
using Pims.Core.Security;
using Swashbuckle.AspNetCore.Annotations;
Expand Down
64 changes: 64 additions & 0 deletions source/backend/api/Areas/Documents/DocumentQueueController.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
using System.Collections.Generic;
using MapsterMapper;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Pims.Api.Services;
using Pims.Core.Api.Policies;
using Pims.Core.Json;
using Pims.Core.Security;
using Pims.Dal.Entities.Models;
using Swashbuckle.AspNetCore.Annotations;

namespace Pims.Api.Controllers
{
/// <summary>
/// DocumentQueueController class, provides endpoints to handle document queue requests.
/// </summary>
[Authorize]
[ApiController]
[ApiVersion("1.0")]
[Route("v{version:apiVersion}/documents/queue")]
[Route("/documents")]
public class DocumentQueueController : ControllerBase
{
#region Variables
private readonly IDocumentQueueService _documentQueueService;
private readonly IMapper _mapper;
#endregion

#region Constructors

/// <summary>
/// Creates a new instance of a DocumentQueueController class.
/// </summary>
/// <param name="documentQueueService"></param>
/// <param name="mapper"></param>
public DocumentQueueController(IDocumentQueueService documentQueueService, IMapper mapper)
{
_documentQueueService = documentQueueService;
_mapper = mapper;
}
#endregion

#region Endpoints

/// <summary>
/// Search for Document Queue items via filter.
/// </summary>
/// <returns></returns>
[HttpGet("search")]
[HasPermission(Permissions.SystemAdmin)]
[Produces("application/json")]
[ProducesResponseType(typeof(List<Models.Concepts.Document.DocumentTypeModel>), 200)]
[SwaggerOperation(Tags = new[] { "document-types" })]
[TypeFilter(typeof(NullJsonResultFilter))]
public IActionResult GetDocumentTypes([FromBody] DocumentQueueFilter filter)
{
var queuedDocuments = _documentQueueService.SearchDocumentQueue(filter);
var documentQueueModels = _mapper.Map<List<Models.Concepts.Document.DocumentQueueModel>>(queuedDocuments);
return new JsonResult(documentQueueModels);
}

#endregion
}
}
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Pims.Api.Models.Config;
using Pims.Api.Repositories.Rest;
using Pims.Core.Api.Repositories.Rest;

namespace Pims.Api.Repositories.Cdogs
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Pims.Api.Models.Config;
using Pims.Api.Repositories.Rest;
using Pims.Core.Api.Repositories.Rest;

namespace Pims.Api.Repositories.Mayan
{
Expand Down
Loading

0 comments on commit 2ebb2e6

Please sign in to comment.