From 7c953cb9f17794ba0451f545d6c7f02ec28452fd Mon Sep 17 00:00:00 2001 From: devinleighsmith <41091511+devinleighsmith@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:52:52 -0800 Subject: [PATCH] add automatic db deployment to test, uat. add keycloak sync to uat. update action definitions to allow for env specific secrets for keycloak sync. (#3573) * add automatic db deployment to test. for testing purposes only, revert. typo. skip unecessary steps. temp. Revert "for testing purposes only, revert." This reverts commit 629967c17bcd64974bb78a2d1e0ca88fc65a7b8a. Revert "skip unecessary steps." This reverts commit bedc62732b44d43062acddddbe551d2501fa6d48. * code review comments. * test correction. --- .github/workflows/ci-cd-pims-dev.yml | 2 +- .github/workflows/retag-dev-to-test.yml | 30 +- .github/workflows/retag-test-to-uat.yml | 55 +- .../PIMS_PROPERTY_SPATIAL_CORRECTION.dtproj | 8 +- ...MS_PROPERTY_SPATIAL_CORRECTION.dtproj.user | 2 + .../Package.dtsx | 1230 ++++++++++------- openshift/4.0/templates/jobs/db-deploy.yaml | 14 +- .../dal/Repositories/PropertyRepository.cs | 2 +- .../Repositories/PropertyRepositoryTest.cs | 1 + .../mapSideBar/acquisition/common/models.ts | 9 +- .../edit/PropertyActivityEditForm.tsx | 3 + 11 files changed, 821 insertions(+), 535 deletions(-) diff --git a/.github/workflows/ci-cd-pims-dev.yml b/.github/workflows/ci-cd-pims-dev.yml index 1331cc9c83..c42a1d2bde 100644 --- a/.github/workflows/ci-cd-pims-dev.yml +++ b/.github/workflows/ci-cd-pims-dev.yml @@ -123,7 +123,7 @@ jobs: - name: call scripts to upgrade database shell: bash run: | - oc process -f ./openshift/4.0/templates/jobs/db-deploy.yaml -p SERVER_NAME=sqldevtst.th.gov.bc.ca -p DB_NAME=PIMS_DEV -p NAMESPACE=3cd915-dev | oc create -f - | grep -oP "(?<=job\.batch/)[^\s]*" | (read JOB_NAME; oc wait --for=condition=complete job/$JOB_NAME --timeout=120s) + oc process -f ./openshift/4.0/templates/jobs/db-deploy.yaml -p DB_SECRET_NAME=pims-database -p GIT_BRANCH=dev -p SERVER_NAME=sqldevtst.th.gov.bc.ca -p DB_NAME=PIMS_DEV -p NAMESPACE=3cd915-dev | oc create -f - | grep -oP "(?<=job\.batch/)[^\s]*" | (read JOB_NAME; oc wait --for=condition=complete job/$JOB_NAME --timeout=120s) sync-keycloak: name: Sync Keycloak diff --git a/.github/workflows/retag-dev-to-test.yml b/.github/workflows/retag-dev-to-test.yml index 4435350eb8..8da2e51f9b 100644 --- a/.github/workflows/retag-dev-to-test.yml +++ b/.github/workflows/retag-dev-to-test.yml @@ -5,7 +5,7 @@ env: OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }} OPENSHIFT_TOOLS_NAMESPACE: "3cd915-tools" MS_TEAMS_WEBHOOK_BUILD_CHANNEL: ${{ secrets.MS_TEAMS_WEBHOOK_URI_BUILD_CHANNEL }} - AUTH__KEYCLOAK__SECRET: ${{ secrets.KEYCLOAK_SECRET }} + AUTH__KEYCLOAK__SECRET: ${{ secrets.KEYCLOAK_SECRET_TEST }} AUTH__KEYCLOAK__SERVICEACCOUNT__SECRET: ${{ secrets.KEYCLOAK_SERVICEACCOUNT_SECRET }} sync-directory: ./tools/keycloak/sync ASPNETCORE_ENVIRONMENT: "Test" @@ -64,9 +64,33 @@ jobs: ./openshift/4.0/player.sh deploy api $DESTINATION -apply ./openshift/4.0/player.sh deploy app $DESTINATION -apply + # the command: + # 1) creates an openshift job with generated name to avoid name conflict, substituting the variables in the template. + # 2) greps the generated name from the previous step. + # 3) waits for the job to complete using the generated name. + database-upgrade: + name: Upgrade database + needs: [deploy] + runs-on: ubuntu-latest + steps: + - name: Checkout Source Code + uses: actions/checkout@v3 + - name: Login to OpenShift + uses: redhat-actions/oc-login@v1 + with: + openshift_server_url: ${{ env.OPENSHIFT_SERVER }} + openshift_token: ${{ env.OPENSHIFT_TOKEN }} + insecure_skip_tls_verify: true + namespace: 3cd915-dev + - name: call scripts to upgrade database + shell: bash + run: | + oc process -f ./openshift/4.0/templates/jobs/db-deploy.yaml -p DB_SECRET_NAME=pims-database-test -p GIT_BRANCH=dev -p SERVER_NAME=sqldevtst.th.gov.bc.ca -p DB_NAME=PIMS_TST -p NAMESPACE=3cd915-dev | oc create -f - | grep -oP "(?<=job\.batch/)[^\s]*" | (read JOB_NAME; oc wait --for=condition=complete job/$JOB_NAME --timeout=120s) + + sync-keycloak: name: Sync Keycloak - needs: deploy + needs: database-upgrade runs-on: ubuntu-latest steps: - name: Checkout Source Code @@ -92,7 +116,7 @@ jobs: ci-cd-end-notification: name: CI-CD End Notification to Teams Channel runs-on: ubuntu-latest - needs: deploy + needs: sync-keycloak steps: - name: check workflow status uses: martialonline/workflow-status@v4 diff --git a/.github/workflows/retag-test-to-uat.yml b/.github/workflows/retag-test-to-uat.yml index 280511f7d4..9adcb162f7 100644 --- a/.github/workflows/retag-test-to-uat.yml +++ b/.github/workflows/retag-test-to-uat.yml @@ -5,6 +5,9 @@ env: OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }} OPENSHIFT_TOOLS_NAMESPACE: "3cd915-tools" MS_TEAMS_WEBHOOK_BUILD_CHANNEL: ${{ secrets.MS_TEAMS_WEBHOOK_URI_BUILD_CHANNEL }} + AUTH__KEYCLOAK__SECRET: ${{ secrets.KEYCLOAK_SECRET_UAT }} + AUTH__KEYCLOAK__SERVICEACCOUNT__SECRET: ${{ secrets.KEYCLOAK_SERVICEACCOUNT_SECRET }} + sync-directory: ./tools/keycloak/sync ## variables for scripts under git\openshift\4.0\scripts\oc-*.sh APP_PORT: 8080 @@ -59,11 +62,59 @@ jobs: run: | ./openshift/4.0/player.sh deploy api $DESTINATION -apply ./openshift/4.0/player.sh deploy app $DESTINATION -apply + + # the command: + # 1) creates an openshift job with generated name to avoid name conflict, substituting the variables in the template. + # 2) greps the generated name from the previous step. + # 3) waits for the job to complete using the generated name. + database-upgrade: + name: Upgrade database + needs: [deploy] + runs-on: ubuntu-latest + steps: + - name: Checkout Source Code + uses: actions/checkout@v3 + - name: Login to OpenShift + uses: redhat-actions/oc-login@v1 + with: + openshift_server_url: ${{ env.OPENSHIFT_SERVER }} + openshift_token: ${{ env.OPENSHIFT_TOKEN }} + insecure_skip_tls_verify: true + namespace: 3cd915-test + - name: call scripts to upgrade database + shell: bash + run: | + oc process -f ./openshift/4.0/templates/jobs/db-deploy.yaml -p DB_SECRET_NAME=pims-database -p GIT_BRANCH=test -p SERVER_NAME=sqlprd.th.gov.bc.ca -p DB_NAME=PIMS_UAT -p NAMESPACE=3cd915-test | oc create -f - | grep -oP "(?<=job\.batch/)[^\s]*" | (read JOB_NAME; oc wait --for=condition=complete job/$JOB_NAME --timeout=120s) + + sync-keycloak: + name: Sync Keycloak + needs: database-upgrade + runs-on: ubuntu-latest + steps: + - name: Checkout Source Code + uses: actions/checkout@v3 + + - name: Setup .NET 6 + uses: actions/setup-dotnet@v2 + with: + dotnet-version: "6.0.x" + + - name: Install dependencies for keycloak sync + run: dotnet restore + working-directory: ${{env.sync-directory}} + + - name: Build keycloak sync + run: dotnet build + working-directory: ${{env.sync-directory}} + + - name: Start keycloak sync + run: dotnet run + working-directory: ${{env.sync-directory}} ci-cd-end-notification: name: CI-CD End Notification to Teams Channel runs-on: ubuntu-latest - needs: deploy + needs: sync-keycloak steps: - name: check workflow status uses: martialonline/workflow-status@v4 @@ -73,6 +124,6 @@ jobs: with: github-token: ${{ github.token }} ms-teams-webhook-uri: ${{ env.MS_TEAMS_WEBHOOK_BUILD_CHANNEL }} - notification-summary: PIMS Release DEV to TEST COMPLETED with status ${{ steps.check.outputs.status }} + notification-summary: PIMS Release TEST to UAT COMPLETED with status ${{ steps.check.outputs.status }} notification-color: 17a2b8 timezone: America/Los_Angeles diff --git a/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION.dtproj b/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION.dtproj index 74fd313a84..bf9a26060b 100644 --- a/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION.dtproj +++ b/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION.dtproj @@ -41,10 +41,10 @@ Package 1 0 - 36 + 47 - {E6831E32-A690-41BA-A665-7CF969E8783F} + {C036C22A-FB08-4CA9-8625-018613EC0B33} 8 @@ -77,7 +77,7 @@ 0 0 0 - Data Source=sqldevtst.th.gov.bc.ca;Initial Catalog=PIMS_TST;Provider=SQLNCLI11.1;Integrated Security=SSPI;Auto Translate=False;Application Name=SSIS-Package-{C854C770-B13C-477E-8DD4-44721E1B9FCF}sqldevtst.th.gov.bc.ca.PIMS_DEV; + Data Source=sqldevtst.th.gov.bc.ca;Initial Catalog=PIMS_DEV;Provider=SQLNCLI11.1;Integrated Security=SSPI;Auto Translate=False;Application Name=SSIS-Package-{C854C770-B13C-477E-8DD4-44721E1B9FCF}sqldevtst.th.gov.bc.ca.PIMS_DEV; 18 @@ -137,7 +137,7 @@ 0 0 0 - PIMS_TST + PIMS_DEV 18 diff --git a/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION.dtproj.user b/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION.dtproj.user index 8f318c130b..5acde15791 100644 --- a/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION.dtproj.user +++ b/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION.dtproj.user @@ -8,6 +8,8 @@ false + ssistst.th.gov.bc.ca + /SSISDB/PIMS_DEV/PIMS_PROPERTY_SPATIAL_CORRECTION false diff --git a/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/Package.dtsx b/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/Package.dtsx index 828a833696..a19f112de1 100644 --- a/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/Package.dtsx +++ b/etl/PIMS_PROPERTY_SPATIAL_CORRECTION/PIMS_PROPERTY_SPATIAL_CORRECTION/Package.dtsx @@ -12,8 +12,8 @@ DTS:ObjectName="Package" DTS:PackageType="5" DTS:ProtectionLevel="0" - DTS:VersionBuild="36" - DTS:VersionGUID="{E6831E32-A690-41BA-A665-7CF969E8783F}"> + DTS:VersionBuild="47" + DTS:VersionGUID="{C036C22A-FB08-4CA9-8625-018613EC0B33}"> 8 @@ -26,7 +26,7 @@ + DTS:ConnectionString="Data Source=sqldevtst.th.gov.bc.ca;Initial Catalog=PIMS_DEV;Provider=SQLNCLI11.1;Integrated Security=SSPI;Auto Translate=False;Application Name=SSIS-Package-{C854C770-B13C-477E-8DD4-44721E1B9FCF}sqldevtst.th.gov.bc.ca.PIMS_DEV;" /> @@ -44,7 +44,7 @@ @@ -137,120 +137,6 @@ - - - 0 - UPDATE PIMS_PROPERTY SET LOCATION = geometry::STGeomFromText(?, 3008), CONCURRENCY_CONTROL_NUMBER = CONCURRENCY_CONTROL_NUMBER + 1 WHERE PROPERTY_ID = ? - 1252 - - - - - - - - - - - - - - 65 - - - - - 81 - - - - - - - - - - - - - - - - - - SC_4664d38d6d774888963e21557351c901 msBuild SC_4664d38d6d774888963e21557351c901 - {19A4DA25-7A35-4FC3-9BDA-B04939D13BB1} + {9DFA908A-4CC6-4FE2-93C1-582187F61F61} @@ -2671,380 +2566,380 @@ namespace Priority_Queue @@ -3156,6 +3051,132 @@ AAAAAAAAAAAAAAA=]]> + + + 0 + [dbo].[TEMP_PROPERTY_LOCATION] + + + 1252 + false + 3 + false + false + TABLOCK,CHECK_CONSTRAINTS,ROWS_PER_BATCH = 1000 + 2147483647 + + + + + + + + + + + + + + + + + + + + + + + + + + startId="Package\Correct PIMS Locations\Script Component.Outputs[Output 0]" /> + + + + + + + + + + + + + + + + + + + + + + + @@ -3189,11 +3284,89 @@ AAAAAAAAAAAAAAA=]]> design-time-name="Package"> + Capacity="8" xmlns="clr-namespace:Microsoft.SqlServer.IntegrationServices.Designer.Model.Serialization;assembly=Microsoft.SqlServer.IntegrationServices.Graph" xmlns:mssgle="clr-namespace:Microsoft.SqlServer.Graph.LayoutEngine;assembly=Microsoft.SqlServer.Graph" xmlns:assembly="http://schemas.microsoft.com/winfx/2006/xaml"> + TopLeft="60,107.5" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3214,13 +3387,9 @@ AAAAAAAAAAAAAAA=]]> Size="159,42" Id="Package\Correct PIMS Locations\Conditional Split" TopLeft="63.5,209.5" /> - + Id="Package\Correct PIMS Locations.Paths[Output 0]" + TopLeft="143,149.5"> - + + Id="Package\Correct PIMS Locations.Paths[OLE DB Source Output]" + TopLeft="143,47.5"> + + Id="Package\Correct PIMS Locations.Paths[VALID_NEW_LOCATION]" + TopLeft="143,251.5"> + End="313.5,67"> + End="0,33.25" /> + + + + - + @@ -3302,5 +3487,18 @@ AAAAAAAAAAAAAAA=]]> + + + + DataSourceViewID + + + TableInfoObjectType + Table + + + ]]> \ No newline at end of file diff --git a/openshift/4.0/templates/jobs/db-deploy.yaml b/openshift/4.0/templates/jobs/db-deploy.yaml index 9734387026..86252df56e 100644 --- a/openshift/4.0/templates/jobs/db-deploy.yaml +++ b/openshift/4.0/templates/jobs/db-deploy.yaml @@ -30,8 +30,8 @@ objects: touch /usr/config/psp/source/database/mssql/.env; PATH="/opt/mssql-tools/bin:$PATH"; cd /usr/config/psp; - git fetch origin dev; - git reset --h origin/dev; + git fetch origin ${GIT_BRANCH}; + git reset --h origin/${GIT_BRANCH}; cd /usr/config/psp/source/database/mssql/scripts/dbscripts; chmod 777 db-upgrade.sh; ./db-upgrade.sh; @@ -43,12 +43,12 @@ objects: - name: DB_USER valueFrom: secretKeyRef: - name: pims-database + name: ${DB_SECRET_NAME} key: DB_USER - name: DB_PASSWORD valueFrom: secretKeyRef: - name: pims-database + name: ${DB_SECRET_NAME} key: DB_PASSWORD resources: {} terminationMessagePath: /dev/termination-log @@ -65,3 +65,9 @@ parameters: - name: NAMESPACE description: the openshift namespace to run this job in. required: true + - name: GIT_BRANCH + description: the git branch to pull the database scripts from. + required: true + - name: DB_SECRET_NAME + description: the git branch to pull the database scripts from. + required: true diff --git a/source/backend/dal/Repositories/PropertyRepository.cs b/source/backend/dal/Repositories/PropertyRepository.cs index 53b2849a72..6be4a2230a 100644 --- a/source/backend/dal/Repositories/PropertyRepository.cs +++ b/source/backend/dal/Repositories/PropertyRepository.cs @@ -451,7 +451,7 @@ public HashSet GetMatchingIds(PropertyFilterCriteria filter) if (!string.IsNullOrEmpty(filter.LeasePayRcvblType)) { query = query.Where(p => - p.PimsPropertyLeases.Any(pl => pl.Lease.LeasePayRvblTypeCode == filter.LeasePayRcvblType || filter.LeasePayRcvblType == "all")); + p.PimsPropertyLeases.Any(pl => (pl.Lease.LeasePayRvblTypeCode == filter.LeasePayRcvblType || filter.LeasePayRcvblType == "all") && (pl.Lease.OrigExpiryDate >= DateTime.Now.Date || pl.Lease.PimsLeaseTerms.Any(t => t.TermExpiryDate == null || t.TermExpiryDate >= DateTime.Now.Date)))); } // Anomalies diff --git a/source/backend/tests/unit/dal/Repositories/PropertyRepositoryTest.cs b/source/backend/tests/unit/dal/Repositories/PropertyRepositoryTest.cs index a3bfb687e1..3a1ac57e48 100644 --- a/source/backend/tests/unit/dal/Repositories/PropertyRepositoryTest.cs +++ b/source/backend/tests/unit/dal/Repositories/PropertyRepositoryTest.cs @@ -140,6 +140,7 @@ public void GetMatchingIds_LeaseRcbvl_All_Success() var repository = CreateRepositoryWithPermissions(Permissions.PropertyView); var property = EntityHelper.CreateProperty(100); var lease = EntityHelper.CreateLease(1, addProperty:false); + lease.OrigExpiryDate = DateTime.Now.AddDays(1); property.PimsPropertyLeases.Add(new PimsPropertyLease() { PropertyId = property.Internal_Id, LeaseId = lease.Internal_Id, Lease = lease }); _helper.AddAndSaveChanges(property); diff --git a/source/frontend/src/features/mapSideBar/acquisition/common/models.ts b/source/frontend/src/features/mapSideBar/acquisition/common/models.ts index e4cc32ec86..f539b9d4dd 100644 --- a/source/frontend/src/features/mapSideBar/acquisition/common/models.ts +++ b/source/frontend/src/features/mapSideBar/acquisition/common/models.ts @@ -1,4 +1,4 @@ -import { isEmpty } from 'lodash'; +import { isEmpty, isNumber } from 'lodash'; import { fromApiOrganization, fromApiPerson, IContactSearchResult } from '@/interfaces'; import { Api_AcquisitionFileOwner, Api_AcquisitionFileTeam } from '@/models/api/AcquisitionFile'; @@ -42,9 +42,10 @@ export class AcquisitionTeamFormModel { person: undefined, organizationId: organizationId ?? undefined, organization: undefined, - primaryContactId: Number.isFinite(this.primaryContactId) - ? Number(this.primaryContactId) - : undefined, + primaryContactId: + !!this.primaryContactId && isNumber(+this.primaryContactId) + ? Number(this.primaryContactId) + : undefined, teamProfileType: toTypeCode(this.contactTypeCode), teamProfileTypeCode: this.contactTypeCode, }; diff --git a/source/frontend/src/features/mapSideBar/property/tabs/propertyDetailsManagement/activity/edit/PropertyActivityEditForm.tsx b/source/frontend/src/features/mapSideBar/property/tabs/propertyDetailsManagement/activity/edit/PropertyActivityEditForm.tsx index 73b356736e..67ccb5f9a1 100644 --- a/source/frontend/src/features/mapSideBar/property/tabs/propertyDetailsManagement/activity/edit/PropertyActivityEditForm.tsx +++ b/source/frontend/src/features/mapSideBar/property/tabs/propertyDetailsManagement/activity/edit/PropertyActivityEditForm.tsx @@ -52,6 +52,9 @@ export const PropertyActivityEditForm: React.FunctionComponent< const initialForm = useMemo(() => { const initialModel = PropertyActivityFormModel.fromApi(props.activity); + if (props.activity === undefined) { + initialModel.activityStatusCode = 'NOTSTARTED'; + } setActivityType(initialModel.activityTypeCode); return initialModel; }, [props.activity]);