From 4aeb172a90b8f68808488423ee5f2589fb300f71 Mon Sep 17 00:00:00 2001 From: abimichel Date: Wed, 24 Jul 2024 17:56:46 -0700 Subject: [PATCH] organize folders --- ...ish-pmt.yml => docker-publish-pmt-dbt.yml} | 31 +- README.md | 21 +- logs/dbt.log | 55 - .../dag}/pmt_dlh_load_dal_extract.py | 0 .../dag}/pmt_dlh_load_facts_dims.py | 0 {pmt => permitting}/dbt/logs/dbt.log | 0 {pmt => permitting}/dbt/pmt_dbt/.gitignore | 1 - {pmt => permitting}/dbt/pmt_dbt/.user.yml | 0 {pmt => permitting}/dbt/pmt_dbt/Dockerfile | 0 {pmt => permitting}/dbt/pmt_dbt/README.md | 0 .../dbt/pmt_dbt/analyses/.gitkeep | 0 .../dbt/pmt_dbt/dbt_project.yml | 0 .../dbt/pmt_dbt/macros/.gitkeep | 0 .../pmt_dbt/macros/generate_schema_name.sql | 0 .../models/example/my_first_dbt_model.sql | 0 .../models/example/my_second_dbt_model.sql | 0 .../dbt/pmt_dbt/models/example/schema.yml | 0 {pmt => permitting}/dbt/pmt_dbt/profiles.yml | 0 .../dbt/pmt_dbt/seeds/.gitkeep | 0 .../dbt/pmt_dbt/snapshots/.gitkeep | 0 .../pmt_dbt/snapshots/dal/extract_permits.sql | 216 ++-- .../dpl/dim_authorization_status.sql | 128 +- .../snapshots/dpl/dim_business_area.sql | 56 +- .../pmt_dbt/snapshots/dpl/dim_location.sql | 116 +- .../pmt_dbt/snapshots/dpl/dim_ministry.sql | 114 +- .../dbt/pmt_dbt/snapshots/dpl/dim_org.sql | 72 +- .../snapshots/dpl/dim_permit_status.sql | 116 +- .../pmt_dbt/snapshots/dpl/dim_permit_type.sql | 120 +- .../dbt/pmt_dbt/snapshots/dpl/dim_project.sql | 142 +-- .../snapshots/dpl/dim_source_system.sql | 70 +- .../pmt_dbt/snapshots/dpl/fact_permits.sql | 1078 ++++++++--------- .../dbt/pmt_dbt/snapshots/dpl/qa_dpl_pmt.yml | 0 .../dbt/pmt_dbt/snapshots/dpl/src_pmt.yml | 0 .../dbt/pmt_dbt/tests/.gitkeep | 0 {pmt/DDL => permitting/ddl}/schema.sql | 0 {pmt/DDL => permitting/ddl}/tables.sql | 460 +++---- {pmt/DDL => permitting/ddl}/views.sql | 0 37 files changed, 1377 insertions(+), 1419 deletions(-) rename .github/workflows/{dbt-docker-publish-pmt.yml => docker-publish-pmt-dbt.yml} (66%) delete mode 100644 logs/dbt.log rename {pmt/af/DAG => permitting/dag}/pmt_dlh_load_dal_extract.py (100%) rename {pmt/af/DAG => permitting/dag}/pmt_dlh_load_facts_dims.py (100%) rename {pmt => permitting}/dbt/logs/dbt.log (100%) rename {pmt => permitting}/dbt/pmt_dbt/.gitignore (69%) rename {pmt => permitting}/dbt/pmt_dbt/.user.yml (100%) rename {pmt => permitting}/dbt/pmt_dbt/Dockerfile (100%) rename {pmt => permitting}/dbt/pmt_dbt/README.md (100%) rename {pmt => permitting}/dbt/pmt_dbt/analyses/.gitkeep (100%) rename {pmt => permitting}/dbt/pmt_dbt/dbt_project.yml (100%) rename {pmt => permitting}/dbt/pmt_dbt/macros/.gitkeep (100%) rename {pmt => permitting}/dbt/pmt_dbt/macros/generate_schema_name.sql (100%) rename {pmt => permitting}/dbt/pmt_dbt/models/example/my_first_dbt_model.sql (100%) rename {pmt => permitting}/dbt/pmt_dbt/models/example/my_second_dbt_model.sql (100%) rename {pmt => permitting}/dbt/pmt_dbt/models/example/schema.yml (100%) rename {pmt => permitting}/dbt/pmt_dbt/profiles.yml (100%) rename {pmt => permitting}/dbt/pmt_dbt/seeds/.gitkeep (100%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/.gitkeep (100%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dal/extract_permits.sql (97%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/dim_authorization_status.sql (96%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/dim_business_area.sql (98%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/dim_location.sql (96%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/dim_ministry.sql (98%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/dim_org.sql (97%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/dim_permit_status.sql (96%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/dim_permit_type.sql (96%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/dim_project.sql (96%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/dim_source_system.sql (95%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/fact_permits.sql (98%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/qa_dpl_pmt.yml (100%) rename {pmt => permitting}/dbt/pmt_dbt/snapshots/dpl/src_pmt.yml (100%) rename {pmt => permitting}/dbt/pmt_dbt/tests/.gitkeep (100%) rename {pmt/DDL => permitting/ddl}/schema.sql (100%) rename {pmt/DDL => permitting/ddl}/tables.sql (97%) rename {pmt/DDL => permitting/ddl}/views.sql (100%) diff --git a/.github/workflows/dbt-docker-publish-pmt.yml b/.github/workflows/docker-publish-pmt-dbt.yml similarity index 66% rename from .github/workflows/dbt-docker-publish-pmt.yml rename to .github/workflows/docker-publish-pmt-dbt.yml index 7a17c0e..8b720db 100644 --- a/.github/workflows/dbt-docker-publish-pmt.yml +++ b/.github/workflows/docker-publish-pmt-dbt.yml @@ -2,15 +2,12 @@ name: Push to GHCR on: push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] + branches: ["main" ] env: - # Vish: image name is ghcr.io/bcgov/nr-dap-dlh-pmt:main REGISTRY: ghcr.io - DBT_PATH: pmt/dbt/pmt_dbt/ - IMAGE_NAME: ${{ github.repository }}-pmt + DOCKERFILE_PATH: permitting/dbt/pmt_dbt/ + IMAGE_NAME: ${{ github.repository }}-pmt-dbt jobs: build: @@ -26,9 +23,7 @@ jobs: - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 - with: - cosign-release: 'v2.1.1' + uses: sigstore/cosign-installer@v3.3.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 @@ -51,19 +46,19 @@ jobs: id: build-and-push uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 with: - # Vish: adding this to help the action find the Dockerfile to build from - context: ${{ env.DBT_PATH }}/ + # DF-NOTE: to help the action find the Dockerfile to build from + context: ${{ env.DOCKERFILE_PATH }}/ push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - #- name: Sign the published Docker image - #if: ${{ github.event_name != 'pull_request' }} - #env: + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable - #TAGS: ${{ steps.meta.outputs.tags }} - #DIGEST: ${{ steps.build-and-push.outputs.digest }} - - #run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/README.md b/README.md index a187078..d2cd90a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ # nr-dap-dlh -Holds the objects for the data lakehouse part of the data analytics platform. +This repository holds code and artifacts for the Data Analytics Platform (DAP). + +The Data Lakehouse (DLH) allows for the summarization, transformation and combination of structured and semi-structured data, including the consumption of replicated data assets from the Operation Data Store (ODS), APIs and Object Storage. + +## File Structure + +- **ETL Tools (such as Data Build Tool (DBT) projects):** + `nr-dap-dlh/{domain}/{name of tool}/{Files}` + +- **Python or R Scripts** + `nr-dap-dlh/{domain}/scripts/{Files}` + +- **DDL (Data Definition Language)** Includes commands like CREATE, ALTER, and COMMENT that define and modify database structure. + `nr-dap-dlh/{domain}/ddl/{Filename = name of the table}` + +- **DQL (Data Query Language) / DML (Data Manipulation Language)** Includes commands like SELECT, INSERT, UPDATE, and DELETE that manipulate data within the database. + `nr-dap-dlh/{domain}/dml/{Files}` + +- **DCL (Data Control Language)** Includes commands like GRANT and REVOKE that control access to the data in the database. + `nr-dap-dlh/{domain}/dcl/{Files}` \ No newline at end of file diff --git a/logs/dbt.log b/logs/dbt.log deleted file mode 100644 index c369782..0000000 --- a/logs/dbt.log +++ /dev/null @@ -1,55 +0,0 @@ -13:45:48.463405 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [, , ]} - - -============================== 13:45:48.467412 | 080aa687-e9bf-47e0-9b51-2cd1e020ef1d ============================== -13:45:48.467412 [info ] [MainThread]: Running with dbt=1.6.2 -13:45:48.468404 [debug] [MainThread]: running dbt with arguments {'printer_width': '80', 'indirect_selection': 'eager', 'write_json': 'True', 'log_cache_events': 'False', 'partial_parse': 'True', 'cache_selected_only': 'False', 'profiles_dir': 'C:\\Users\\VREDDY\\.dbt', 'debug': 'False', 'fail_fast': 'False', 'log_path': 'logs', 'version_check': 'True', 'warn_error': 'None', 'use_colors': 'True', 'use_experimental_parser': 'False', 'no_print': 'None', 'quiet': 'False', 'log_format': 'default', 'static_parser': 'True', 'introspect': 'True', 'warn_error_options': 'WarnErrorOptions(include=[], exclude=[])', 'target_path': 'None', 'invocation_command': 'dbt init', 'send_anonymous_usage_stats': 'True'} -13:46:04.910652 [debug] [MainThread]: Starter project path: C:\Users\VREDDY\AppData\Local\Programs\Python\Python311\Lib\site-packages\dbt\include\starter_project -13:46:04.962637 [info ] [MainThread]: -Your new dbt project "pmt_dbt" was created! - -For more information on how to configure the profiles.yml file, -please consult the dbt documentation here: - - https://docs.getdbt.com/docs/configure-your-profile - -One more thing: - -Need help? Don't hesitate to reach out to us via GitHub issues or on Slack: - - https://community.getdbt.com/ - -Happy modeling! - -13:46:04.963648 [info ] [MainThread]: Setting up your profile. -13:46:15.758932 [debug] [MainThread]: Command `dbt init` succeeded at 13:46:15.758932 after 27.32 seconds -13:46:15.758932 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [, , ]} -13:46:15.759950 [debug] [MainThread]: Flushing usage events -14:37:37.560373 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [, , ]} - - -============================== 14:37:37.564380 | a5264c3f-486c-42e9-be6b-2ac0a72f5e49 ============================== -14:37:37.564380 [info ] [MainThread]: Running with dbt=1.6.2 -14:37:37.564380 [debug] [MainThread]: running dbt with arguments {'printer_width': '80', 'indirect_selection': 'eager', 'write_json': 'True', 'log_cache_events': 'False', 'partial_parse': 'True', 'cache_selected_only': 'False', 'profiles_dir': 'C:\\Users\\VREDDY\\.dbt', 'debug': 'False', 'warn_error': 'None', 'log_path': 'logs', 'version_check': 'True', 'fail_fast': 'False', 'use_colors': 'True', 'use_experimental_parser': 'False', 'no_print': 'None', 'quiet': 'False', 'warn_error_options': 'WarnErrorOptions(include=[], exclude=[])', 'introspect': 'True', 'static_parser': 'True', 'log_format': 'default', 'target_path': 'None', 'invocation_command': 'dbt snapshot --select fact_permits', 'send_anonymous_usage_stats': 'True'} -14:37:37.565373 [error] [MainThread]: Encountered an error: -Runtime Error - No dbt_project.yml found at expected path C:\Users\VREDDY\Softwares\nr-dap-dlh\dbt_project.yml - Verify that each entry within packages.yml (and their transitive dependencies) contains a file named dbt_project.yml - -14:37:37.566372 [debug] [MainThread]: Command `dbt snapshot` failed at 14:37:37.566372 after 0.04 seconds -14:37:37.566372 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [, , ]} -14:37:37.566372 [debug] [MainThread]: Flushing usage events -13:17:47.324010 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [, , ]} - - -============================== 13:17:47.334168 | 82b3665c-e5af-42e3-bce5-e00a293fe9d5 ============================== -13:17:47.334168 [info ] [MainThread]: Running with dbt=1.6.2 -13:17:47.334168 [debug] [MainThread]: running dbt with arguments {'printer_width': '80', 'indirect_selection': 'eager', 'write_json': 'True', 'log_cache_events': 'False', 'partial_parse': 'True', 'cache_selected_only': 'False', 'profiles_dir': 'C:\\Users\\VREDDY\\.dbt', 'fail_fast': 'False', 'debug': 'False', 'log_path': 'logs', 'version_check': 'True', 'warn_error': 'None', 'use_colors': 'True', 'use_experimental_parser': 'False', 'no_print': 'None', 'quiet': 'False', 'log_format': 'default', 'invocation_command': 'dbt docs generate', 'static_parser': 'True', 'warn_error_options': 'WarnErrorOptions(include=[], exclude=[])', 'target_path': 'None', 'introspect': 'True', 'send_anonymous_usage_stats': 'True'} -13:17:47.335148 [error] [MainThread]: Encountered an error: -Runtime Error - No dbt_project.yml found at expected path C:\Users\VREDDY\Softwares\nr-dap-dlh\dbt_project.yml - Verify that each entry within packages.yml (and their transitive dependencies) contains a file named dbt_project.yml - -13:17:47.336167 [debug] [MainThread]: Command `dbt docs generate` failed at 13:17:47.336167 after 0.04 seconds -13:17:47.336167 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [, , ]} -13:17:47.336167 [debug] [MainThread]: Flushing usage events diff --git a/pmt/af/DAG/pmt_dlh_load_dal_extract.py b/permitting/dag/pmt_dlh_load_dal_extract.py similarity index 100% rename from pmt/af/DAG/pmt_dlh_load_dal_extract.py rename to permitting/dag/pmt_dlh_load_dal_extract.py diff --git a/pmt/af/DAG/pmt_dlh_load_facts_dims.py b/permitting/dag/pmt_dlh_load_facts_dims.py similarity index 100% rename from pmt/af/DAG/pmt_dlh_load_facts_dims.py rename to permitting/dag/pmt_dlh_load_facts_dims.py diff --git a/pmt/dbt/logs/dbt.log b/permitting/dbt/logs/dbt.log similarity index 100% rename from pmt/dbt/logs/dbt.log rename to permitting/dbt/logs/dbt.log diff --git a/pmt/dbt/pmt_dbt/.gitignore b/permitting/dbt/pmt_dbt/.gitignore similarity index 69% rename from pmt/dbt/pmt_dbt/.gitignore rename to permitting/dbt/pmt_dbt/.gitignore index c1c12b2..49f147c 100644 --- a/pmt/dbt/pmt_dbt/.gitignore +++ b/permitting/dbt/pmt_dbt/.gitignore @@ -2,4 +2,3 @@ target/ dbt_packages/ logs/ -profiles.yml diff --git a/pmt/dbt/pmt_dbt/.user.yml b/permitting/dbt/pmt_dbt/.user.yml similarity index 100% rename from pmt/dbt/pmt_dbt/.user.yml rename to permitting/dbt/pmt_dbt/.user.yml diff --git a/pmt/dbt/pmt_dbt/Dockerfile b/permitting/dbt/pmt_dbt/Dockerfile similarity index 100% rename from pmt/dbt/pmt_dbt/Dockerfile rename to permitting/dbt/pmt_dbt/Dockerfile diff --git a/pmt/dbt/pmt_dbt/README.md b/permitting/dbt/pmt_dbt/README.md similarity index 100% rename from pmt/dbt/pmt_dbt/README.md rename to permitting/dbt/pmt_dbt/README.md diff --git a/pmt/dbt/pmt_dbt/analyses/.gitkeep b/permitting/dbt/pmt_dbt/analyses/.gitkeep similarity index 100% rename from pmt/dbt/pmt_dbt/analyses/.gitkeep rename to permitting/dbt/pmt_dbt/analyses/.gitkeep diff --git a/pmt/dbt/pmt_dbt/dbt_project.yml b/permitting/dbt/pmt_dbt/dbt_project.yml similarity index 100% rename from pmt/dbt/pmt_dbt/dbt_project.yml rename to permitting/dbt/pmt_dbt/dbt_project.yml diff --git a/pmt/dbt/pmt_dbt/macros/.gitkeep b/permitting/dbt/pmt_dbt/macros/.gitkeep similarity index 100% rename from pmt/dbt/pmt_dbt/macros/.gitkeep rename to permitting/dbt/pmt_dbt/macros/.gitkeep diff --git a/pmt/dbt/pmt_dbt/macros/generate_schema_name.sql b/permitting/dbt/pmt_dbt/macros/generate_schema_name.sql similarity index 100% rename from pmt/dbt/pmt_dbt/macros/generate_schema_name.sql rename to permitting/dbt/pmt_dbt/macros/generate_schema_name.sql diff --git a/pmt/dbt/pmt_dbt/models/example/my_first_dbt_model.sql b/permitting/dbt/pmt_dbt/models/example/my_first_dbt_model.sql similarity index 100% rename from pmt/dbt/pmt_dbt/models/example/my_first_dbt_model.sql rename to permitting/dbt/pmt_dbt/models/example/my_first_dbt_model.sql diff --git a/pmt/dbt/pmt_dbt/models/example/my_second_dbt_model.sql b/permitting/dbt/pmt_dbt/models/example/my_second_dbt_model.sql similarity index 100% rename from pmt/dbt/pmt_dbt/models/example/my_second_dbt_model.sql rename to permitting/dbt/pmt_dbt/models/example/my_second_dbt_model.sql diff --git a/pmt/dbt/pmt_dbt/models/example/schema.yml b/permitting/dbt/pmt_dbt/models/example/schema.yml similarity index 100% rename from pmt/dbt/pmt_dbt/models/example/schema.yml rename to permitting/dbt/pmt_dbt/models/example/schema.yml diff --git a/pmt/dbt/pmt_dbt/profiles.yml b/permitting/dbt/pmt_dbt/profiles.yml similarity index 100% rename from pmt/dbt/pmt_dbt/profiles.yml rename to permitting/dbt/pmt_dbt/profiles.yml diff --git a/pmt/dbt/pmt_dbt/seeds/.gitkeep b/permitting/dbt/pmt_dbt/seeds/.gitkeep similarity index 100% rename from pmt/dbt/pmt_dbt/seeds/.gitkeep rename to permitting/dbt/pmt_dbt/seeds/.gitkeep diff --git a/pmt/dbt/pmt_dbt/snapshots/.gitkeep b/permitting/dbt/pmt_dbt/snapshots/.gitkeep similarity index 100% rename from pmt/dbt/pmt_dbt/snapshots/.gitkeep rename to permitting/dbt/pmt_dbt/snapshots/.gitkeep diff --git a/pmt/dbt/pmt_dbt/snapshots/dal/extract_permits.sql b/permitting/dbt/pmt_dbt/snapshots/dal/extract_permits.sql similarity index 97% rename from pmt/dbt/pmt_dbt/snapshots/dal/extract_permits.sql rename to permitting/dbt/pmt_dbt/snapshots/dal/extract_permits.sql index e869913..93531d6 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dal/extract_permits.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dal/extract_permits.sql @@ -1,109 +1,109 @@ -{% snapshot extract_permits %} - - {{ - config( - target_schema='pmt_dal', - strategy='check', - unique_key='unqid', - check_cols=['ministry_code','business_area_code','permit_status_code','permit_status_date','permit_issue_date','app_received_date','app_update_date','app_decision_date','app_issuance_date','app_accepted_date','app_rejected_date','app_adjudication_date','harvest_area_sq_m','harvest_auth_status','map_feature_id','permit_type_code','src_sys_description','ministry_description','business_area_description','authorization_status_description','permit_status_description','permit_type_description'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with extract_data as ( - SELECT fpt.src_sys_code, - ss.src_sys_description, - fpt.ministry_code, - mnt.ministry_description, - fpt.business_area_code, - bar.business_area_description, - fpt.application_id, - fpt.permit_id, - fpt.app_file_id, - fpt.authorization_status_code, - ast.authorization_status_description, - fpt.project_id, - prj.project_name, - fpt.permit_status_code, - pst.permit_status_description, - fpt.permit_status_date, - fpt.permit_issue_date, - fpt.permit_issue_expire_date, - fpt.app_received_date, - fpt.app_update_date, - fpt.app_decision_date, - fpt.app_issuance_date, - fpt.app_accepted_date, - fpt.app_rejected_date, - fpt.app_adjudication_date, - fpt.harvest_auth_status, - fpt.harvest_area_sq_m, - fpt.permit_org_unit_code, - case when fpt.src_sys_code = 'ATS' then null else org.org_unit_description end as district_description, - case when fpt.src_sys_code = 'ATS' then null else org.org_unit_code end as district_code, - case when fpt.src_sys_code = 'ATS' then null else org.rollup_district_description end as rollup_district_description, - case when fpt.src_sys_code = 'ATS' then null else org.rollup_district_code end as rollup_district_code, - case when fpt.src_sys_code = 'ATS' then case when atsorg.org_unit_description is not null then atsorg.org_unit_description else - prj.project_region_description end else org.rollup_region_description end as region_name, - case when fpt.src_sys_code = 'ATS' then atsorg.org_unit_code else org.rollup_region_code end as region_code, - case when fpt.src_sys_code = 'ATS' then atsorg.roll_up_area_code else org.roll_up_area_code end as area_code, - case when fpt.src_sys_code = 'ATS' then atsorg.roll_up_area_description else org.roll_up_area_description end as area_description, - fpt.application_age, - fpt.map_feature_id, - fpt.permit_type_code, - ptp.permit_type_description, - fpt.unqid - FROM {{ ref('fact_permits') }} fpt - left join {{ ref('dim_authorization_status') }} ast - on (fpt.authorization_status_code = ast.authorization_status_code - and ast.src_sys_code = fpt.src_sys_code - and fpt.dbt_valid_to is null - and ast.dbt_valid_to is null) - left join {{ ref('dim_business_area') }} bar - on (bar.business_area_code = fpt.business_area_code - and bar.src_sys_code = 'CORP' - and fpt.dbt_valid_to is null - and bar.dbt_valid_to is null) - left join {{ ref('dim_ministry') }} mnt - on (mnt.ministry_code = fpt.ministry_code - and mnt.src_sys_code = 'CORP' - and fpt.dbt_valid_to is null - and mnt.dbt_valid_to is null) - left join {{ ref('dim_org') }} org - on (org.org_unit_code = fpt.permit_org_unit_code - and org.src_sys_code = 'CORP' - and fpt.dbt_valid_to is null - and org.dbt_valid_to is null) - left join {{ ref('dim_permit_status') }} pst - on (pst.permit_status_code = fpt.permit_status_code - and pst.src_sys_code = fpt.src_sys_code - and fpt.dbt_valid_to is null - and pst.dbt_valid_to is null) - left join {{ ref('dim_permit_type') }} ptp - on (ptp.permit_type_code = fpt.permit_type_code - and ptp.src_sys_code = fpt.src_sys_code - and fpt.dbt_valid_to is null - and ptp.dbt_valid_to is null) - left join {{ ref('dim_project') }} prj - on (prj.project_id = fpt.project_id - and prj.src_sys_code = fpt.src_sys_code - and fpt.dbt_valid_to is null - and prj.dbt_valid_to is null) - left join {{ ref('dim_org') }} atsorg - on (replace(atsorg.org_unit_description,' Natural Resource Region','') = case when prj.project_region_description = 'Kootenay Boundary' then 'Kootenay-Boundary' - when prj.project_region_description ='Thompson Okanagan' then 'Thompson-Okanagan' - when prj.project_region_description ='North East' then 'Northeast' - else prj.project_region_description end - and atsorg.src_sys_code = 'CORP' - and atsorg.org_unit_description !~'OBSOLETE' and atsorg.org_unit_description like '%Natural Resource Region%' - and atsorg.dbt_valid_to is null) - left join {{ ref('dim_source_system') }} ss - on (ss.src_sys_code = fpt.src_sys_code - and fpt.dbt_valid_to is null - and ss.dbt_valid_to is null) -) ---insert into pmt_dal.extract_permits -Select * from extract_data - +{% snapshot extract_permits %} + + {{ + config( + target_schema='pmt_dal', + strategy='check', + unique_key='unqid', + check_cols=['ministry_code','business_area_code','permit_status_code','permit_status_date','permit_issue_date','app_received_date','app_update_date','app_decision_date','app_issuance_date','app_accepted_date','app_rejected_date','app_adjudication_date','harvest_area_sq_m','harvest_auth_status','map_feature_id','permit_type_code','src_sys_description','ministry_description','business_area_description','authorization_status_description','permit_status_description','permit_type_description'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with extract_data as ( + SELECT fpt.src_sys_code, + ss.src_sys_description, + fpt.ministry_code, + mnt.ministry_description, + fpt.business_area_code, + bar.business_area_description, + fpt.application_id, + fpt.permit_id, + fpt.app_file_id, + fpt.authorization_status_code, + ast.authorization_status_description, + fpt.project_id, + prj.project_name, + fpt.permit_status_code, + pst.permit_status_description, + fpt.permit_status_date, + fpt.permit_issue_date, + fpt.permit_issue_expire_date, + fpt.app_received_date, + fpt.app_update_date, + fpt.app_decision_date, + fpt.app_issuance_date, + fpt.app_accepted_date, + fpt.app_rejected_date, + fpt.app_adjudication_date, + fpt.harvest_auth_status, + fpt.harvest_area_sq_m, + fpt.permit_org_unit_code, + case when fpt.src_sys_code = 'ATS' then null else org.org_unit_description end as district_description, + case when fpt.src_sys_code = 'ATS' then null else org.org_unit_code end as district_code, + case when fpt.src_sys_code = 'ATS' then null else org.rollup_district_description end as rollup_district_description, + case when fpt.src_sys_code = 'ATS' then null else org.rollup_district_code end as rollup_district_code, + case when fpt.src_sys_code = 'ATS' then case when atsorg.org_unit_description is not null then atsorg.org_unit_description else + prj.project_region_description end else org.rollup_region_description end as region_name, + case when fpt.src_sys_code = 'ATS' then atsorg.org_unit_code else org.rollup_region_code end as region_code, + case when fpt.src_sys_code = 'ATS' then atsorg.roll_up_area_code else org.roll_up_area_code end as area_code, + case when fpt.src_sys_code = 'ATS' then atsorg.roll_up_area_description else org.roll_up_area_description end as area_description, + fpt.application_age, + fpt.map_feature_id, + fpt.permit_type_code, + ptp.permit_type_description, + fpt.unqid + FROM {{ ref('fact_permits') }} fpt + left join {{ ref('dim_authorization_status') }} ast + on (fpt.authorization_status_code = ast.authorization_status_code + and ast.src_sys_code = fpt.src_sys_code + and fpt.dbt_valid_to is null + and ast.dbt_valid_to is null) + left join {{ ref('dim_business_area') }} bar + on (bar.business_area_code = fpt.business_area_code + and bar.src_sys_code = 'CORP' + and fpt.dbt_valid_to is null + and bar.dbt_valid_to is null) + left join {{ ref('dim_ministry') }} mnt + on (mnt.ministry_code = fpt.ministry_code + and mnt.src_sys_code = 'CORP' + and fpt.dbt_valid_to is null + and mnt.dbt_valid_to is null) + left join {{ ref('dim_org') }} org + on (org.org_unit_code = fpt.permit_org_unit_code + and org.src_sys_code = 'CORP' + and fpt.dbt_valid_to is null + and org.dbt_valid_to is null) + left join {{ ref('dim_permit_status') }} pst + on (pst.permit_status_code = fpt.permit_status_code + and pst.src_sys_code = fpt.src_sys_code + and fpt.dbt_valid_to is null + and pst.dbt_valid_to is null) + left join {{ ref('dim_permit_type') }} ptp + on (ptp.permit_type_code = fpt.permit_type_code + and ptp.src_sys_code = fpt.src_sys_code + and fpt.dbt_valid_to is null + and ptp.dbt_valid_to is null) + left join {{ ref('dim_project') }} prj + on (prj.project_id = fpt.project_id + and prj.src_sys_code = fpt.src_sys_code + and fpt.dbt_valid_to is null + and prj.dbt_valid_to is null) + left join {{ ref('dim_org') }} atsorg + on (replace(atsorg.org_unit_description,' Natural Resource Region','') = case when prj.project_region_description = 'Kootenay Boundary' then 'Kootenay-Boundary' + when prj.project_region_description ='Thompson Okanagan' then 'Thompson-Okanagan' + when prj.project_region_description ='North East' then 'Northeast' + else prj.project_region_description end + and atsorg.src_sys_code = 'CORP' + and atsorg.org_unit_description !~'OBSOLETE' and atsorg.org_unit_description like '%Natural Resource Region%' + and atsorg.dbt_valid_to is null) + left join {{ ref('dim_source_system') }} ss + on (ss.src_sys_code = fpt.src_sys_code + and fpt.dbt_valid_to is null + and ss.dbt_valid_to is null) +) +--insert into pmt_dal.extract_permits +Select * from extract_data + {% endsnapshot %} \ No newline at end of file diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_authorization_status.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_authorization_status.sql similarity index 96% rename from pmt/dbt/pmt_dbt/snapshots/dpl/dim_authorization_status.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/dim_authorization_status.sql index f7f3f08..de55028 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_authorization_status.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_authorization_status.sql @@ -1,64 +1,64 @@ -{% snapshot dim_authorization_status %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['authorization_status_description'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with ats_data as ( -Select 'ATS' as src_sys_code -,aasc.authorization_status_code as authorization_status_code -,aasc.name as authorization_status_description -,'ATS' || '|'|| coalesce(cast(aasc.authorization_status_code as varchar),'~') as unqid -from {{source ('ats','ats_authorization_status_codes') }} aasc -where 1=1 -), -fta_data as ( - Select distinct - 'FTA' as src_sys_code - ,tasc.tenure_application_state_code as authorization_status_code - ,tasc.description AS authorization_status_description - ,'FTA' || '|'|| coalesce(cast(tasc.tenure_application_state_code as varchar),'~') as unqid - FROM {{source ('fta','tenure_application_state_code') }} tasc -) -,rrs_rp_data as ( - Select distinct - 'RRS_RP' as src_sys_code - ,rasc.road_application_status_code as authorization_status_code - ,rasc.description AS authorization_status_description - ,'RRS_RP' || '|'|| coalesce(cast(rasc.road_application_status_code as varchar),'~') as unqid - FROM {{source ('rrs','road_application_status_code') }} rasc -) -,rrs_rup_data as ( - Select distinct - 'RRS_RUP' as src_sys_code - ,rs.submission_status_code as authorization_status_code - ,ssc.description AS authorization_status_description - ,'RRS_RUP' || '|'|| coalesce(cast(rs.submission_status_code as varchar),'~') as unqid - FROM {{source ('rrs','road_submission') }} rs - left join {{source ('rrs','submission_status_code') }} ssc ON (rs.submission_status_code = ssc.submission_status_code) - where 1=1 - and rs.road_submission_type_code = 'RUP' - and rs.submission_status_code is not null -) ---insert into pmt_dpl.dim_authorization_status -select * - from ats_data - -union ALL -select * from fta_data - -union ALL -select * from rrs_rup_data - -union ALL -select * from rrs_rp_data - - -{% endsnapshot %} +{% snapshot dim_authorization_status %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['authorization_status_description'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with ats_data as ( +Select 'ATS' as src_sys_code +,aasc.authorization_status_code as authorization_status_code +,aasc.name as authorization_status_description +,'ATS' || '|'|| coalesce(cast(aasc.authorization_status_code as varchar),'~') as unqid +from {{source ('ats','ats_authorization_status_codes') }} aasc +where 1=1 +), +fta_data as ( + Select distinct + 'FTA' as src_sys_code + ,tasc.tenure_application_state_code as authorization_status_code + ,tasc.description AS authorization_status_description + ,'FTA' || '|'|| coalesce(cast(tasc.tenure_application_state_code as varchar),'~') as unqid + FROM {{source ('fta','tenure_application_state_code') }} tasc +) +,rrs_rp_data as ( + Select distinct + 'RRS_RP' as src_sys_code + ,rasc.road_application_status_code as authorization_status_code + ,rasc.description AS authorization_status_description + ,'RRS_RP' || '|'|| coalesce(cast(rasc.road_application_status_code as varchar),'~') as unqid + FROM {{source ('rrs','road_application_status_code') }} rasc +) +,rrs_rup_data as ( + Select distinct + 'RRS_RUP' as src_sys_code + ,rs.submission_status_code as authorization_status_code + ,ssc.description AS authorization_status_description + ,'RRS_RUP' || '|'|| coalesce(cast(rs.submission_status_code as varchar),'~') as unqid + FROM {{source ('rrs','road_submission') }} rs + left join {{source ('rrs','submission_status_code') }} ssc ON (rs.submission_status_code = ssc.submission_status_code) + where 1=1 + and rs.road_submission_type_code = 'RUP' + and rs.submission_status_code is not null +) +--insert into pmt_dpl.dim_authorization_status +select * + from ats_data + +union ALL +select * from fta_data + +union ALL +select * from rrs_rup_data + +union ALL +select * from rrs_rp_data + + +{% endsnapshot %} diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_business_area.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_business_area.sql similarity index 98% rename from pmt/dbt/pmt_dbt/snapshots/dpl/dim_business_area.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/dim_business_area.sql index 88d9828..74dfb9f 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_business_area.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_business_area.sql @@ -1,29 +1,29 @@ -{% snapshot dim_business_area %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['business_area_description'], - invalidate_hard_deletes=True, - bind=False, - ) - }} -with corp_data as ( - Select 'CORP' as src_sys_code,'AGRI' as business_area_code,'Agriculture' as business_area_description,'CORP|AGRI' as unqid union all - Select 'CORP' as src_sys_code,'FAW' as business_area_code,'Fish and Wildlife' as business_area_description,'CORP|FAW' as unqid union all - Select 'CORP' as src_sys_code,'FOR' as business_area_code,'Forests' as business_area_description,'CORP|FOR' as unqid union all - Select 'CORP' as src_sys_code,'LAN' as business_area_code,'Lands' as business_area_description,'CORP|LAN' as unqid union all - Select 'CORP' as src_sys_code,'MIN' as business_area_code,'Mines' as business_area_description,'CORP|MIN' as unqid union all - Select 'CORP' as src_sys_code,'PAR' as business_area_code,'Parks' as business_area_description,'CORP|PAR' as unqid union all - Select 'CORP' as src_sys_code,'ROA' as business_area_code,'Roads' as business_area_description,'CORP|ROA' as unqid union all - Select 'CORP' as src_sys_code,'RECS' as business_area_code,'Recreation Sites and Trails' as business_area_description,'CORP|RECS' as unqid union all - Select 'CORP' as src_sys_code,'WAT' as business_area_code,'Water' as business_area_description,'CORP|WAT' as unqid -) ---insert into pmt_dpl.dim_business_area -select * - from corp_data - - +{% snapshot dim_business_area %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['business_area_description'], + invalidate_hard_deletes=True, + bind=False, + ) + }} +with corp_data as ( + Select 'CORP' as src_sys_code,'AGRI' as business_area_code,'Agriculture' as business_area_description,'CORP|AGRI' as unqid union all + Select 'CORP' as src_sys_code,'FAW' as business_area_code,'Fish and Wildlife' as business_area_description,'CORP|FAW' as unqid union all + Select 'CORP' as src_sys_code,'FOR' as business_area_code,'Forests' as business_area_description,'CORP|FOR' as unqid union all + Select 'CORP' as src_sys_code,'LAN' as business_area_code,'Lands' as business_area_description,'CORP|LAN' as unqid union all + Select 'CORP' as src_sys_code,'MIN' as business_area_code,'Mines' as business_area_description,'CORP|MIN' as unqid union all + Select 'CORP' as src_sys_code,'PAR' as business_area_code,'Parks' as business_area_description,'CORP|PAR' as unqid union all + Select 'CORP' as src_sys_code,'ROA' as business_area_code,'Roads' as business_area_description,'CORP|ROA' as unqid union all + Select 'CORP' as src_sys_code,'RECS' as business_area_code,'Recreation Sites and Trails' as business_area_description,'CORP|RECS' as unqid union all + Select 'CORP' as src_sys_code,'WAT' as business_area_code,'Water' as business_area_description,'CORP|WAT' as unqid +) +--insert into pmt_dpl.dim_business_area +select * + from corp_data + + {% endsnapshot %} \ No newline at end of file diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_location.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_location.sql similarity index 96% rename from pmt/dbt/pmt_dbt/snapshots/dpl/dim_location.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/dim_location.sql index db9b4c3..5ab9a0a 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_location.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_location.sql @@ -1,59 +1,59 @@ -{% snapshot dim_location %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['location_code'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with ats_data as -( select distinct - 'ATS' as src_sys_code - ,proj.location as location_code - ,'ATS' || '|'|| coalesce(cast(proj.location as varchar),'~') as unqid ---from fdw_ods_ats_replication.ats_projects proj ---inner join fdw_ods_ats_replication.ats_managing_fcbc_regions amfr -from {{ source('ats','ats_projects') }} proj -inner join {{ source('ats','ats_managing_fcbc_regions') }} amfr -on (proj.managing_fcbc_region_id = amfr.managing_fcbc_region_id) -where proj.project_status_code = '1' -) -, fta_data as -(--FTA does not have location as of now - Select - 'FTA' as src_sys_code - ,null as location_code - ,'FTA' || '|'|| coalesce(cast(null as varchar),'~') as unqid -) -, rrs_rp_data as -(--RRS_RP does not have location as of now - Select - 'RRS_RP' as src_sys_code - ,null as location_code - ,'RRS_RP' || '|'|| coalesce(cast(null as varchar),'~') as unqid -) -, rrs_rup_data as -(--RRS_RUP does not have location as of now - Select - 'RRS_RUP' as src_sys_code - ,null as location_code - ,'RRS_RUP' || '|'|| coalesce(cast(null as varchar),'~') as unqid -)--insert into pmt_dpl.dim_location -select * from ats_data - -union ALL -select * from fta_data - -union ALL -select * from rrs_rup_data - -union ALL -select * from rrs_rp_data - - +{% snapshot dim_location %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['location_code'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with ats_data as +( select distinct + 'ATS' as src_sys_code + ,proj.location as location_code + ,'ATS' || '|'|| coalesce(cast(proj.location as varchar),'~') as unqid +--from fdw_ods_ats_replication.ats_projects proj +--inner join fdw_ods_ats_replication.ats_managing_fcbc_regions amfr +from {{ source('ats','ats_projects') }} proj +inner join {{ source('ats','ats_managing_fcbc_regions') }} amfr +on (proj.managing_fcbc_region_id = amfr.managing_fcbc_region_id) +where proj.project_status_code = '1' +) +, fta_data as +(--FTA does not have location as of now + Select + 'FTA' as src_sys_code + ,null as location_code + ,'FTA' || '|'|| coalesce(cast(null as varchar),'~') as unqid +) +, rrs_rp_data as +(--RRS_RP does not have location as of now + Select + 'RRS_RP' as src_sys_code + ,null as location_code + ,'RRS_RP' || '|'|| coalesce(cast(null as varchar),'~') as unqid +) +, rrs_rup_data as +(--RRS_RUP does not have location as of now + Select + 'RRS_RUP' as src_sys_code + ,null as location_code + ,'RRS_RUP' || '|'|| coalesce(cast(null as varchar),'~') as unqid +)--insert into pmt_dpl.dim_location +select * from ats_data + +union ALL +select * from fta_data + +union ALL +select * from rrs_rup_data + +union ALL +select * from rrs_rp_data + + {% endsnapshot %} \ No newline at end of file diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_ministry.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_ministry.sql similarity index 98% rename from pmt/dbt/pmt_dbt/snapshots/dpl/dim_ministry.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/dim_ministry.sql index f455292..3445801 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_ministry.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_ministry.sql @@ -1,58 +1,58 @@ -{% snapshot dim_ministry %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['ministry_description'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with corp_data as ( - select distinct 'CORP' as src_sys_code - ,CASE - WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'EMLI' - WHEN ats.authorization_instrument_id = 27 THEN 'FOR' - WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'AGR' - WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107]) THEN 'ENV' - ELSE NULL - END AS ministry_code - ,CASE - WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'Ministry of Energy, Mines and Low Carbon Innovation' - WHEN ats.authorization_instrument_id = 27 THEN 'Ministry of Forests' - WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'Ministry of Agriculture' - WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107]) THEN 'Ministry of Environment' - ELSE NULL - END AS ministry_description - - ,'CORP' || '|'|| coalesce(cast(CASE - WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'EMLI' - WHEN ats.authorization_instrument_id = 27 THEN 'FOR' - WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'AGR' - WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107]) THEN 'ENV' - ELSE NULL - END as varchar),'~') as unqid - from {{ source('ats','ats_authorizations') }} ats - LEFT JOIN {{ source('ats','ats_authorization_status_codes') }} aasc - ON(ats.authorization_status_code = aasc.authorization_status_code) - LEFT JOIN {{ source('ats','ats_authorization_instruments') }} aai - ON(ats.authorization_instrument_id = aai.authorization_instrument_id) - where 1=1 - AND aasc.authorization_status_code <> '1' - AND (ats.authorization_instrument_id = - ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061, 69, 542, 1481, 1121, 1124, - 1125, 1122, 421, 68, 541, 70, 1, 71, 124, 181, 73, 701, 702, 33, 704, 503, 82, 482, 710, 709, 1241, 481, 483, 502, 716, 708, 80, 39, 686, 685, 132, 84, 711, 684, 41, 110, 29, - 714, 102, 105, 717, 521, 524, 103, 501, 718, 18, 720, 281, 89, 25, 24, 3, 4, 683, 713, 1101, 1402, 1401, 57, 44, - 1201, 45, 48, 851, 50, 941, 99, 51, 852, 58, 981, 52, 1081, 1083, 1082, 56, 1001, 401, 65, 881, 861, 846, 841, 843, 842, 847, 845, 441, 921, 11, 66, 100, 72, 2, 20, 682, 19, 1421, 1441, 761, - 341, 901, 27, 762, 10, 1181, 113, 1022, 1021, 119, 126, 28, 30, 1023, 1261, 641, 16, 1041, 721, 1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107] - )) - ) ---insert into pmt_dpl.dim_ministry -select * from corp_data -where ministry_code is not null - - +{% snapshot dim_ministry %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['ministry_description'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with corp_data as ( + select distinct 'CORP' as src_sys_code + ,CASE + WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'EMLI' + WHEN ats.authorization_instrument_id = 27 THEN 'FOR' + WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'AGR' + WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107]) THEN 'ENV' + ELSE NULL + END AS ministry_code + ,CASE + WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'Ministry of Energy, Mines and Low Carbon Innovation' + WHEN ats.authorization_instrument_id = 27 THEN 'Ministry of Forests' + WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'Ministry of Agriculture' + WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107]) THEN 'Ministry of Environment' + ELSE NULL + END AS ministry_description + + ,'CORP' || '|'|| coalesce(cast(CASE + WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'EMLI' + WHEN ats.authorization_instrument_id = 27 THEN 'FOR' + WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'AGR' + WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107]) THEN 'ENV' + ELSE NULL + END as varchar),'~') as unqid + from {{ source('ats','ats_authorizations') }} ats + LEFT JOIN {{ source('ats','ats_authorization_status_codes') }} aasc + ON(ats.authorization_status_code = aasc.authorization_status_code) + LEFT JOIN {{ source('ats','ats_authorization_instruments') }} aai + ON(ats.authorization_instrument_id = aai.authorization_instrument_id) + where 1=1 + AND aasc.authorization_status_code <> '1' + AND (ats.authorization_instrument_id = + ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061, 69, 542, 1481, 1121, 1124, + 1125, 1122, 421, 68, 541, 70, 1, 71, 124, 181, 73, 701, 702, 33, 704, 503, 82, 482, 710, 709, 1241, 481, 483, 502, 716, 708, 80, 39, 686, 685, 132, 84, 711, 684, 41, 110, 29, + 714, 102, 105, 717, 521, 524, 103, 501, 718, 18, 720, 281, 89, 25, 24, 3, 4, 683, 713, 1101, 1402, 1401, 57, 44, + 1201, 45, 48, 851, 50, 941, 99, 51, 852, 58, 981, 52, 1081, 1083, 1082, 56, 1001, 401, 65, 881, 861, 846, 841, 843, 842, 847, 845, 441, 921, 11, 66, 100, 72, 2, 20, 682, 19, 1421, 1441, 761, + 341, 901, 27, 762, 10, 1181, 113, 1022, 1021, 119, 126, 28, 30, 1023, 1261, 641, 16, 1041, 721, 1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107] + )) + ) +--insert into pmt_dpl.dim_ministry +select * from corp_data +where ministry_code is not null + + {% endsnapshot %} \ No newline at end of file diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_org.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_org.sql similarity index 97% rename from pmt/dbt/pmt_dbt/snapshots/dpl/dim_org.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/dim_org.sql index ef20b6b..78d0708 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_org.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_org.sql @@ -1,37 +1,37 @@ -{% snapshot dim_org %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['org_unit_description'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with corp_data as ( - select 'CORP' as src_sys_code - ,org.org_unit_code as org_unit_code - ,org.org_unit_name as org_unit_description - ,org.rollup_region_code as rollup_region_code - ,reg.org_unit_name as rollup_region_description - ,org.rollup_dist_code as rollup_district_code - ,rdis.org_unit_name as rollup_district_description - ,case when org.rollup_dist_code in ('DCC','DQU','DMH','DRM','DSE','DOS','DCS','DKA') then 'SA' - when org.rollup_dist_code in ('DFN','DPC','DPG','DMK','DVA','DKM','DSS','DND') then 'NA' - when org.rollup_dist_code in ('DSC','DSQ','DCK','DNI','DSI','DQC','DCR') then 'CA' end as roll_up_area_code - ,case when org.rollup_dist_code in ('DCC','DQU','DMH','DRM','DSE','DOS','DCS','DKA') then 'South Area' - when org.rollup_dist_code in ('DFN','DPC','DPG','DMK','DVA','DKM','DSS','DND') then 'North Area' - when org.rollup_dist_code in ('DSC','DSQ','DCK','DNI','DSI','DQC','DCR') then 'Coast Area' end as roll_up_area_description - -- Combine multiple keys into a single composite_key - ,'CORP' || '|'|| coalesce(cast(org.org_unit_code as varchar),'~') as unqid - from {{ source('fta','org_unit') }} org - left join {{ source('fta','org_unit') }} reg on (org.rollup_region_code = reg.org_unit_code) - left join {{ source('fta','org_unit') }} rdis on (org.rollup_dist_code = rdis.org_unit_code) - ) ---insert into pmt_dpl.dim_org - select * from corp_data - +{% snapshot dim_org %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['org_unit_description'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with corp_data as ( + select 'CORP' as src_sys_code + ,org.org_unit_code as org_unit_code + ,org.org_unit_name as org_unit_description + ,org.rollup_region_code as rollup_region_code + ,reg.org_unit_name as rollup_region_description + ,org.rollup_dist_code as rollup_district_code + ,rdis.org_unit_name as rollup_district_description + ,case when org.rollup_dist_code in ('DCC','DQU','DMH','DRM','DSE','DOS','DCS','DKA') then 'SA' + when org.rollup_dist_code in ('DFN','DPC','DPG','DMK','DVA','DKM','DSS','DND') then 'NA' + when org.rollup_dist_code in ('DSC','DSQ','DCK','DNI','DSI','DQC','DCR') then 'CA' end as roll_up_area_code + ,case when org.rollup_dist_code in ('DCC','DQU','DMH','DRM','DSE','DOS','DCS','DKA') then 'South Area' + when org.rollup_dist_code in ('DFN','DPC','DPG','DMK','DVA','DKM','DSS','DND') then 'North Area' + when org.rollup_dist_code in ('DSC','DSQ','DCK','DNI','DSI','DQC','DCR') then 'Coast Area' end as roll_up_area_description + -- Combine multiple keys into a single composite_key + ,'CORP' || '|'|| coalesce(cast(org.org_unit_code as varchar),'~') as unqid + from {{ source('fta','org_unit') }} org + left join {{ source('fta','org_unit') }} reg on (org.rollup_region_code = reg.org_unit_code) + left join {{ source('fta','org_unit') }} rdis on (org.rollup_dist_code = rdis.org_unit_code) + ) +--insert into pmt_dpl.dim_org + select * from corp_data + {% endsnapshot %} \ No newline at end of file diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_permit_status.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_permit_status.sql similarity index 96% rename from pmt/dbt/pmt_dbt/snapshots/dpl/dim_permit_status.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/dim_permit_status.sql index d4300e1..8b82f96 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_permit_status.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_permit_status.sql @@ -1,58 +1,58 @@ -{% snapshot dim_permit_status %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['permit_status_description'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with ats_data as ( - Select 'ATS' as src_sys_code - ,null as permit_status_code - ,null as permit_status_description - ,'ATS' || '|'|| coalesce(cast(null as varchar),'~') as unqid -) -,fta_data as ( - Select distinct - 'FTA' as src_sys_code - ,hsc.harvest_auth_status_code as permit_status_code - ,hsc.description as permit_status_description - ,'FTA' || '|'|| coalesce(cast(hsc.harvest_auth_status_code as varchar),'~') as unqid - FROM {{ source('fta','harvest_auth_status_code') }} hsc -) -,rrs_rp_data as ( - Select distinct - 'RRS_RP' as src_sys_code - ,rtsc.road_tenure_status_code as permit_status_code - ,rtsc.description AS permit_status_description - ,'RRS_RP' || '|'|| coalesce(cast(rtsc.road_tenure_status_code as varchar),'~') as unqid - FROM {{ source('rrs','road_tenure_status_code') }} rtsc -) -,rrs_rup_data as ( - Select distinct - 'RRS_RUP' as src_sys_code - ,null as permit_status_code - ,null AS permit_status_description - ,'RRS_RUP' || '|'|| coalesce(cast(null as varchar),'~') as unqid -) ---insert into pmt_dpl.dim_permit_status -select * - from ats_data - -union ALL -select * from fta_data - -union ALL -select * from rrs_rup_data - -union ALL -select * from rrs_rp_data - - - -{% endsnapshot %} +{% snapshot dim_permit_status %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['permit_status_description'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with ats_data as ( + Select 'ATS' as src_sys_code + ,null as permit_status_code + ,null as permit_status_description + ,'ATS' || '|'|| coalesce(cast(null as varchar),'~') as unqid +) +,fta_data as ( + Select distinct + 'FTA' as src_sys_code + ,hsc.harvest_auth_status_code as permit_status_code + ,hsc.description as permit_status_description + ,'FTA' || '|'|| coalesce(cast(hsc.harvest_auth_status_code as varchar),'~') as unqid + FROM {{ source('fta','harvest_auth_status_code') }} hsc +) +,rrs_rp_data as ( + Select distinct + 'RRS_RP' as src_sys_code + ,rtsc.road_tenure_status_code as permit_status_code + ,rtsc.description AS permit_status_description + ,'RRS_RP' || '|'|| coalesce(cast(rtsc.road_tenure_status_code as varchar),'~') as unqid + FROM {{ source('rrs','road_tenure_status_code') }} rtsc +) +,rrs_rup_data as ( + Select distinct + 'RRS_RUP' as src_sys_code + ,null as permit_status_code + ,null AS permit_status_description + ,'RRS_RUP' || '|'|| coalesce(cast(null as varchar),'~') as unqid +) +--insert into pmt_dpl.dim_permit_status +select * + from ats_data + +union ALL +select * from fta_data + +union ALL +select * from rrs_rup_data + +union ALL +select * from rrs_rp_data + + + +{% endsnapshot %} diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_permit_type.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_permit_type.sql similarity index 96% rename from pmt/dbt/pmt_dbt/snapshots/dpl/dim_permit_type.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/dim_permit_type.sql index 1c0cc4a..11df768 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_permit_type.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_permit_type.sql @@ -1,61 +1,61 @@ -{% snapshot dim_permit_type %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['permit_type_description'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with ats_data as ( - Select distinct 'ATS' as src_sys_code - ,aai.authorization_instrument_id::varchar(25) as permit_type_code - ,aai.authorization_instrument_name as permit_type_description - ,'ATS' || '|'|| coalesce(cast(aai.authorization_instrument_id as varchar),'~') as unqid - from {{source ('ats','ats_authorization_instruments') }} aai - where 1=1 -) -, -fta_data as ( - Select distinct 'FTA' as src_sys_code - ,pfu.file_type_code::varchar(25) AS permit_type_code - , ftc.description AS permit_type_description - ,'FTA' || '|'|| coalesce(cast(pfu.file_type_code as varchar(25)),'~') as unqid - from {{source ('fta','prov_forest_use') }} pfu - left join {{source ('fta','file_type_code') }} ftc - ON (ftc.file_type_code = pfu.file_type_code) -) -, -rrs_rp_data as ( - Select 'RRS_RP' as src_sys_code - ,rttc.road_tenure_type_code as permit_type_code - ,rttc.description as permit_type_description - ,'RRS_RP' || '|'|| coalesce(cast(rttc.road_tenure_type_code as varchar(25)),'~') as unqid - from {{source ('rrs','road_tenure_type_code') }} rttc -) -, -rrs_rup_data as ( - Select 'RRS_RUP' as src_sys_code - ,'RUP' as permit_type_code - ,'Road Use Permit' as permit_type_description - ,'RRS_RUP' || '|'|| coalesce(cast('RUP' as varchar(25)),'~') as unqid -) ---insert into pmt_dpl.dim_permit_type -select * - from ats_data - -union ALL -select * from fta_data - -union ALL -select * from rrs_rup_data - -union ALL -select * from rrs_rp_data - - +{% snapshot dim_permit_type %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['permit_type_description'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with ats_data as ( + Select distinct 'ATS' as src_sys_code + ,aai.authorization_instrument_id::varchar(25) as permit_type_code + ,aai.authorization_instrument_name as permit_type_description + ,'ATS' || '|'|| coalesce(cast(aai.authorization_instrument_id as varchar),'~') as unqid + from {{source ('ats','ats_authorization_instruments') }} aai + where 1=1 +) +, +fta_data as ( + Select distinct 'FTA' as src_sys_code + ,pfu.file_type_code::varchar(25) AS permit_type_code + , ftc.description AS permit_type_description + ,'FTA' || '|'|| coalesce(cast(pfu.file_type_code as varchar(25)),'~') as unqid + from {{source ('fta','prov_forest_use') }} pfu + left join {{source ('fta','file_type_code') }} ftc + ON (ftc.file_type_code = pfu.file_type_code) +) +, +rrs_rp_data as ( + Select 'RRS_RP' as src_sys_code + ,rttc.road_tenure_type_code as permit_type_code + ,rttc.description as permit_type_description + ,'RRS_RP' || '|'|| coalesce(cast(rttc.road_tenure_type_code as varchar(25)),'~') as unqid + from {{source ('rrs','road_tenure_type_code') }} rttc +) +, +rrs_rup_data as ( + Select 'RRS_RUP' as src_sys_code + ,'RUP' as permit_type_code + ,'Road Use Permit' as permit_type_description + ,'RRS_RUP' || '|'|| coalesce(cast('RUP' as varchar(25)),'~') as unqid +) +--insert into pmt_dpl.dim_permit_type +select * + from ats_data + +union ALL +select * from fta_data + +union ALL +select * from rrs_rup_data + +union ALL +select * from rrs_rp_data + + {% endsnapshot %} \ No newline at end of file diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_project.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_project.sql similarity index 96% rename from pmt/dbt/pmt_dbt/snapshots/dpl/dim_project.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/dim_project.sql index 6c3e1fe..8fb706e 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_project.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_project.sql @@ -1,72 +1,72 @@ -{% snapshot dim_project %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['project_name','project_location_code','project_status_code','project_region_description'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with ats_data as ( - select 'ATS' as src_sys_code, - prj.project_id:: varchar(20), - --replace(replace(prj.location::text, chr(10), NULL::text), chr(13), NULL::text) AS project_location, ---not sure what these replace values are supposed to do? returns null - prj.project_name, - prj.location as project_location_code, - prj.project_status_code, - amfr.region_name as project_region_description, - 'ATS' || '|'|| coalesce(cast(prj.project_id as varchar),'~') as unqid - from {{source ('ats','ats_projects') }} prj - left join {{source ('ats','ats_managing_fcbc_regions') }} amfr ---dim_regions - on (prj.managing_fcbc_region_id = amfr.managing_fcbc_region_id ) - where 1=1 - and prj.project_status_code = '1' -), -fta_data as ( - Select distinct 'FTA' as src_sys_code - ,null as project_id - ,null as project_name - ,null as project_location_code - ,null as project_status_code - ,null as project_region_description - ,'FTA' || '|'|| coalesce(cast(null as varchar(5)),'~') as unqid - -) -, -rrs_rp_data as ( - Select 'RRS_RP' as src_sys_code - ,null as project_id - ,null as project_name - ,null as project_location_code - ,null as project_status_code - ,null as project_region_description - ,'RRS_RP' || '|'|| coalesce(cast(null as varchar(5)),'~') as unqid -) -, -rrs_rup_data as ( - Select 'RRS_RUP' as src_sys_code - ,null as project_id - ,null as project_name - ,null as project_location_code - ,null as project_status_code - ,null as project_region_description - ,'RRS_RUP' || '|'|| coalesce(cast(null as varchar(5)),'~') as unqid -) ---insert into pmt_dpl.dim_project -select * from ats_data - -union ALL -select * from fta_data - -union ALL -select * from rrs_rup_data - -union ALL -select * from rrs_rp_data - - +{% snapshot dim_project %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['project_name','project_location_code','project_status_code','project_region_description'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with ats_data as ( + select 'ATS' as src_sys_code, + prj.project_id:: varchar(20), + --replace(replace(prj.location::text, chr(10), NULL::text), chr(13), NULL::text) AS project_location, ---not sure what these replace values are supposed to do? returns null + prj.project_name, + prj.location as project_location_code, + prj.project_status_code, + amfr.region_name as project_region_description, + 'ATS' || '|'|| coalesce(cast(prj.project_id as varchar),'~') as unqid + from {{source ('ats','ats_projects') }} prj + left join {{source ('ats','ats_managing_fcbc_regions') }} amfr ---dim_regions + on (prj.managing_fcbc_region_id = amfr.managing_fcbc_region_id ) + where 1=1 + and prj.project_status_code = '1' +), +fta_data as ( + Select distinct 'FTA' as src_sys_code + ,null as project_id + ,null as project_name + ,null as project_location_code + ,null as project_status_code + ,null as project_region_description + ,'FTA' || '|'|| coalesce(cast(null as varchar(5)),'~') as unqid + +) +, +rrs_rp_data as ( + Select 'RRS_RP' as src_sys_code + ,null as project_id + ,null as project_name + ,null as project_location_code + ,null as project_status_code + ,null as project_region_description + ,'RRS_RP' || '|'|| coalesce(cast(null as varchar(5)),'~') as unqid +) +, +rrs_rup_data as ( + Select 'RRS_RUP' as src_sys_code + ,null as project_id + ,null as project_name + ,null as project_location_code + ,null as project_status_code + ,null as project_region_description + ,'RRS_RUP' || '|'|| coalesce(cast(null as varchar(5)),'~') as unqid +) +--insert into pmt_dpl.dim_project +select * from ats_data + +union ALL +select * from fta_data + +union ALL +select * from rrs_rup_data + +union ALL +select * from rrs_rp_data + + {% endsnapshot %} \ No newline at end of file diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_source_system.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_source_system.sql similarity index 95% rename from pmt/dbt/pmt_dbt/snapshots/dpl/dim_source_system.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/dim_source_system.sql index d0c8bb4..c687d06 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/dim_source_system.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/dim_source_system.sql @@ -1,36 +1,36 @@ -{% snapshot dim_source_system %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['src_sys_description'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with corp_data as ( - select 'ATS' as src_sys_code, - 'Authorization Tracking System' as src_sys_description, - 'ATS' as unqid - union ALL - select 'FTA' as src_sys_code, - 'Forest Tenure Administration' as src_sys_description, - 'FTA' as unqid - union ALL - select 'RRS_RP' as src_sys_code, - 'Resource Roads System - Road Permits' as src_sys_description, - 'RRS_RP' as unqid - union ALL - select 'CORP' as src_sys_code, - 'Corporate' as src_sys_description, - 'CORP' as unqid - -) ---insert into pmt_dpl.dim_source_system -select * from corp_data - - +{% snapshot dim_source_system %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['src_sys_description'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with corp_data as ( + select 'ATS' as src_sys_code, + 'Authorization Tracking System' as src_sys_description, + 'ATS' as unqid + union ALL + select 'FTA' as src_sys_code, + 'Forest Tenure Administration' as src_sys_description, + 'FTA' as unqid + union ALL + select 'RRS_RP' as src_sys_code, + 'Resource Roads System - Road Permits' as src_sys_description, + 'RRS_RP' as unqid + union ALL + select 'CORP' as src_sys_code, + 'Corporate' as src_sys_description, + 'CORP' as unqid + +) +--insert into pmt_dpl.dim_source_system +select * from corp_data + + {% endsnapshot %} \ No newline at end of file diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/fact_permits.sql b/permitting/dbt/pmt_dbt/snapshots/dpl/fact_permits.sql similarity index 98% rename from pmt/dbt/pmt_dbt/snapshots/dpl/fact_permits.sql rename to permitting/dbt/pmt_dbt/snapshots/dpl/fact_permits.sql index 930c802..4425e07 100644 --- a/pmt/dbt/pmt_dbt/snapshots/dpl/fact_permits.sql +++ b/permitting/dbt/pmt_dbt/snapshots/dpl/fact_permits.sql @@ -1,540 +1,540 @@ -{% snapshot fact_permits %} - - {{ - config( - target_schema='pmt_dpl', - strategy='check', - unique_key='unqid', - check_cols=['ministry_code','business_area_code','permit_status_code','permit_status_date','permit_issue_date','app_received_date','app_update_date','app_decision_date','app_issuance_date','app_accepted_date','app_rejected_date','app_adjudication_date','harvest_area_sq_m','harvest_auth_status','map_feature_id','permit_type_code'], - invalidate_hard_deletes=True, - bind=False, - ) - }} - -with ats_data as ( - Select - 'ATS' as src_sys_code - ,CASE - WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'EMLI' - WHEN ats.authorization_instrument_id = 27 THEN 'FOR' - WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'AGR' - WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107]) THEN 'ENV' - ELSE NULL --- ELSE NULL is causing null ministries - END AS ministry_code - ,CASE - WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'MIN' - WHEN ats.authorization_instrument_id = ANY (ARRAY[69, 542, 1481, 1121, 1124, 1125, 1122, 421, 68, 541, 70, 1, 71, 124]) THEN 'WAT' - WHEN ats.authorization_instrument_id = ANY (ARRAY[181, 73, 701, 702, 33, 704, 503, 82, 482, 710, 709, 1241, 481, 483, 502, 716, 708, 80, 39, 686, 685, 132, 84, 711, 684, 41, 110, 29, 714, 102, 105, 717, 521, 524, 103, 501, 718, 18, 720, 281, 89, 25, 24, 3, 4, 683, 713]) THEN 'LAN' - WHEN ats.authorization_instrument_id = ANY (ARRAY[1101, 1402, 1401, 57, 44, 1201, 45, 48, 851, 50, 941, 99, 51, 852, 58, 981, 52, 1081, 1083, 1082, 56, 1001, 401, 65, 881, 861, 846, 841, 843, 842, 847, 845, 441, 921, 11, 66, 100, 72]) THEN 'FAW' - WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'AGRI' - WHEN ats.authorization_instrument_id = ANY (ARRAY[761, 341, 901, 27, 762, 10, 1181, 113, 1022, 1021, 119, 126, 28, 30, 1023, 1261, 641, 16, 1041, 721]) THEN 'FOR' - WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342]) THEN 'PAR' - WHEN ats.authorization_instrument_id = ANY (ARRAY[21, 107]) THEN 'RECS' - ELSE NULL - END as business_area_code - ,ats.authorization_id::varchar as application_id - ,null ::varchar as permit_id - ,ats.file_number as app_file_id - ,ats.authorization_status_code as authorization_status_code - ,ats.project_id::varchar(20) as project_id - ,null as permit_status_code - ,null::date as permit_status_date - ,null::date as permit_issue_date - ,null::date as permit_issue_expire_date - ,ats.application_received_date as app_received_date - ,coalesce(ats.fcbc_process_complete_date, ats.adjudication_date,ats.application_accepted_date,ats.application_accepted_date) as app_update_date - ,ats.fcbc_process_complete_date as app_decision_date - ,ats.adjudication_date as app_issuance_date - ,ats.application_accepted_date as app_accepted_date - ,null::date as app_rejected_date - ,ats.adjudication_date as app_adjudication_date - ,null as harvest_auth_status - ,null::int as harvest_area_sq_m - ,null as permit_org_unit_code - ,floor( - CASE - WHEN ats.adjudication_date IS NULL THEN EXTRACT(epoch FROM CURRENT_DATE::timestamp without time zone - ats.application_received_date) / 86400 - ELSE EXTRACT(epoch FROM ats.adjudication_date - ats.application_received_date) / 86400 - END) AS application_age - ,null::bigint as map_feature_id - ,ats.authorization_instrument_id::varchar(30) as permit_type_code - ,'ATS' || '|'|| coalesce(cast(ats.authorization_id as varchar),'~')||'|'||coalesce(cast(ats.project_id as varchar),'~') as unqid -from {{source ('ats','ats_authorizations') }} ats -LEFT JOIN {{source ('ats','ats_authorization_status_codes') }} aasc - ON(ats.authorization_status_code = aasc.authorization_status_code) -LEFT JOIN {{source ('ats','ats_authorization_instruments') }} aai - ON(ats.authorization_instrument_id = aai.authorization_instrument_id) -where 1=1 -AND aasc.authorization_status_code = '1' -AND (ats.authorization_instrument_id = -ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061, 69, 542, 1481, 1121, 1124, -1125, 1122, 421, 68, 541, 70, 1, 71, 124, 181, 73, 701, 702, 33, 704, 503, 82, 482, 710, 709, 1241, 481, 483, 502, 716, 708, 80, 39, 686, 685, 132, 84, 711, 684, 41, 110, 29, -714, 102, 105, 717, 521, 524, 103, 501, 718, 18, 720, 281, 89, 25, 24, 3, 4, 683, 713, 1101, 1402, 1401, 57, 44, -1201, 45, 48, 851, 50, 941, 99, 51, 852, 58, 981, 52, 1081, 1083, 1082, 56, 1001, 401, 65, 881, 861, 846, 841, 843, 842, 847, 845, 441, 921, 11, 66, 100, 72, 2, 20, 682, 19, 1421, 1441, 761, -341, 901, 27, 762, 10, 1181, 113, 1022, 1021, 119, 126, 28, 30, 1023, 1261, 641, 16, 1041, 721, 1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107] -)) -) -,fta_data as -( -Select -'FTA' as src_sys_code -,'FOR' as ministry_code -,'FORESTS' as business_area_code -,tenure_app_id::varchar as application_id -,cutting_permit_id as permit_id -,forest_file_id as app_file_id -,tenure_application_state_code as authorization_status_code -,null as project_id -,permit_status_code as permit_status_code -,adjudication_date as permit_status_date -,issuance_date as permit_issue_date -,expiry_date as permit_issue_expire_date -,submission_date as app_received_date -,adjudication_date as app_update_date -,decision_date as app_decision_date -,issuance_date as app_issuance_date -,submission_date as app_accepted_date -,case when tenure_application_state_code = 'REJ' then adjudication_date end as app_rejected_date -,adjudication_date as app_adjudication_date -,permit_status_code as harvest_auth_status -,harvest_area::int as harvest_area_sq_m -,org_unit_code as permit_org_unit_code -,extract(epoch from age)/86400 as application_age -,map_feature_id as map_feature_id -,permit_type_code as permit_type_code -,'FTA' || '|'|| coalesce(cast(forest_file_id as varchar),'~')||'|'||coalesce(cast( cutting_permit_id as varchar),'~') as unqid -from -( SELECT - hva.forest_file_id , - ou.org_unit_code as org_unit_code, - pfu.file_type_code AS permit_type_code, - ftc.description AS permit_type, - COALESCE(hva.cutting_permit_id, hva.forest_file_id) AS cutting_permit_id, - htc.description AS havesting_auth_type, - hasc.harvest_auth_status_code as permit_status_code, - tatc.description AS tenure_application_type, - tasc.description AS tenure_application_state_code, - hva.tenure_term, - hva.status_date, - hva.issue_date, - hva.expiry_date, - hva.extend_date, - CASE - WHEN hva.extend_date IS NOT NULL THEN hva.extend_date - WHEN hva.expiry_date IS NOT NULL THEN hva.expiry_date - ELSE GREATEST(hva.extend_date, hva.expiry_date) - END AS current_expiry_date_calc, - hva.location, - hva.harvest_area, - ta.tenure_app_id, - ta.tenure_appname, - ta.submission_id, - ta.client_number, - ta.adjudication_ind, - ta.adjudication_date, - ta.submission_date, - ta.utm_zone, - ta.decision_date, - ta.issuance_date, - CASE - WHEN ta.tenure_application_state_code = 'APP' THEN ta.decision_date - ta.submission_date - WHEN ta.tenure_application_state_code = ANY (ARRAY['ISS', 'REJ']) THEN ta.issuance_date - ta.submission_date - ELSE CURRENT_DATE::timestamp without time zone - ta.submission_date - END AS age, - null as geometry, - hag.map_feature_id, - NULL AS project_name, - NULL AS project_description, - NULL AS property_subclass, - NULL AS project_location, - NULL AS utm_easting, - NULL AS utm_northing, - NULL AS utm_zone, - hva.hva_skey - FROM ( SELECT harvesting_authority.hva_skey, - harvesting_authority.forest_file_id, - harvesting_authority.cutting_permit_id, - harvesting_authority.harvesting_authority_id, - harvesting_authority.forest_district, - harvesting_authority.district_admn_zone, - harvesting_authority.geographic_district, - harvesting_authority.mgmt_unit_id, - harvesting_authority.mgmt_unit_type_code, - harvesting_authority.licence_to_cut_code, - harvesting_authority.harvest_type_code, - harvesting_authority.harvest_auth_status_code, - harvesting_authority.tenure_term, - harvesting_authority.status_date, - harvesting_authority.issue_date, - harvesting_authority.expiry_date, - harvesting_authority.extend_date, - harvesting_authority.extend_count, - harvesting_authority.harvest_auth_extend_reas_code, - harvesting_authority.quota_type_code, - harvesting_authority.crown_lands_region_code, - harvesting_authority.salvage_type_code, - harvesting_authority.cascade_split_code, - harvesting_authority.catastrophic_ind, - harvesting_authority.crown_granted_ind, - harvesting_authority.cruise_based_ind, - harvesting_authority.deciduous_ind, - harvesting_authority.bcaa_folio_number, - harvesting_authority.location, - harvesting_authority.higher_level_plan_reference, - harvesting_authority.harvest_area, - harvesting_authority.retirement_date, - harvesting_authority.revision_count, - harvesting_authority.entry_userid, - harvesting_authority.entry_timestamp, - harvesting_authority.update_userid, - harvesting_authority.update_timestamp, - harvesting_authority.is_waste_assessment_required, - harvesting_authority.is_cp_extensn_appl_fee_waived, - harvesting_authority.is_cp_extension_appl_fee_paid, - harvesting_authority.is_within_fibre_recovery_zone, - harvesting_authority.harvesting_authority_guid - FROM {{source ('fta','harvesting_authority') }} - WHERE (harvesting_authority.forest_file_id IN ( SELECT zz.forest_file_id - FROM ( SELECT a.forest_file_id, - count(a.cutting_permit_id) AS count - FROM {{source ('fta','harvesting_authority') }} a - JOIN ( SELECT DISTINCT harvesting_authority_1.forest_file_id - FROM {{source ('fta','harvesting_authority') }} harvesting_authority_1 - WHERE harvesting_authority_1.cutting_permit_id IS NULL) z ON a.forest_file_id = z.forest_file_id - GROUP BY a.forest_file_id - HAVING count(a.cutting_permit_id) = 0) zz))) hva - JOIN {{source ('fta','prov_forest_use') }} pfu ON hva.forest_file_id = pfu.forest_file_id - JOIN {{source ('fta','file_type_code') }} ftc ON ftc.file_type_code = pfu.file_type_code - LEFT JOIN {{source ('fta','harvest_authority_geom') }} hag ON hva.hva_skey = hag.hva_skey - JOIN {{source ('fta','harvest_type_code') }} htc ON hva.harvest_type_code = htc.harvest_type_code - JOIN {{source ('fta','harvest_auth_status_code') }} hasc ON hasc.harvest_auth_status_code = hva.harvest_auth_status_code - LEFT JOIN {{source ('fta','tenure_application_map_feature') }} tamf ON tamf.map_feature_id = hag.map_feature_id - LEFT JOIN {{source ('fta','tenure_application') }} ta ON ta.forest_file_id = hva.forest_file_id and hva.cutting_permit_id = ta.cutting_permit_id - left join {{source ('fta','org_unit') }} ou on (ta.org_unit_no = ou.org_unit_no) - LEFT JOIN {{source ('fta','tenure_application_state_code') }} tasc ON tasc.tenure_application_state_code = ta.tenure_application_state_code - LEFT JOIN {{source ('fta','tenure_application_type_code') }} tatc ON tatc.tenure_application_type_code = ta.tenure_application_type_code - WHERE (ftc.file_type_code = ANY (ARRAY['A01', 'A02', 'A03', 'A04', 'A05', 'A11', 'A18', 'A28', 'A29', 'A30', 'A31', 'A41', 'A44', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B09', 'B15', 'B16', 'E01', 'E02', 'H01', 'H02', 'S01', 'S02'])) AND (tasc.description = ANY (ARRAY['Inbox', 'Lobby'])) - -UNION ALL - SELECT hva.forest_file_id , - ou.org_unit_code as org_unit_code, - pfu.file_type_code AS permit_type_code, - ftc.description AS permit_type, - COALESCE(hva.cutting_permit_id, hva.forest_file_id) AS cutting_permit_id, - htc.description AS havesting_auth_type, - hasc.harvest_auth_status_code as permit_status_code, - tatc.description AS tenure_application_type, - tasc.description AS tenure_application_state_code, - hva.tenure_term, - hva.status_date, - hva.issue_date, - hva.expiry_date, - hva.extend_date, - CASE - WHEN hva.extend_date IS NOT NULL THEN hva.extend_date - WHEN hva.expiry_date IS NOT NULL THEN hva.expiry_date - ELSE GREATEST(hva.extend_date, hva.expiry_date) - END AS current_expiry_date_calc, - hva.location, - hva.harvest_area, - ta.tenure_app_id, - ta.tenure_appname, - ta.submission_id, - ta.client_number, - ta.adjudication_ind, - ta.adjudication_date, - ta.submission_date, - ta.utm_zone, - ta.decision_date, - ta.issuance_date, - CASE - WHEN ta.tenure_application_state_code = 'APP' THEN ta.decision_date - ta.submission_date - WHEN ta.tenure_application_state_code = ANY (ARRAY['ISS', 'REJ']) THEN ta.issuance_date - ta.submission_date - ELSE CURRENT_DATE::timestamp without time zone - ta.submission_date - END AS age, - null as geometry, - hag.map_feature_id, - NULL AS project_name, - NULL AS project_description, - NULL AS property_subclass, - NULL AS project_location, - NULL AS utm_easting, - NULL AS utm_northing, - NULL AS utm_zone, - hva.hva_skey - FROM ( SELECT harvesting_authority.hva_skey, - harvesting_authority.forest_file_id, - harvesting_authority.cutting_permit_id, - harvesting_authority.harvesting_authority_id, - harvesting_authority.forest_district, - harvesting_authority.district_admn_zone, - harvesting_authority.geographic_district, - harvesting_authority.mgmt_unit_id, - harvesting_authority.mgmt_unit_type_code, - harvesting_authority.licence_to_cut_code, - harvesting_authority.harvest_type_code, - harvesting_authority.harvest_auth_status_code, - harvesting_authority.tenure_term, - harvesting_authority.status_date, - harvesting_authority.issue_date, - harvesting_authority.expiry_date, - harvesting_authority.extend_date, - harvesting_authority.extend_count, - harvesting_authority.harvest_auth_extend_reas_code, - harvesting_authority.quota_type_code, - harvesting_authority.crown_lands_region_code, - harvesting_authority.salvage_type_code, - harvesting_authority.cascade_split_code, - harvesting_authority.catastrophic_ind, - harvesting_authority.crown_granted_ind, - harvesting_authority.cruise_based_ind, - harvesting_authority.deciduous_ind, - harvesting_authority.bcaa_folio_number, - harvesting_authority.location, - harvesting_authority.higher_level_plan_reference, - harvesting_authority.harvest_area, - harvesting_authority.retirement_date, - harvesting_authority.revision_count, - harvesting_authority.entry_userid, - harvesting_authority.entry_timestamp, - harvesting_authority.update_userid, - harvesting_authority.update_timestamp, - harvesting_authority.is_waste_assessment_required, - harvesting_authority.is_cp_extensn_appl_fee_waived, - harvesting_authority.is_cp_extension_appl_fee_paid, - harvesting_authority.is_within_fibre_recovery_zone, - harvesting_authority.harvesting_authority_guid - FROM {{source ('fta','harvesting_authority') }} - WHERE (harvesting_authority.forest_file_id IN ( SELECT zz.forest_file_id - FROM ( SELECT a.forest_file_id, - count(a.cutting_permit_id) AS count - FROM {{source ('fta','harvesting_authority') }} a - JOIN ( SELECT DISTINCT harvesting_authority_1.forest_file_id - FROM {{source ('fta','harvesting_authority') }} harvesting_authority_1 - WHERE harvesting_authority_1.cutting_permit_id IS NOT NULL) z ON a.forest_file_id = z.forest_file_id - GROUP BY a.forest_file_id - HAVING count(a.cutting_permit_id) > 0) zz))) hva - JOIN {{source ('fta','prov_forest_use') }} pfu ON hva.forest_file_id = pfu.forest_file_id - JOIN {{source ('fta','file_type_code') }} ftc ON ftc.file_type_code = pfu.file_type_code - LEFT JOIN {{source ('fta','harvest_authority_geom') }} hag ON hva.hva_skey = hag.hva_skey - JOIN {{source ('fta','harvest_type_code') }} htc ON hva.harvest_type_code = htc.harvest_type_code - JOIN {{source ('fta','harvest_auth_status_code') }} hasc ON hasc.harvest_auth_status_code = hva.harvest_auth_status_code - LEFT JOIN {{source ('fta','tenure_application_map_feature') }} tamf ON tamf.map_feature_id = hag.map_feature_id - LEFT JOIN ( - select row_number()over(partition by forest_file_id, cutting_permit_id order by tenure_app_id desc) rn, * - from {{source ('fta','tenure_application') }} - ) ta ON - ta.forest_file_id = hva.forest_file_id - and hva.cutting_permit_id = ta.cutting_permit_id - and ta.rn =1 - left join {{source ('fta','org_unit') }} ou on (ta.org_unit_no = ou.org_unit_no) - LEFT JOIN {{source ('fta','tenure_application_state_code') }} tasc ON tasc.tenure_application_state_code = ta.tenure_application_state_code - LEFT JOIN {{source ('fta','tenure_application_type_code') }} tatc ON tatc.tenure_application_type_code = ta.tenure_application_type_code - WHERE (ftc.file_type_code = ANY (ARRAY['A01', 'A02', 'A03', 'A04', 'A05', 'A11', 'A18', 'A28', 'A29', 'A30', 'A31', 'A41', 'A44', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B09', 'B15', 'B16', 'E01', 'E02', 'H01', 'H02', 'S01', 'S02'])) AND (tasc.description = ANY (ARRAY['Inbox', 'Lobby'])) - )fta limit 100 -) -, rrs_rup_data as ( - Select - 'RRS_RUP' as src_sys_code - ,'FOR' as ministry_code - ,'ROADS' as business_area_code - ,permit_id as application_id - ,permit_id as permit_id - ,road_submission_id::varchar(32) as app_file_id - ,submission_status_code as authorization_status_code - ,null as project_id - ,null as permit_status_code - ,update_date::date as permit_status_date - ,null::date as permit_issue_date - ,null::date as permit_issue_expire_date - ,received_date::date as app_received_date - ,update_date::date as app_update_date - ,null::date as app_decision_date - ,null::date as app_issuance_date - ,received_date::date as app_accepted_date - ,null::date as app_rejected_date - ,update_date::date as app_adjudication_date - ,null as harvest_auth_status - ,null::int as harvest_area_sq_m - ,null as permit_org_unit_code - ,age as application_age - ,null ::bigint as map_feature_id - ,authorization_type as permit_type_code - ,'RRS_RUP' || '|'|| coalesce(cast(road_submission_id as varchar),'~')||'|'||coalesce(cast( permit_id as varchar),'~') as unqid - from - ( - SELECT - rup.permit_id::character varying AS permit_id, - rup.permit_type AS authorization_type, - rup.submission_status_code, - rup.road_submission_id, - rup.permit_id AS authorization_id, - NULL::timestamp(0) without time zone AS status_date, - NULL::timestamp(0) without time zone AS issue_date, - NULL::timestamp(0) without time zone AS current_expiry_date_calc, - NULL AS location, - NULL::integer AS harvest_area, - rup.received_date, - NULL::timestamp(0) without time zone AS app_decision_date, - NULL::timestamp(0) without time zone AS app_issuance_date, - NULL AS project_name, - NULL AS project_description, - NULL AS property_subclass, - NULL AS project_location, - NULL AS project_id, - NULL::timestamp without time zone AS accepted_date, - NULL::timestamp without time zone AS rejected_date, - NULL::timestamp without time zone AS adjudication_date, - NULL AS permit_status, - NULL::bigint AS map_feature_id, - NULL AS road_section_id, - NULL AS road_section_status, - NULL AS organization_unit_name, - NULL AS nrsos_smart_form_id, - rup.update_date, - NULL::character varying AS ats_region_name, - NULL::bigint AS hva_skey, - NULL::bytea AS road_section_guid, - floor(EXTRACT(epoch FROM CURRENT_DATE::timestamp without time zone - rup.received_date) / 86400::numeric) AS age - FROM ( SELECT - rs.nrsos_smart_form_id AS permit_id, - rs.road_submission_id, - 'RUP' AS permit_type, - rs.nrsos_smart_form_id, - rs.submission_date::date AS received_date, - rs.submission_status_code, - rs.update_date, - NULL AS project_name, - NULL AS project_description, - NULL AS property_subclass, - NULL AS project_location, - NULL AS utm_easting, - NULL AS utm_northing, - NULL AS utm_zone - FROM {{source ('rrs','road_submission') }} rs - WHERE - rs.road_submission_type_code = 'RUP' AND - rs.nrsos_smart_form_id IS NOT NULL AND - rs.submission_status_code = ANY (ARRAY['PRO'::character varying, 'SUB'::character varying]) - ) rup - ) rup -) -,rrs_rp_data as ( - Select - 'RRS_RP' as src_sys_code - ,'FOR' as ministry_code - ,'ROADS' as business_area_code - ,permit_id as application_id - ,permit_id as permit_id - ,application_id::varchar(32) as app_file_id - ,application_status as authorization_status_code - ,null as project_id - ,road_tenure_status_code as permit_status_code - ,adjudication_date::date as permit_status_date - ,app_issuance_date::date as permit_issue_date - ,null::date as permit_issue_expire_date - ,received_date::date as app_received_date - ,adjudication_date as app_update_date - ,app_decision_date::date as app_decision_date - ,app_issuance_date::date as app_issuance_date - ,received_date::date as app_accepted_date - ,case when application_status = 'REJ' then adjudication_date end as app_rejected_date - ,adjudication_date::date as app_adjudication_date - ,road_tenure_status_code as harvest_auth_status - ,null::int as harvest_area_sq_m - ,org_unit_code as permit_org_unit_code - ,age as application_age - ,map_feature_id as map_feature_id - ,permit_type as permit_type_code - ,'RRS_RP' || '|'|| coalesce(cast(application_id as varchar),'~')||'|'||coalesce(cast( permit_id as varchar),'~') as unqid - from - ( - SELECT - rp.permit_id::character varying AS permit_id, - rp.permit_type , - rp.application_id , - rp.application_status, - rp.road_tenure_status_code, - rp.application_submission_date AS received_date, - rp.application_decision_date::date AS app_decision_date, - rp.application_issuance_date::date AS app_issuance_date, - rp.application_adjudication_date::date AS adjudication_date, - rp.map_feature_id, - rp.road_section_id, - rp.road_section_status, - rp.org_unit_code, - rp.application_update_date AS update_date, - floor( - CASE - WHEN (rp.application_status = 'Approved' OR rp.application_status = 'Issued') AND rp.application_issuance_date IS NOT NULL THEN EXTRACT(epoch FROM rp.application_issuance_date - rp.application_submission_date) / 86400 - WHEN (rp.application_status = 'Approved' OR rp.application_status = 'Issued') AND rp.application_decision_date IS NOT NULL THEN EXTRACT(epoch FROM rp.application_decision_date - rp.application_submission_date) / 86400 - WHEN rp.application_status = 'Rejected' THEN EXTRACT(epoch FROM rp.application_update_date - rp.application_submission_date) / 86400 - ELSE EXTRACT(epoch FROM CURRENT_DATE::timestamp without time zone - rp.application_submission_date) / 86400 - END::numeric) AS age - - FROM ( SELECT - z.permit_id, - z.road_tenure_type_code AS permit_type, - z.road_tenure_status AS permit_status, - z.road_application_id AS application_id, - z.road_application_status AS application_status, - z.road_tenure_status_code, - z.map_feature_id, - z.road_section_id, - z.road_section_status, - z.org_unit_code, - z.submission_date AS application_submission_date, - z.adjudication_date AS application_adjudication_date, - z.decision_date AS application_decision_date, - z.issuance_date AS application_issuance_date, - z.create_date AS application_create_date, - z.update_date AS application_update_date, - z.geometry_road_section_guid - FROM ( SELECT t.resource_road_tenure_id AS permit_id, - t.road_tenure_type_code, - rtsc.description AS road_tenure_status, - ra.road_application_id, - rasc.description AS road_application_status, - t.road_tenure_status_code, - mf.map_feature_id, - rs.road_section_id, - rssc.description AS road_section_status, - rs.current_ind, - mf.feature_record_type_code, - mf.feature_business_identifier, - ra.submission_date, - ra.adjudication_date, - ra.decision_date, - ra.issuance_date, - ra.create_date, - ra.update_date, - fou.org_unit_code, - rs.geometry_road_section_guid - FROM {{source ('rrs','resource_road_tenure') }} t - JOIN {{source ('rrs','road_application') }} ra ON encode(ra.resource_road_tenure_guid, 'hex') = encode(t.resource_road_tenure_guid, 'hex') - JOIN (Select *, row_number()over( partition by encode(road_application_guid, 'hex') order by map_feature_id desc ) rn from {{source ('rrs','road_appl_map_feature') }} ) mf - ON (encode(mf.road_application_guid, 'hex') = encode(ra.road_application_guid, 'hex') and mf.rn =1) - JOIN {{source ('rrs','road_section') }} rs ON encode(mf.feature_record_guid, 'hex') = encode(rs.road_section_guid, 'hex') - JOIN {{source ('rrs','road_feature_class_sdw') }} fc ON encode(mf.road_feature_class_sdw_guid, 'hex') = encode(fc.road_feature_class_sdw_guid, 'hex') - JOIN {{source ('rrs','road_org_unit_sdw') }} ou ON encode(ou.road_org_unit_sdw_guid, 'hex') = encode(ra.metadata_org_unit_sdw_guid, 'hex') - left join {{source ('fta','org_unit') }} fou on (fou.org_unit_name = ou.organization_unit_name) - LEFT JOIN {{source ('rrs','road_application_status_code') }} rasc ON ra.road_application_status_code = rasc.road_application_status_code - LEFT JOIN {{source ('rrs','road_tenure_type_code') }} rttc ON t.road_tenure_type_code = rttc.road_tenure_type_code - LEFT JOIN {{source ('rrs','road_tenure_status_code') }} rtsc ON t.road_tenure_status_code = rtsc.road_tenure_status_code - LEFT JOIN {{source ('rrs','road_section_status_code') }} rssc ON rssc.road_section_status_code = rs.road_section_status_code - LEFT JOIN {{source ('rrs','road_submission') }} rsu ON encode(rsu.road_submission_guid, 'hex') = encode(ra.road_submission_guid, 'hex') - WHERE t.road_tenure_type_code <> 'B40' AND (rsu.road_submission_type_code = ANY (ARRAY['RP'::character varying, 'SRP'::character varying, 'SRL'::character varying]))) z - WHERE z.road_application_status = ANY (ARRAY['Inbox'::character varying, 'Lobby'::character varying])) rp - )rp -) ---insert into pmt_dpl.fact_permits -select * from ats_data -union ALL -select * from fta_data -union ALL -select * from rrs_rup_data -union ALL -select * from rrs_rp_data - - +{% snapshot fact_permits %} + + {{ + config( + target_schema='pmt_dpl', + strategy='check', + unique_key='unqid', + check_cols=['ministry_code','business_area_code','permit_status_code','permit_status_date','permit_issue_date','app_received_date','app_update_date','app_decision_date','app_issuance_date','app_accepted_date','app_rejected_date','app_adjudication_date','harvest_area_sq_m','harvest_auth_status','map_feature_id','permit_type_code'], + invalidate_hard_deletes=True, + bind=False, + ) + }} + +with ats_data as ( + Select + 'ATS' as src_sys_code + ,CASE + WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'EMLI' + WHEN ats.authorization_instrument_id = 27 THEN 'FOR' + WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'AGR' + WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107]) THEN 'ENV' + ELSE NULL --- ELSE NULL is causing null ministries + END AS ministry_code + ,CASE + WHEN ats.authorization_instrument_id = ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061]) THEN 'MIN' + WHEN ats.authorization_instrument_id = ANY (ARRAY[69, 542, 1481, 1121, 1124, 1125, 1122, 421, 68, 541, 70, 1, 71, 124]) THEN 'WAT' + WHEN ats.authorization_instrument_id = ANY (ARRAY[181, 73, 701, 702, 33, 704, 503, 82, 482, 710, 709, 1241, 481, 483, 502, 716, 708, 80, 39, 686, 685, 132, 84, 711, 684, 41, 110, 29, 714, 102, 105, 717, 521, 524, 103, 501, 718, 18, 720, 281, 89, 25, 24, 3, 4, 683, 713]) THEN 'LAN' + WHEN ats.authorization_instrument_id = ANY (ARRAY[1101, 1402, 1401, 57, 44, 1201, 45, 48, 851, 50, 941, 99, 51, 852, 58, 981, 52, 1081, 1083, 1082, 56, 1001, 401, 65, 881, 861, 846, 841, 843, 842, 847, 845, 441, 921, 11, 66, 100, 72]) THEN 'FAW' + WHEN ats.authorization_instrument_id = ANY (ARRAY[2, 20, 682, 19, 1421, 1441]) THEN 'AGRI' + WHEN ats.authorization_instrument_id = ANY (ARRAY[761, 341, 901, 27, 762, 10, 1181, 113, 1022, 1021, 119, 126, 28, 30, 1023, 1261, 641, 16, 1041, 721]) THEN 'FOR' + WHEN ats.authorization_instrument_id = ANY (ARRAY[1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342]) THEN 'PAR' + WHEN ats.authorization_instrument_id = ANY (ARRAY[21, 107]) THEN 'RECS' + ELSE NULL + END as business_area_code + ,ats.authorization_id::varchar as application_id + ,null ::varchar as permit_id + ,ats.file_number as app_file_id + ,ats.authorization_status_code as authorization_status_code + ,ats.project_id::varchar(20) as project_id + ,null as permit_status_code + ,null::date as permit_status_date + ,null::date as permit_issue_date + ,null::date as permit_issue_expire_date + ,ats.application_received_date as app_received_date + ,coalesce(ats.fcbc_process_complete_date, ats.adjudication_date,ats.application_accepted_date,ats.application_accepted_date) as app_update_date + ,ats.fcbc_process_complete_date as app_decision_date + ,ats.adjudication_date as app_issuance_date + ,ats.application_accepted_date as app_accepted_date + ,null::date as app_rejected_date + ,ats.adjudication_date as app_adjudication_date + ,null as harvest_auth_status + ,null::int as harvest_area_sq_m + ,null as permit_org_unit_code + ,floor( + CASE + WHEN ats.adjudication_date IS NULL THEN EXTRACT(epoch FROM CURRENT_DATE::timestamp without time zone - ats.application_received_date) / 86400 + ELSE EXTRACT(epoch FROM ats.adjudication_date - ats.application_received_date) / 86400 + END) AS application_age + ,null::bigint as map_feature_id + ,ats.authorization_instrument_id::varchar(30) as permit_type_code + ,'ATS' || '|'|| coalesce(cast(ats.authorization_id as varchar),'~')||'|'||coalesce(cast(ats.project_id as varchar),'~') as unqid +from {{source ('ats','ats_authorizations') }} ats +LEFT JOIN {{source ('ats','ats_authorization_status_codes') }} aasc + ON(ats.authorization_status_code = aasc.authorization_status_code) +LEFT JOIN {{source ('ats','ats_authorization_instruments') }} aai + ON(ats.authorization_instrument_id = aai.authorization_instrument_id) +where 1=1 +AND aasc.authorization_status_code = '1' +AND (ats.authorization_instrument_id = +ANY (ARRAY[961, 120, 1141, 76, 741, 742, 581, 742, 582, 744, 743, 583, 1061, 69, 542, 1481, 1121, 1124, +1125, 1122, 421, 68, 541, 70, 1, 71, 124, 181, 73, 701, 702, 33, 704, 503, 82, 482, 710, 709, 1241, 481, 483, 502, 716, 708, 80, 39, 686, 685, 132, 84, 711, 684, 41, 110, 29, +714, 102, 105, 717, 521, 524, 103, 501, 718, 18, 720, 281, 89, 25, 24, 3, 4, 683, 713, 1101, 1402, 1401, 57, 44, +1201, 45, 48, 851, 50, 941, 99, 51, 852, 58, 981, 52, 1081, 1083, 1082, 56, 1001, 401, 65, 881, 861, 846, 841, 843, 842, 847, 845, 441, 921, 11, 66, 100, 72, 2, 20, 682, 19, 1421, 1441, 761, +341, 901, 27, 762, 10, 1181, 113, 1022, 1021, 119, 126, 28, 30, 1023, 1261, 641, 16, 1041, 721, 1282, 1281, 1343, 1344, 1361, 1301, 1302, 1303, 1304, 1283, 60, 1341, 1305, 1306, 1342, 21, 107] +)) +) +,fta_data as +( +Select +'FTA' as src_sys_code +,'FOR' as ministry_code +,'FORESTS' as business_area_code +,tenure_app_id::varchar as application_id +,cutting_permit_id as permit_id +,forest_file_id as app_file_id +,tenure_application_state_code as authorization_status_code +,null as project_id +,permit_status_code as permit_status_code +,adjudication_date as permit_status_date +,issuance_date as permit_issue_date +,expiry_date as permit_issue_expire_date +,submission_date as app_received_date +,adjudication_date as app_update_date +,decision_date as app_decision_date +,issuance_date as app_issuance_date +,submission_date as app_accepted_date +,case when tenure_application_state_code = 'REJ' then adjudication_date end as app_rejected_date +,adjudication_date as app_adjudication_date +,permit_status_code as harvest_auth_status +,harvest_area::int as harvest_area_sq_m +,org_unit_code as permit_org_unit_code +,extract(epoch from age)/86400 as application_age +,map_feature_id as map_feature_id +,permit_type_code as permit_type_code +,'FTA' || '|'|| coalesce(cast(forest_file_id as varchar),'~')||'|'||coalesce(cast( cutting_permit_id as varchar),'~') as unqid +from +( SELECT + hva.forest_file_id , + ou.org_unit_code as org_unit_code, + pfu.file_type_code AS permit_type_code, + ftc.description AS permit_type, + COALESCE(hva.cutting_permit_id, hva.forest_file_id) AS cutting_permit_id, + htc.description AS havesting_auth_type, + hasc.harvest_auth_status_code as permit_status_code, + tatc.description AS tenure_application_type, + tasc.description AS tenure_application_state_code, + hva.tenure_term, + hva.status_date, + hva.issue_date, + hva.expiry_date, + hva.extend_date, + CASE + WHEN hva.extend_date IS NOT NULL THEN hva.extend_date + WHEN hva.expiry_date IS NOT NULL THEN hva.expiry_date + ELSE GREATEST(hva.extend_date, hva.expiry_date) + END AS current_expiry_date_calc, + hva.location, + hva.harvest_area, + ta.tenure_app_id, + ta.tenure_appname, + ta.submission_id, + ta.client_number, + ta.adjudication_ind, + ta.adjudication_date, + ta.submission_date, + ta.utm_zone, + ta.decision_date, + ta.issuance_date, + CASE + WHEN ta.tenure_application_state_code = 'APP' THEN ta.decision_date - ta.submission_date + WHEN ta.tenure_application_state_code = ANY (ARRAY['ISS', 'REJ']) THEN ta.issuance_date - ta.submission_date + ELSE CURRENT_DATE::timestamp without time zone - ta.submission_date + END AS age, + null as geometry, + hag.map_feature_id, + NULL AS project_name, + NULL AS project_description, + NULL AS property_subclass, + NULL AS project_location, + NULL AS utm_easting, + NULL AS utm_northing, + NULL AS utm_zone, + hva.hva_skey + FROM ( SELECT harvesting_authority.hva_skey, + harvesting_authority.forest_file_id, + harvesting_authority.cutting_permit_id, + harvesting_authority.harvesting_authority_id, + harvesting_authority.forest_district, + harvesting_authority.district_admn_zone, + harvesting_authority.geographic_district, + harvesting_authority.mgmt_unit_id, + harvesting_authority.mgmt_unit_type_code, + harvesting_authority.licence_to_cut_code, + harvesting_authority.harvest_type_code, + harvesting_authority.harvest_auth_status_code, + harvesting_authority.tenure_term, + harvesting_authority.status_date, + harvesting_authority.issue_date, + harvesting_authority.expiry_date, + harvesting_authority.extend_date, + harvesting_authority.extend_count, + harvesting_authority.harvest_auth_extend_reas_code, + harvesting_authority.quota_type_code, + harvesting_authority.crown_lands_region_code, + harvesting_authority.salvage_type_code, + harvesting_authority.cascade_split_code, + harvesting_authority.catastrophic_ind, + harvesting_authority.crown_granted_ind, + harvesting_authority.cruise_based_ind, + harvesting_authority.deciduous_ind, + harvesting_authority.bcaa_folio_number, + harvesting_authority.location, + harvesting_authority.higher_level_plan_reference, + harvesting_authority.harvest_area, + harvesting_authority.retirement_date, + harvesting_authority.revision_count, + harvesting_authority.entry_userid, + harvesting_authority.entry_timestamp, + harvesting_authority.update_userid, + harvesting_authority.update_timestamp, + harvesting_authority.is_waste_assessment_required, + harvesting_authority.is_cp_extensn_appl_fee_waived, + harvesting_authority.is_cp_extension_appl_fee_paid, + harvesting_authority.is_within_fibre_recovery_zone, + harvesting_authority.harvesting_authority_guid + FROM {{source ('fta','harvesting_authority') }} + WHERE (harvesting_authority.forest_file_id IN ( SELECT zz.forest_file_id + FROM ( SELECT a.forest_file_id, + count(a.cutting_permit_id) AS count + FROM {{source ('fta','harvesting_authority') }} a + JOIN ( SELECT DISTINCT harvesting_authority_1.forest_file_id + FROM {{source ('fta','harvesting_authority') }} harvesting_authority_1 + WHERE harvesting_authority_1.cutting_permit_id IS NULL) z ON a.forest_file_id = z.forest_file_id + GROUP BY a.forest_file_id + HAVING count(a.cutting_permit_id) = 0) zz))) hva + JOIN {{source ('fta','prov_forest_use') }} pfu ON hva.forest_file_id = pfu.forest_file_id + JOIN {{source ('fta','file_type_code') }} ftc ON ftc.file_type_code = pfu.file_type_code + LEFT JOIN {{source ('fta','harvest_authority_geom') }} hag ON hva.hva_skey = hag.hva_skey + JOIN {{source ('fta','harvest_type_code') }} htc ON hva.harvest_type_code = htc.harvest_type_code + JOIN {{source ('fta','harvest_auth_status_code') }} hasc ON hasc.harvest_auth_status_code = hva.harvest_auth_status_code + LEFT JOIN {{source ('fta','tenure_application_map_feature') }} tamf ON tamf.map_feature_id = hag.map_feature_id + LEFT JOIN {{source ('fta','tenure_application') }} ta ON ta.forest_file_id = hva.forest_file_id and hva.cutting_permit_id = ta.cutting_permit_id + left join {{source ('fta','org_unit') }} ou on (ta.org_unit_no = ou.org_unit_no) + LEFT JOIN {{source ('fta','tenure_application_state_code') }} tasc ON tasc.tenure_application_state_code = ta.tenure_application_state_code + LEFT JOIN {{source ('fta','tenure_application_type_code') }} tatc ON tatc.tenure_application_type_code = ta.tenure_application_type_code + WHERE (ftc.file_type_code = ANY (ARRAY['A01', 'A02', 'A03', 'A04', 'A05', 'A11', 'A18', 'A28', 'A29', 'A30', 'A31', 'A41', 'A44', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B09', 'B15', 'B16', 'E01', 'E02', 'H01', 'H02', 'S01', 'S02'])) AND (tasc.description = ANY (ARRAY['Inbox', 'Lobby'])) + +UNION ALL + SELECT hva.forest_file_id , + ou.org_unit_code as org_unit_code, + pfu.file_type_code AS permit_type_code, + ftc.description AS permit_type, + COALESCE(hva.cutting_permit_id, hva.forest_file_id) AS cutting_permit_id, + htc.description AS havesting_auth_type, + hasc.harvest_auth_status_code as permit_status_code, + tatc.description AS tenure_application_type, + tasc.description AS tenure_application_state_code, + hva.tenure_term, + hva.status_date, + hva.issue_date, + hva.expiry_date, + hva.extend_date, + CASE + WHEN hva.extend_date IS NOT NULL THEN hva.extend_date + WHEN hva.expiry_date IS NOT NULL THEN hva.expiry_date + ELSE GREATEST(hva.extend_date, hva.expiry_date) + END AS current_expiry_date_calc, + hva.location, + hva.harvest_area, + ta.tenure_app_id, + ta.tenure_appname, + ta.submission_id, + ta.client_number, + ta.adjudication_ind, + ta.adjudication_date, + ta.submission_date, + ta.utm_zone, + ta.decision_date, + ta.issuance_date, + CASE + WHEN ta.tenure_application_state_code = 'APP' THEN ta.decision_date - ta.submission_date + WHEN ta.tenure_application_state_code = ANY (ARRAY['ISS', 'REJ']) THEN ta.issuance_date - ta.submission_date + ELSE CURRENT_DATE::timestamp without time zone - ta.submission_date + END AS age, + null as geometry, + hag.map_feature_id, + NULL AS project_name, + NULL AS project_description, + NULL AS property_subclass, + NULL AS project_location, + NULL AS utm_easting, + NULL AS utm_northing, + NULL AS utm_zone, + hva.hva_skey + FROM ( SELECT harvesting_authority.hva_skey, + harvesting_authority.forest_file_id, + harvesting_authority.cutting_permit_id, + harvesting_authority.harvesting_authority_id, + harvesting_authority.forest_district, + harvesting_authority.district_admn_zone, + harvesting_authority.geographic_district, + harvesting_authority.mgmt_unit_id, + harvesting_authority.mgmt_unit_type_code, + harvesting_authority.licence_to_cut_code, + harvesting_authority.harvest_type_code, + harvesting_authority.harvest_auth_status_code, + harvesting_authority.tenure_term, + harvesting_authority.status_date, + harvesting_authority.issue_date, + harvesting_authority.expiry_date, + harvesting_authority.extend_date, + harvesting_authority.extend_count, + harvesting_authority.harvest_auth_extend_reas_code, + harvesting_authority.quota_type_code, + harvesting_authority.crown_lands_region_code, + harvesting_authority.salvage_type_code, + harvesting_authority.cascade_split_code, + harvesting_authority.catastrophic_ind, + harvesting_authority.crown_granted_ind, + harvesting_authority.cruise_based_ind, + harvesting_authority.deciduous_ind, + harvesting_authority.bcaa_folio_number, + harvesting_authority.location, + harvesting_authority.higher_level_plan_reference, + harvesting_authority.harvest_area, + harvesting_authority.retirement_date, + harvesting_authority.revision_count, + harvesting_authority.entry_userid, + harvesting_authority.entry_timestamp, + harvesting_authority.update_userid, + harvesting_authority.update_timestamp, + harvesting_authority.is_waste_assessment_required, + harvesting_authority.is_cp_extensn_appl_fee_waived, + harvesting_authority.is_cp_extension_appl_fee_paid, + harvesting_authority.is_within_fibre_recovery_zone, + harvesting_authority.harvesting_authority_guid + FROM {{source ('fta','harvesting_authority') }} + WHERE (harvesting_authority.forest_file_id IN ( SELECT zz.forest_file_id + FROM ( SELECT a.forest_file_id, + count(a.cutting_permit_id) AS count + FROM {{source ('fta','harvesting_authority') }} a + JOIN ( SELECT DISTINCT harvesting_authority_1.forest_file_id + FROM {{source ('fta','harvesting_authority') }} harvesting_authority_1 + WHERE harvesting_authority_1.cutting_permit_id IS NOT NULL) z ON a.forest_file_id = z.forest_file_id + GROUP BY a.forest_file_id + HAVING count(a.cutting_permit_id) > 0) zz))) hva + JOIN {{source ('fta','prov_forest_use') }} pfu ON hva.forest_file_id = pfu.forest_file_id + JOIN {{source ('fta','file_type_code') }} ftc ON ftc.file_type_code = pfu.file_type_code + LEFT JOIN {{source ('fta','harvest_authority_geom') }} hag ON hva.hva_skey = hag.hva_skey + JOIN {{source ('fta','harvest_type_code') }} htc ON hva.harvest_type_code = htc.harvest_type_code + JOIN {{source ('fta','harvest_auth_status_code') }} hasc ON hasc.harvest_auth_status_code = hva.harvest_auth_status_code + LEFT JOIN {{source ('fta','tenure_application_map_feature') }} tamf ON tamf.map_feature_id = hag.map_feature_id + LEFT JOIN ( + select row_number()over(partition by forest_file_id, cutting_permit_id order by tenure_app_id desc) rn, * + from {{source ('fta','tenure_application') }} + ) ta ON + ta.forest_file_id = hva.forest_file_id + and hva.cutting_permit_id = ta.cutting_permit_id + and ta.rn =1 + left join {{source ('fta','org_unit') }} ou on (ta.org_unit_no = ou.org_unit_no) + LEFT JOIN {{source ('fta','tenure_application_state_code') }} tasc ON tasc.tenure_application_state_code = ta.tenure_application_state_code + LEFT JOIN {{source ('fta','tenure_application_type_code') }} tatc ON tatc.tenure_application_type_code = ta.tenure_application_type_code + WHERE (ftc.file_type_code = ANY (ARRAY['A01', 'A02', 'A03', 'A04', 'A05', 'A11', 'A18', 'A28', 'A29', 'A30', 'A31', 'A41', 'A44', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B09', 'B15', 'B16', 'E01', 'E02', 'H01', 'H02', 'S01', 'S02'])) AND (tasc.description = ANY (ARRAY['Inbox', 'Lobby'])) + )fta limit 100 +) +, rrs_rup_data as ( + Select + 'RRS_RUP' as src_sys_code + ,'FOR' as ministry_code + ,'ROADS' as business_area_code + ,permit_id as application_id + ,permit_id as permit_id + ,road_submission_id::varchar(32) as app_file_id + ,submission_status_code as authorization_status_code + ,null as project_id + ,null as permit_status_code + ,update_date::date as permit_status_date + ,null::date as permit_issue_date + ,null::date as permit_issue_expire_date + ,received_date::date as app_received_date + ,update_date::date as app_update_date + ,null::date as app_decision_date + ,null::date as app_issuance_date + ,received_date::date as app_accepted_date + ,null::date as app_rejected_date + ,update_date::date as app_adjudication_date + ,null as harvest_auth_status + ,null::int as harvest_area_sq_m + ,null as permit_org_unit_code + ,age as application_age + ,null ::bigint as map_feature_id + ,authorization_type as permit_type_code + ,'RRS_RUP' || '|'|| coalesce(cast(road_submission_id as varchar),'~')||'|'||coalesce(cast( permit_id as varchar),'~') as unqid + from + ( + SELECT + rup.permit_id::character varying AS permit_id, + rup.permit_type AS authorization_type, + rup.submission_status_code, + rup.road_submission_id, + rup.permit_id AS authorization_id, + NULL::timestamp(0) without time zone AS status_date, + NULL::timestamp(0) without time zone AS issue_date, + NULL::timestamp(0) without time zone AS current_expiry_date_calc, + NULL AS location, + NULL::integer AS harvest_area, + rup.received_date, + NULL::timestamp(0) without time zone AS app_decision_date, + NULL::timestamp(0) without time zone AS app_issuance_date, + NULL AS project_name, + NULL AS project_description, + NULL AS property_subclass, + NULL AS project_location, + NULL AS project_id, + NULL::timestamp without time zone AS accepted_date, + NULL::timestamp without time zone AS rejected_date, + NULL::timestamp without time zone AS adjudication_date, + NULL AS permit_status, + NULL::bigint AS map_feature_id, + NULL AS road_section_id, + NULL AS road_section_status, + NULL AS organization_unit_name, + NULL AS nrsos_smart_form_id, + rup.update_date, + NULL::character varying AS ats_region_name, + NULL::bigint AS hva_skey, + NULL::bytea AS road_section_guid, + floor(EXTRACT(epoch FROM CURRENT_DATE::timestamp without time zone - rup.received_date) / 86400::numeric) AS age + FROM ( SELECT + rs.nrsos_smart_form_id AS permit_id, + rs.road_submission_id, + 'RUP' AS permit_type, + rs.nrsos_smart_form_id, + rs.submission_date::date AS received_date, + rs.submission_status_code, + rs.update_date, + NULL AS project_name, + NULL AS project_description, + NULL AS property_subclass, + NULL AS project_location, + NULL AS utm_easting, + NULL AS utm_northing, + NULL AS utm_zone + FROM {{source ('rrs','road_submission') }} rs + WHERE + rs.road_submission_type_code = 'RUP' AND + rs.nrsos_smart_form_id IS NOT NULL AND + rs.submission_status_code = ANY (ARRAY['PRO'::character varying, 'SUB'::character varying]) + ) rup + ) rup +) +,rrs_rp_data as ( + Select + 'RRS_RP' as src_sys_code + ,'FOR' as ministry_code + ,'ROADS' as business_area_code + ,permit_id as application_id + ,permit_id as permit_id + ,application_id::varchar(32) as app_file_id + ,application_status as authorization_status_code + ,null as project_id + ,road_tenure_status_code as permit_status_code + ,adjudication_date::date as permit_status_date + ,app_issuance_date::date as permit_issue_date + ,null::date as permit_issue_expire_date + ,received_date::date as app_received_date + ,adjudication_date as app_update_date + ,app_decision_date::date as app_decision_date + ,app_issuance_date::date as app_issuance_date + ,received_date::date as app_accepted_date + ,case when application_status = 'REJ' then adjudication_date end as app_rejected_date + ,adjudication_date::date as app_adjudication_date + ,road_tenure_status_code as harvest_auth_status + ,null::int as harvest_area_sq_m + ,org_unit_code as permit_org_unit_code + ,age as application_age + ,map_feature_id as map_feature_id + ,permit_type as permit_type_code + ,'RRS_RP' || '|'|| coalesce(cast(application_id as varchar),'~')||'|'||coalesce(cast( permit_id as varchar),'~') as unqid + from + ( + SELECT + rp.permit_id::character varying AS permit_id, + rp.permit_type , + rp.application_id , + rp.application_status, + rp.road_tenure_status_code, + rp.application_submission_date AS received_date, + rp.application_decision_date::date AS app_decision_date, + rp.application_issuance_date::date AS app_issuance_date, + rp.application_adjudication_date::date AS adjudication_date, + rp.map_feature_id, + rp.road_section_id, + rp.road_section_status, + rp.org_unit_code, + rp.application_update_date AS update_date, + floor( + CASE + WHEN (rp.application_status = 'Approved' OR rp.application_status = 'Issued') AND rp.application_issuance_date IS NOT NULL THEN EXTRACT(epoch FROM rp.application_issuance_date - rp.application_submission_date) / 86400 + WHEN (rp.application_status = 'Approved' OR rp.application_status = 'Issued') AND rp.application_decision_date IS NOT NULL THEN EXTRACT(epoch FROM rp.application_decision_date - rp.application_submission_date) / 86400 + WHEN rp.application_status = 'Rejected' THEN EXTRACT(epoch FROM rp.application_update_date - rp.application_submission_date) / 86400 + ELSE EXTRACT(epoch FROM CURRENT_DATE::timestamp without time zone - rp.application_submission_date) / 86400 + END::numeric) AS age + + FROM ( SELECT + z.permit_id, + z.road_tenure_type_code AS permit_type, + z.road_tenure_status AS permit_status, + z.road_application_id AS application_id, + z.road_application_status AS application_status, + z.road_tenure_status_code, + z.map_feature_id, + z.road_section_id, + z.road_section_status, + z.org_unit_code, + z.submission_date AS application_submission_date, + z.adjudication_date AS application_adjudication_date, + z.decision_date AS application_decision_date, + z.issuance_date AS application_issuance_date, + z.create_date AS application_create_date, + z.update_date AS application_update_date, + z.geometry_road_section_guid + FROM ( SELECT t.resource_road_tenure_id AS permit_id, + t.road_tenure_type_code, + rtsc.description AS road_tenure_status, + ra.road_application_id, + rasc.description AS road_application_status, + t.road_tenure_status_code, + mf.map_feature_id, + rs.road_section_id, + rssc.description AS road_section_status, + rs.current_ind, + mf.feature_record_type_code, + mf.feature_business_identifier, + ra.submission_date, + ra.adjudication_date, + ra.decision_date, + ra.issuance_date, + ra.create_date, + ra.update_date, + fou.org_unit_code, + rs.geometry_road_section_guid + FROM {{source ('rrs','resource_road_tenure') }} t + JOIN {{source ('rrs','road_application') }} ra ON encode(ra.resource_road_tenure_guid, 'hex') = encode(t.resource_road_tenure_guid, 'hex') + JOIN (Select *, row_number()over( partition by encode(road_application_guid, 'hex') order by map_feature_id desc ) rn from {{source ('rrs','road_appl_map_feature') }} ) mf + ON (encode(mf.road_application_guid, 'hex') = encode(ra.road_application_guid, 'hex') and mf.rn =1) + JOIN {{source ('rrs','road_section') }} rs ON encode(mf.feature_record_guid, 'hex') = encode(rs.road_section_guid, 'hex') + JOIN {{source ('rrs','road_feature_class_sdw') }} fc ON encode(mf.road_feature_class_sdw_guid, 'hex') = encode(fc.road_feature_class_sdw_guid, 'hex') + JOIN {{source ('rrs','road_org_unit_sdw') }} ou ON encode(ou.road_org_unit_sdw_guid, 'hex') = encode(ra.metadata_org_unit_sdw_guid, 'hex') + left join {{source ('fta','org_unit') }} fou on (fou.org_unit_name = ou.organization_unit_name) + LEFT JOIN {{source ('rrs','road_application_status_code') }} rasc ON ra.road_application_status_code = rasc.road_application_status_code + LEFT JOIN {{source ('rrs','road_tenure_type_code') }} rttc ON t.road_tenure_type_code = rttc.road_tenure_type_code + LEFT JOIN {{source ('rrs','road_tenure_status_code') }} rtsc ON t.road_tenure_status_code = rtsc.road_tenure_status_code + LEFT JOIN {{source ('rrs','road_section_status_code') }} rssc ON rssc.road_section_status_code = rs.road_section_status_code + LEFT JOIN {{source ('rrs','road_submission') }} rsu ON encode(rsu.road_submission_guid, 'hex') = encode(ra.road_submission_guid, 'hex') + WHERE t.road_tenure_type_code <> 'B40' AND (rsu.road_submission_type_code = ANY (ARRAY['RP'::character varying, 'SRP'::character varying, 'SRL'::character varying]))) z + WHERE z.road_application_status = ANY (ARRAY['Inbox'::character varying, 'Lobby'::character varying])) rp + )rp +) +--insert into pmt_dpl.fact_permits +select * from ats_data +union ALL +select * from fta_data +union ALL +select * from rrs_rup_data +union ALL +select * from rrs_rp_data + + {% endsnapshot %} \ No newline at end of file diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/qa_dpl_pmt.yml b/permitting/dbt/pmt_dbt/snapshots/dpl/qa_dpl_pmt.yml similarity index 100% rename from pmt/dbt/pmt_dbt/snapshots/dpl/qa_dpl_pmt.yml rename to permitting/dbt/pmt_dbt/snapshots/dpl/qa_dpl_pmt.yml diff --git a/pmt/dbt/pmt_dbt/snapshots/dpl/src_pmt.yml b/permitting/dbt/pmt_dbt/snapshots/dpl/src_pmt.yml similarity index 100% rename from pmt/dbt/pmt_dbt/snapshots/dpl/src_pmt.yml rename to permitting/dbt/pmt_dbt/snapshots/dpl/src_pmt.yml diff --git a/pmt/dbt/pmt_dbt/tests/.gitkeep b/permitting/dbt/pmt_dbt/tests/.gitkeep similarity index 100% rename from pmt/dbt/pmt_dbt/tests/.gitkeep rename to permitting/dbt/pmt_dbt/tests/.gitkeep diff --git a/pmt/DDL/schema.sql b/permitting/ddl/schema.sql similarity index 100% rename from pmt/DDL/schema.sql rename to permitting/ddl/schema.sql diff --git a/pmt/DDL/tables.sql b/permitting/ddl/tables.sql similarity index 97% rename from pmt/DDL/tables.sql rename to permitting/ddl/tables.sql index 950009b..9c03733 100644 --- a/pmt/DDL/tables.sql +++ b/permitting/ddl/tables.sql @@ -1,231 +1,231 @@ -CREATE TABLE IF NOT EXISTS pmt_dpl.dim_authorization_status -( - src_sys_code varchar(10), - authorization_status_code varchar(30), - authorization_status_description varchar(100), - unqid varchar(40), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_authorization_status TO pmt_write_role; -GRANT SELECT ON pmt_dpl.dim_authorization_status TO pmt_read_role; - -CREATE TABLE IF NOT EXISTS pmt_dpl.dim_business_area -( - src_sys_code varchar(10), - business_area_code varchar(10), - business_area_description varchar(75), - unqid varchar(20), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_business_area TO pmt_write_role; -GRANT SELECT ON pmt_dpl.dim_business_area TO pmt_read_role; - -CREATE TABLE IF NOT EXISTS pmt_dpl.dim_location -( - src_sys_code varchar(20), - location_code varchar(200), - project_code varchar(125), - unqid varchar(220), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT INSERT, DELETE, UPDATE, SELECT ON pmt_dpl.dim_location TO pmt_write_role; -GRANT SELECT ON pmt_dpl.dim_location TO pmt_read_role; - -CREATE TABLE IF NOT EXISTS pmt_dpl.dim_ministry -( - src_sys_code varchar(10), - ministry_code varchar(10), - ministry_description varchar(75), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_ministry TO pmt_write_role; -GRANT SELECT ON pmt_dpl.dim_ministry TO pmt_read_role; - -CREATE TABLE IF NOT EXISTS pmt_dpl.dim_org -( - src_sys_code varchar(256), - org_unit_code varchar(6), - org_unit_description varchar(100), - rollup_region_code varchar(6), - rollup_region_description varchar(100), - rollup_district_code varchar(6), - rollup_district_description varchar(100), - roll_up_area_code varchar(256), - roll_up_area_description varchar(256), - unqid varchar(256), - dbt_scd_id varchar(256), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_org TO pmt_write_role; -GRANT SELECT ON pmt_dpl.dim_org TO pmt_read_role; - -CREATE TABLE IF NOT EXISTS pmt_dpl.dim_permit_status -( - src_sys_code varchar(10), - permit_status_code varchar(30), - permit_status_description varchar(100), - unqid varchar(40), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_permit_status TO pmt_write_role; -GRANT SELECT ON pmt_dpl.dim_permit_status TO pmt_read_role; - -CREATE TABLE IF NOT EXISTS pmt_dpl.dim_permit_type -( - src_sys_code varchar(10), - permit_type_code varchar(30), - permit_type_description varchar(120), - unqid varchar(30), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_permit_type TO pmt_write_role; -GRANT SELECT ON pmt_dpl.dim_permit_type TO pmt_read_role; - -CREATE TABLE IF NOT EXISTS pmt_dpl.dim_project -( - src_sys_code varchar(20), - project_id varchar(20), - project_name varchar(125), - project_location_code varchar(200), - project_status_code varchar(6), - project_region_description varchar(125), - unqid varchar(30), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT INSERT, DELETE, UPDATE, SELECT ON pmt_dpl.dim_project TO pmt_write_role; -GRANT SELECT ON pmt_dpl.dim_project TO pmt_read_role; - -CREATE TABLE IF NOT EXISTS pmt_dpl.dim_source_system -( - src_sys_code varchar(20), - src_sys_description varchar(100), - unqid varchar(20), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT INSERT, DELETE, UPDATE, SELECT ON pmt_dpl.dim_source_system TO pmt_write_role; -GRANT SELECT ON pmt_dpl.dim_source_system TO pmt_read_role; - -CREATE TABLE IF NOT EXISTS pmt_dpl.fact_permits -( - src_sys_code varchar(75), - ministry_code varchar(10), - business_area_code varchar(10), - application_id varchar(35), - permit_id varchar(30), - app_file_id varchar(35), - authorization_status_code varchar(30), - project_id varchar(30), - permit_status_code varchar(30), - permit_status_date timestamp, - permit_issue_date timestamp, - permit_issue_expire_date timestamp, - app_received_date timestamp, - app_update_date timestamp, - app_decision_date timestamp, - app_issuance_date timestamp, - app_accepted_date timestamp, - app_rejected_date timestamp, - app_adjudication_date timestamp, - harvest_auth_status varchar(30), - harvest_area_sq_m integer, - permit_org_unit_code varchar(5), - application_age numeric, - map_feature_id bigint, - permit_type_code varchar, - unqid varchar(70), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.fact_permits TO pmt_write_role; -GRANT SELECT ON pmt_dpl.fact_permits TO pmt_read_role; - - -CREATE TABLE IF NOT EXISTS pmt_dal.extract_permits -( - src_sys_code varchar(75), - src_sys_description varchar(100), - ministry_code varchar(10), - ministry_description varchar(75), - business_area_code varchar(10), - business_area_description varchar(75), - application_id varchar(35), - permit_id varchar(30), - app_file_id varchar(35), - authorization_status_code varchar(30), - authorization_status_description varchar(100), - project_id varchar(30), - project_name varchar(125), - permit_status_code varchar(30), - permit_status_description varchar(100), - permit_status_date timestamp, - permit_issue_date timestamp, - permit_issue_expire_date timestamp, - app_received_date timestamp, - app_update_date timestamp, - app_decision_date timestamp, - app_issuance_date timestamp, - app_accepted_date timestamp, - app_rejected_date timestamp, - app_adjudication_date timestamp, - harvest_auth_status varchar(30), - harvest_area_sq_m integer, - permit_org_unit_code varchar(5), - district_description varchar(100), - district_code varchar(5), - rollup_district_description varchar(100), - rollup_district_code varchar(5), - region_name varchar(100), - region_code varchar(6), - area_code varchar(5), - area_description varchar(256), - application_age numeric, - map_feature_id bigint, - permit_type_code varchar, - permit_type_description varchar(120), - unqid varchar(70), - dbt_scd_id varchar(100), - dbt_updated_at timestamp, - dbt_valid_from timestamp, - dbt_valid_to timestamp -); - -GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dal.extract_permits TO pmt_write_role; +CREATE TABLE IF NOT EXISTS pmt_dpl.dim_authorization_status +( + src_sys_code varchar(10), + authorization_status_code varchar(30), + authorization_status_description varchar(100), + unqid varchar(40), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_authorization_status TO pmt_write_role; +GRANT SELECT ON pmt_dpl.dim_authorization_status TO pmt_read_role; + +CREATE TABLE IF NOT EXISTS pmt_dpl.dim_business_area +( + src_sys_code varchar(10), + business_area_code varchar(10), + business_area_description varchar(75), + unqid varchar(20), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_business_area TO pmt_write_role; +GRANT SELECT ON pmt_dpl.dim_business_area TO pmt_read_role; + +CREATE TABLE IF NOT EXISTS pmt_dpl.dim_location +( + src_sys_code varchar(20), + location_code varchar(200), + project_code varchar(125), + unqid varchar(220), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT INSERT, DELETE, UPDATE, SELECT ON pmt_dpl.dim_location TO pmt_write_role; +GRANT SELECT ON pmt_dpl.dim_location TO pmt_read_role; + +CREATE TABLE IF NOT EXISTS pmt_dpl.dim_ministry +( + src_sys_code varchar(10), + ministry_code varchar(10), + ministry_description varchar(75), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_ministry TO pmt_write_role; +GRANT SELECT ON pmt_dpl.dim_ministry TO pmt_read_role; + +CREATE TABLE IF NOT EXISTS pmt_dpl.dim_org +( + src_sys_code varchar(256), + org_unit_code varchar(6), + org_unit_description varchar(100), + rollup_region_code varchar(6), + rollup_region_description varchar(100), + rollup_district_code varchar(6), + rollup_district_description varchar(100), + roll_up_area_code varchar(256), + roll_up_area_description varchar(256), + unqid varchar(256), + dbt_scd_id varchar(256), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_org TO pmt_write_role; +GRANT SELECT ON pmt_dpl.dim_org TO pmt_read_role; + +CREATE TABLE IF NOT EXISTS pmt_dpl.dim_permit_status +( + src_sys_code varchar(10), + permit_status_code varchar(30), + permit_status_description varchar(100), + unqid varchar(40), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_permit_status TO pmt_write_role; +GRANT SELECT ON pmt_dpl.dim_permit_status TO pmt_read_role; + +CREATE TABLE IF NOT EXISTS pmt_dpl.dim_permit_type +( + src_sys_code varchar(10), + permit_type_code varchar(30), + permit_type_description varchar(120), + unqid varchar(30), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.dim_permit_type TO pmt_write_role; +GRANT SELECT ON pmt_dpl.dim_permit_type TO pmt_read_role; + +CREATE TABLE IF NOT EXISTS pmt_dpl.dim_project +( + src_sys_code varchar(20), + project_id varchar(20), + project_name varchar(125), + project_location_code varchar(200), + project_status_code varchar(6), + project_region_description varchar(125), + unqid varchar(30), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT INSERT, DELETE, UPDATE, SELECT ON pmt_dpl.dim_project TO pmt_write_role; +GRANT SELECT ON pmt_dpl.dim_project TO pmt_read_role; + +CREATE TABLE IF NOT EXISTS pmt_dpl.dim_source_system +( + src_sys_code varchar(20), + src_sys_description varchar(100), + unqid varchar(20), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT INSERT, DELETE, UPDATE, SELECT ON pmt_dpl.dim_source_system TO pmt_write_role; +GRANT SELECT ON pmt_dpl.dim_source_system TO pmt_read_role; + +CREATE TABLE IF NOT EXISTS pmt_dpl.fact_permits +( + src_sys_code varchar(75), + ministry_code varchar(10), + business_area_code varchar(10), + application_id varchar(35), + permit_id varchar(30), + app_file_id varchar(35), + authorization_status_code varchar(30), + project_id varchar(30), + permit_status_code varchar(30), + permit_status_date timestamp, + permit_issue_date timestamp, + permit_issue_expire_date timestamp, + app_received_date timestamp, + app_update_date timestamp, + app_decision_date timestamp, + app_issuance_date timestamp, + app_accepted_date timestamp, + app_rejected_date timestamp, + app_adjudication_date timestamp, + harvest_auth_status varchar(30), + harvest_area_sq_m integer, + permit_org_unit_code varchar(5), + application_age numeric, + map_feature_id bigint, + permit_type_code varchar, + unqid varchar(70), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dpl.fact_permits TO pmt_write_role; +GRANT SELECT ON pmt_dpl.fact_permits TO pmt_read_role; + + +CREATE TABLE IF NOT EXISTS pmt_dal.extract_permits +( + src_sys_code varchar(75), + src_sys_description varchar(100), + ministry_code varchar(10), + ministry_description varchar(75), + business_area_code varchar(10), + business_area_description varchar(75), + application_id varchar(35), + permit_id varchar(30), + app_file_id varchar(35), + authorization_status_code varchar(30), + authorization_status_description varchar(100), + project_id varchar(30), + project_name varchar(125), + permit_status_code varchar(30), + permit_status_description varchar(100), + permit_status_date timestamp, + permit_issue_date timestamp, + permit_issue_expire_date timestamp, + app_received_date timestamp, + app_update_date timestamp, + app_decision_date timestamp, + app_issuance_date timestamp, + app_accepted_date timestamp, + app_rejected_date timestamp, + app_adjudication_date timestamp, + harvest_auth_status varchar(30), + harvest_area_sq_m integer, + permit_org_unit_code varchar(5), + district_description varchar(100), + district_code varchar(5), + rollup_district_description varchar(100), + rollup_district_code varchar(5), + region_name varchar(100), + region_code varchar(6), + area_code varchar(5), + area_description varchar(256), + application_age numeric, + map_feature_id bigint, + permit_type_code varchar, + permit_type_description varchar(120), + unqid varchar(70), + dbt_scd_id varchar(100), + dbt_updated_at timestamp, + dbt_valid_from timestamp, + dbt_valid_to timestamp +); + +GRANT REFERENCES, INSERT, TRIGGER, DELETE, TRUNCATE, SELECT, UPDATE ON pmt_dal.extract_permits TO pmt_write_role; GRANT SELECT ON pmt_dal.extract_permits TO pmt_read_role; \ No newline at end of file diff --git a/pmt/DDL/views.sql b/permitting/ddl/views.sql similarity index 100% rename from pmt/DDL/views.sql rename to permitting/ddl/views.sql