Merge pull request #86 from answerdigital/feature/OXOMOP-152

Fix bug where some mapped ICD10/ICDO3/OPCS4 codes were not standard o…
answerdigital · Jan 9, 2025 · 627872b · 627872b
2 parents b25f07f + 362ee3e
commit 627872b
Show file tree

Hide file tree

Showing 34 changed files with 492 additions and 168 deletions.
diff --git a/DataQualityDashboard/known_errors.txt b/DataQualityDashboard/known_errors.txt
@@ -1,5 +1,4 @@
-table_measureconditioneracompleteness_condition_era
-field_isrequired_person_gender_concept_id
-field_isstandardvalidconcept_death_cause_concept_id
+table_measurepersoncompleteness_observation
+field_isstandardvalidconcept_person_gender_concept_id
 field_plausiblevaluelow_cdm_source_source_release_date
 field_withinvisitdates_visit_detail_visit_detail_start_date
diff --git a/Database/Migrations/sql/VersionedMigrations/V60__code_map_primary_key.sql b/Database/Migrations/sql/VersionedMigrations/V60__code_map_primary_key.sql
@@ -0,0 +1,6 @@
+alter table omop_staging.concept_code_map
+	add constraint PK_omop_staging_concept_code_map_source_concept_id_target_concept_id
+		primary key (
+			source_concept_id,
+			target_concept_id
+		);
diff --git a/OmopTransformer/Icd10Resolver.cs b/OmopTransformer/Icd10Resolver.cs
@@ -27,9 +27,16 @@ private Dictionary<string, int> GetIcd10Codes()
 
         connection.Open();
 
+        string query = "select " +
+                       "	ccm.target_concept_id as concept_id, " +
+                       "	replace(ccm.source_concept_code, '.', '') as Code  " +
+                       "from omop_staging.concept_code_map ccm " +
+                       "	inner join cdm.concept c " +
+                       "		on ccm.source_concept_id = c.concept_id " +
+                       "where c.vocabulary_id = 'ICD10' ";
         return
             connection
-                .Query<Row>(sql: "select concept_id, replace(concept_code, '.', '') as Code from cdm.concept where vocabulary_id = 'ICD10'")
+                .Query<Row>(sql: query)
                 .ToDictionary(
                     row => row.Code!, 
                     row => row.concept_id);

diff --git a/OmopTransformer/Icdo3Resolver.cs b/OmopTransformer/Icdo3Resolver.cs
@@ -27,9 +27,18 @@ private Dictionary<string, int> GetIcdo3Codes()
 
         connection.Open();
 
+        string query =
+            "select " +
+            "	ccm.target_concept_id as concept_id, " +
+            "	ccm.source_concept_code as concept_code " +
+            "from omop_staging.concept_code_map ccm " +
+            "	inner join cdm.concept c " +
+            "		on ccm.source_concept_id = c.concept_id " +
+            "where c.vocabulary_id = 'ICDO3' ";
+
         return
             connection
-                .Query<Row>(sql: "select concept_id, concept_code as Code from cdm.concept where vocabulary_id = 'ICDO3'")
+                .Query<Row>(query)
                 .ToDictionary(
                     row => row.Code!, 
                     row => row.concept_id);

diff --git a/OmopTransformer/OPCS4Resolver.cs b/OmopTransformer/OPCS4Resolver.cs
@@ -27,9 +27,18 @@ private Dictionary<string, int> GetOpcs4Codes()
 
         connection.Open();
 
+        string query =
+            "select " +
+            "	ccm.target_concept_id as concept_id, " +
+            "	replace(ccm.source_concept_code, '.', '') as Code  " +
+            "from omop_staging.concept_code_map ccm " +
+            "	inner join cdm.concept c " +
+            "		on ccm.source_concept_id = c.concept_id " +
+            "where c.vocabulary_id = 'OPCS4' ";
+
         return
             connection
-                .Query<Row>(sql: "select replace (concept_code, '.', '') as Code, concept_id from cdm.concept where vocabulary_id = 'OPCS4'")
+                .Query<Row>(sql: query)
                 .ToDictionary(
                     row => row.Code!, 
                     row => row.concept_id);

diff --git a/docs/transformation-documentation/CdsAnaestheticDuringLabourDelivery.json b/docs/transformation-documentation/CdsAnaestheticDuringLabourDelivery.json
@@ -109,7 +109,7 @@
     },
     {
       "name": "observation_type_concept_id",
-      "operation_description": "Constant value set to `38000281`. Observation recorded from EHR with text result",
+      "operation_description": "Constant value set to `32818`. EHR administration record",
       "dataSource": null,
       "query": null,
       "lookup_table_markdown": null

diff --git a/docs/transformation-documentation/CdsAnaestheticGivenPostLabourDelivery.json b/docs/transformation-documentation/CdsAnaestheticGivenPostLabourDelivery.json
@@ -101,7 +101,7 @@
     },
     {
       "name": "observation_type_concept_id",
-      "operation_description": "Constant value set to `38000281`. Observation recorded from EHR with text result",
+      "operation_description": "Constant value set to `32818`. EHR administration record",
       "dataSource": null,
       "query": null,
       "lookup_table_markdown": null

diff --git a/docs/transformation-documentation/CdsCarerSupportIndicator.json b/docs/transformation-documentation/CdsCarerSupportIndicator.json
@@ -101,7 +101,7 @@
     },
     {
       "name": "observation_type_concept_id",
-      "operation_description": "Constant value set to `38000281`. Observation recorded from EHR with text result",
+      "operation_description": "Constant value set to `32818`. EHR administration record",
       "dataSource": null,
       "query": null,
       "lookup_table_markdown": null

diff --git a/docs/transformation-documentation/CdsSourceOfReferralForOutpatients.json b/docs/transformation-documentation/CdsSourceOfReferralForOutpatients.json
@@ -109,7 +109,7 @@
     },
     {
       "name": "observation_type_concept_id",
-      "operation_description": "Constant value set to `38000281`. Observation recorded from EHR with text result",
+      "operation_description": "Constant value set to `32818`. EHR administration record",
       "dataSource": null,
       "query": null,
       "lookup_table_markdown": null

diff --git a/docs/transformation-documentation/CdsTotalPreviousPregnancies.json b/docs/transformation-documentation/CdsTotalPreviousPregnancies.json
@@ -101,7 +101,7 @@
     },
     {
       "name": "observation_type_concept_id",
-      "operation_description": "Constant value set to `38000281`. Observation recorded from EHR with text result",
+      "operation_description": "Constant value set to `32818`. EHR administration record",
       "dataSource": null,
       "query": null,
       "lookup_table_markdown": null

diff --git a/docs/transformation-documentation/Death_NhsNumber.md b/docs/transformation-documentation/Death_NhsNumber.md
@@ -31,33 +31,33 @@ has_toc: false
 * `nhs_number` Patient NHS Number [NHS NUMBER](https://www.datadictionary.nhs.uk/data_elements/nhs_number.html)
 
 ```sql
-		;with primarydiagnosis as (
-		select *
-		from omop_staging.sus_ICDDiagnosis
-		where IsPrimaryDiagnosis = 1
-		)
+;with primarydiagnosis as (
+	select *
+	from omop_staging.sus_ICDDiagnosis
+	where IsPrimaryDiagnosis = 1
+)
 
-		select
-		apc.NHSNumber as nhs_number,
-		max(apc.DischargeDateFromHospitalProviderSpell) as death_date,
-		max(apc.DischargeTimeHospitalProviderSpell) as death_time,
-		max(d.DiagnosisICD) as DiagnosisICD
-		from
-		omop_staging.sus_APC apc
-		left join primarydiagnosis d
+select
+	apc.NHSNumber as nhs_number,
+	max(apc.DischargeDateFromHospitalProviderSpell) as death_date,
+	max(apc.DischargeTimeHospitalProviderSpell) as death_time,
+	max(d.DiagnosisICD) as DiagnosisICD
+from omop_staging.sus_APC apc
+	left join primarydiagnosis d
 		on apc.MessageId = d.MessageId
-		where
-		apc.NHSNumber is not null and
-		apc.DischargeDateFromHospitalProviderSpell is not null and
+where
+	apc.NHSNumber is not null and
+	apc.DischargeDateFromHospitalProviderSpell is not null and
+	(
+		apc.DischargeMethodHospitalProviderSpell = '4' -- "Patient died"
+		or
 		(
-		apc.DischargeMethodHospitalProviderSpell = '4'
-		or (
-		apc.DischargeDestinationHospitalProviderSpell = '79'
-		and
-		apc.DischargeMethodHospitalProviderSpell != '5'
-		)
+			apc.DischargeDestinationHospitalProviderSpell = '79' -- Not applicable - PATIENT died or stillbirth
+			and
+			apc.DischargeMethodHospitalProviderSpell != '5' -- not stillbirth
 		)
-		group by apc.NHSNumber
+)
+group by apc.NHSNumber
 
 ```
 

diff --git a/docs/transformation-documentation/Death_cause_concept_id.md b/docs/transformation-documentation/Death_cause_concept_id.md
@@ -13,33 +13,33 @@ Resolve ICD10 codes to OMOP concepts. If code cannot be mapped, map using the pa
 * `DiagnosisICD` Primary Diagnosis [PRIMARY DIAGNOSIS]()
 
 ```sql
-		;with primarydiagnosis as (
-		select *
-		from omop_staging.sus_ICDDiagnosis
-		where IsPrimaryDiagnosis = 1
-		)
+;with primarydiagnosis as (
+	select *
+	from omop_staging.sus_ICDDiagnosis
+	where IsPrimaryDiagnosis = 1
+)
 
-		select
-		apc.NHSNumber as nhs_number,
-		max(apc.DischargeDateFromHospitalProviderSpell) as death_date,
-		max(apc.DischargeTimeHospitalProviderSpell) as death_time,
-		max(d.DiagnosisICD) as DiagnosisICD
-		from
-		omop_staging.sus_APC apc
-		left join primarydiagnosis d
+select
+	apc.NHSNumber as nhs_number,
+	max(apc.DischargeDateFromHospitalProviderSpell) as death_date,
+	max(apc.DischargeTimeHospitalProviderSpell) as death_time,
+	max(d.DiagnosisICD) as DiagnosisICD
+from omop_staging.sus_APC apc
+	left join primarydiagnosis d
 		on apc.MessageId = d.MessageId
-		where
-		apc.NHSNumber is not null and
-		apc.DischargeDateFromHospitalProviderSpell is not null and
+where
+	apc.NHSNumber is not null and
+	apc.DischargeDateFromHospitalProviderSpell is not null and
+	(
+		apc.DischargeMethodHospitalProviderSpell = '4' -- "Patient died"
+		or
 		(
-		apc.DischargeMethodHospitalProviderSpell = '4'
-		or (
-		apc.DischargeDestinationHospitalProviderSpell = '79'
-		and
-		apc.DischargeMethodHospitalProviderSpell != '5'
-		)
+			apc.DischargeDestinationHospitalProviderSpell = '79' -- Not applicable - PATIENT died or stillbirth
+			and
+			apc.DischargeMethodHospitalProviderSpell != '5' -- not stillbirth
 		)
-		group by apc.NHSNumber
+)
+group by apc.NHSNumber
 
 ```
 

diff --git a/docs/transformation-documentation/Death_cause_source_value.md b/docs/transformation-documentation/Death_cause_source_value.md
@@ -12,33 +12,33 @@ has_toc: false
 * `DiagnosisICD` Primary Diagnosis [PRIMARY DIAGNOSIS]()
 
 ```sql
-		;with primarydiagnosis as (
-		select *
-		from omop_staging.sus_ICDDiagnosis
-		where IsPrimaryDiagnosis = 1
-		)
+;with primarydiagnosis as (
+	select *
+	from omop_staging.sus_ICDDiagnosis
+	where IsPrimaryDiagnosis = 1
+)
 
-		select
-		apc.NHSNumber as nhs_number,
-		max(apc.DischargeDateFromHospitalProviderSpell) as death_date,
-		max(apc.DischargeTimeHospitalProviderSpell) as death_time,
-		max(d.DiagnosisICD) as DiagnosisICD
-		from
-		omop_staging.sus_APC apc
-		left join primarydiagnosis d
+select
+	apc.NHSNumber as nhs_number,
+	max(apc.DischargeDateFromHospitalProviderSpell) as death_date,
+	max(apc.DischargeTimeHospitalProviderSpell) as death_time,
+	max(d.DiagnosisICD) as DiagnosisICD
+from omop_staging.sus_APC apc
+	left join primarydiagnosis d
 		on apc.MessageId = d.MessageId
-		where
-		apc.NHSNumber is not null and
-		apc.DischargeDateFromHospitalProviderSpell is not null and
+where
+	apc.NHSNumber is not null and
+	apc.DischargeDateFromHospitalProviderSpell is not null and
+	(
+		apc.DischargeMethodHospitalProviderSpell = '4' -- "Patient died"
+		or
 		(
-		apc.DischargeMethodHospitalProviderSpell = '4'
-		or (
-		apc.DischargeDestinationHospitalProviderSpell = '79'
-		and
-		apc.DischargeMethodHospitalProviderSpell != '5'
-		)
+			apc.DischargeDestinationHospitalProviderSpell = '79' -- Not applicable - PATIENT died or stillbirth
+			and
+			apc.DischargeMethodHospitalProviderSpell != '5' -- not stillbirth
 		)
-		group by apc.NHSNumber
+)
+group by apc.NHSNumber
 
 ```
 

diff --git a/docs/transformation-documentation/Death_death_date.md b/docs/transformation-documentation/Death_death_date.md
@@ -33,33 +33,33 @@ Converts text to dates.
 * `death_date` Discharge date of the patient's spell. [DISCHARGE DATE (HOSPITAL PROVIDER SPELL)](https://www.datadictionary.nhs.uk/data_elements/discharge_date__hospital_provider_spell_.html)
 
 ```sql
-		;with primarydiagnosis as (
-		select *
-		from omop_staging.sus_ICDDiagnosis
-		where IsPrimaryDiagnosis = 1
-		)
+;with primarydiagnosis as (
+	select *
+	from omop_staging.sus_ICDDiagnosis
+	where IsPrimaryDiagnosis = 1
+)
 
-		select
-		apc.NHSNumber as nhs_number,
-		max(apc.DischargeDateFromHospitalProviderSpell) as death_date,
-		max(apc.DischargeTimeHospitalProviderSpell) as death_time,
-		max(d.DiagnosisICD) as DiagnosisICD
-		from
-		omop_staging.sus_APC apc
-		left join primarydiagnosis d
+select
+	apc.NHSNumber as nhs_number,
+	max(apc.DischargeDateFromHospitalProviderSpell) as death_date,
+	max(apc.DischargeTimeHospitalProviderSpell) as death_time,
+	max(d.DiagnosisICD) as DiagnosisICD
+from omop_staging.sus_APC apc
+	left join primarydiagnosis d
 		on apc.MessageId = d.MessageId
-		where
-		apc.NHSNumber is not null and
-		apc.DischargeDateFromHospitalProviderSpell is not null and
+where
+	apc.NHSNumber is not null and
+	apc.DischargeDateFromHospitalProviderSpell is not null and
+	(
+		apc.DischargeMethodHospitalProviderSpell = '4' -- "Patient died"
+		or
 		(
-		apc.DischargeMethodHospitalProviderSpell = '4'
-		or (
-		apc.DischargeDestinationHospitalProviderSpell = '79'
-		and
-		apc.DischargeMethodHospitalProviderSpell != '5'
-		)
+			apc.DischargeDestinationHospitalProviderSpell = '79' -- Not applicable - PATIENT died or stillbirth
+			and
+			apc.DischargeMethodHospitalProviderSpell != '5' -- not stillbirth
 		)
-		group by apc.NHSNumber
+)
+group by apc.NHSNumber
 
 ```