Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

AMB-2307:500 list out of range #280

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

AMB-2307:500 list out of range #280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bd5e561
AMB-2307 : 500 "list index out of range" - when trying to create a re…
Nov 7, 2024
f422235
added code
Nov 7, 2024
8d1ca64
added code
Nov 7, 2024
668930f
added
Nov 7, 2024
0e47fab
added
Nov 7, 2024
1b5d797
Refactored the testcases
Nov 7, 2024
47d3a3e
Updated the python
Nov 11, 2024
005794a
Merge branch 'master' into AMB-2307--500-list-out-of-range
ASubaran Nov 12, 2024
f372ee4
Merge branch 'master' into AMB-2307--500-list-out-of-range
ASubaran Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions .github/workflows/sonarcloud.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,18 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0


- name: Run unittest with coverage-delta
run: |
pip install poetry mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
poetry run coverage run --source=delta_backend -m unittest discover -s delta_backend
poetry run coverage xml -o sonarcloud-coverage-delta.xml

- name: Run unittest with coverage
run: |
pip install poetry moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90
poetry run coverage run --source=backend -m unittest discover -s backend
poetry run coverage xml -o sonarcloud-coverage.xml
pip install poetry moto==4.2.11 coverage botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90
poetry run coverage run --source=backend -m unittest discover -s backend
poetry run coverage xml -o sonarcloud-coverage.xml

- name: SonarCloud Scan
uses: SonarSource/sonarcloud-github-action@master
Expand Down
Binary file added delta_backend/.coverage
View file
Open in desktop
Binary file not shown.
2 changes: 1 addition & 1 deletion delta_backend/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM public.ecr.aws/lambda/python:3.9 as base
FROM public.ecr.aws/lambda/python:3.10 as base

RUN pip install "poetry~=1.5.0"

Expand Down
28 changes: 14 additions & 14 deletions delta_backend/poetry.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion delta_backend/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ authors = ["Your Name <[email protected]>"]
readme = "README.md"

[tool.poetry.dependencies]
python = "~3.9"
python = "~3.10"
boto3 = "~1.26.90"
mypy-boto3-dynamodb = "^1.26.164"

Expand Down
106 changes: 69 additions & 37 deletions delta_backend/src/convert_flat_json.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,46 +10,78 @@ def get_person_details(resource_json, resource_type):
return given_name, family_name
return None, None


professional_given_name, professional_family_name = get_person_details(resource_json, "practitioner")
person_given_name, person_family_name = get_person_details(resource_json, "patient")
professional_given_name, professional_family_name = get_person_details(resource_json, "practitioner")

flat_dict = {
"NHS_NUMBER": resource_json.get("contained", [None, {}])[1].get("identifier", [{}])[0].get("value", None),
"PERSON_FORENAME": person_given_name,
"PERSON_SURNAME": person_family_name,
"PERSON_DOB": resource_json.get("contained", [None, {}])[1].get("birthDate", None),
"PERSON_GENDER_CODE": gender_map.get(resource_json.get("contained", [None, {}])[1].get("gender", 0), None),
"PERSON_POSTCODE": resource_json.get("contained", [None, {}])[1].get("address", [{}])[0].get("postalCode", None),
"DATE_AND_TIME": resource_json.get("occurrenceDateTime", None),
"SITE_CODE": resource_json.get("performer", [None, {}])[1].get("actor", {}).get("identifier", {}).get("value", None),
"SITE_CODE_TYPE_URI": resource_json.get("performer", [None, {}])[1].get("actor", {}).get("identifier", {}).get("system", None),
"UNIQUE_ID": resource_json.get("identifier", [{}])[0].get("value", None),
"UNIQUE_ID_URI": resource_json.get("identifier", [{}])[0].get("system", None),
"ACTION_FLAG": operation, # Based on the operation.
"PERFORMING_PROFESSIONAL_FORENAME": professional_given_name,
"PERFORMING_PROFESSIONAL_SURNAME": professional_family_name,
"RECORDED_DATE": resource_json.get("recorded", None),
"PRIMARY_SOURCE": str(resource_json.get("primarySource", None)).lower(),
"VACCINATION_PROCEDURE_CODE": resource_json.get("extension", [{}])[0].get("valueCodeableConcept", {}).get("coding", [{}])[0].get("code", None),
"VACCINATION_PROCEDURE_TERM": resource_json.get("extension", [{}])[0].get("valueCodeableConcept", {}).get("coding", [{}])[0].get("display", None),
"DOSE_SEQUENCE": resource_json.get("protocolApplied", [{}])[0].get("doseNumberPositiveInt", None),
"VACCINE_PRODUCT_CODE": resource_json.get("vaccineCode", {}).get("coding", [{}])[0].get("code", None),
"VACCINE_PRODUCT_TERM": resource_json.get("vaccineCode", {}).get("coding", [{}])[0].get("display", None),
"VACCINE_MANUFACTURER": resource_json.get("manufacturer", {}).get("display", None),
"BATCH_NUMBER": resource_json.get("lotNumber", None),
"EXPIRY_DATE": resource_json.get("expirationDate", None),
"SITE_OF_VACCINATION_CODE": resource_json.get("site", {}).get("coding", [{}])[0].get("code", None),
"SITE_OF_VACCINATION_TERM": resource_json.get("site", {}).get("coding", [{}])[0].get("display", None),
"ROUTE_OF_VACCINATION_CODE": resource_json.get("route", {}).get("coding", [{}])[0].get("code", None),
"ROUTE_OF_VACCINATION_TERM": resource_json.get("route", {}).get("coding", [{}])[0].get("display", None),
"DOSE_AMOUNT": resource_json.get("doseQuantity", {}).get("value", None),
"DOSE_UNIT_CODE": resource_json.get("doseQuantity", {}).get("code", None),
"DOSE_UNIT_TERM": resource_json.get("doseQuantity", {}).get("unit", None),
"INDICATION_CODE": resource_json.get("reasonCode", [{}])[0].get("coding", [{}])[0].get("code", None),
"LOCATION_CODE": resource_json.get("location", {}).get("identifier", {}).get("value", None),
"LOCATION_CODE_TYPE_URI": resource_json.get("location", {}).get("identifier", {}).get("system", None)
}
"NHS_NUMBER": next(
(identifier.get("value", None)
for contained_resource in resource_json.get("contained", [])
if contained_resource.get("resourceType") == "Patient"
for identifier in contained_resource.get("identifier", [])
if identifier.get("system") == "https://fhir.nhs.uk/Id/nhs-number"),
None
),
"PERSON_FORENAME": person_given_name,
"PERSON_SURNAME": person_family_name,
"PERSON_DOB": next(
(contained_resource.get("birthDate", None)
for contained_resource in resource_json.get("contained", [])
if contained_resource.get("resourceType") == "Patient"),
None
),
"PERSON_GENDER_CODE": gender_map.get(next(
(contained_resource.get("gender", None)
for contained_resource in resource_json.get("contained", [])
if contained_resource.get("resourceType") == "Patient"),
None
)),
"PERSON_POSTCODE": next(
(address.get("postalCode", None)
for contained_resource in resource_json.get("contained", [])
if contained_resource.get("resourceType") == "Patient"
for address in contained_resource.get("address", [])),
None
),
"DATE_AND_TIME": resource_json.get("occurrenceDateTime", None),
"SITE_CODE": next(
(performer.get("actor", {}).get("identifier", {}).get("value", None)
for performer in resource_json.get("performer", [])
if performer.get("actor", {}).get("type") == "Organization"),
None
),
"SITE_CODE_TYPE_URI": next(
(performer.get("actor", {}).get("identifier", {}).get("system", None)
for performer in resource_json.get("performer", [])
if performer.get("actor", {}).get("type") == "Organization"),
None
),
"UNIQUE_ID": resource_json.get("identifier", [{}])[0].get("value", None),
"UNIQUE_ID_URI": resource_json.get("identifier", [{}])[0].get("system", None),
"ACTION_FLAG": operation,
"PERFORMING_PROFESSIONAL_FORENAME": professional_given_name,
"PERFORMING_PROFESSIONAL_SURNAME": professional_family_name,
"RECORDED_DATE": resource_json.get("recorded", None),
"PRIMARY_SOURCE": str(resource_json.get("primarySource", None)).lower(),
"VACCINATION_PROCEDURE_CODE": resource_json.get("extension", [{}])[0].get("valueCodeableConcept", {}).get("coding", [{}])[0].get("code", None),
"VACCINATION_PROCEDURE_TERM": resource_json.get("extension", [{}])[0].get("valueCodeableConcept", {}).get("coding", [{}])[0].get("display", None),
"DOSE_SEQUENCE": resource_json.get("protocolApplied", [{}])[0].get("doseNumberPositiveInt", None),
"VACCINE_PRODUCT_CODE": resource_json.get("vaccineCode", {}).get("coding", [{}])[0].get("code", None),
"VACCINE_PRODUCT_TERM": resource_json.get("vaccineCode", {}).get("coding", [{}])[0].get("display", None),
"VACCINE_MANUFACTURER": resource_json.get("manufacturer", {}).get("display", None),
"BATCH_NUMBER": resource_json.get("lotNumber", None),
"EXPIRY_DATE": resource_json.get("expirationDate", None),
"SITE_OF_VACCINATION_CODE": resource_json.get("site", {}).get("coding", [{}])[0].get("code", None),
"SITE_OF_VACCINATION_TERM": resource_json.get("site", {}).get("coding", [{}])[0].get("display", None),
"ROUTE_OF_VACCINATION_CODE": resource_json.get("route", {}).get("coding", [{}])[0].get("code", None),
"ROUTE_OF_VACCINATION_TERM": resource_json.get("route", {}).get("coding", [{}])[0].get("display", None),
"DOSE_AMOUNT": resource_json.get("doseQuantity", {}).get("value", None),
"DOSE_UNIT_CODE": resource_json.get("doseQuantity", {}).get("code", None),
"DOSE_UNIT_TERM": resource_json.get("doseQuantity", {}).get("unit", None),
"INDICATION_CODE": resource_json.get("reasonCode", [{}])[0].get("coding", [{}])[0].get("code", None),
"LOCATION_CODE": resource_json.get("location", {}).get("identifier", {}).get("value", None),
"LOCATION_CODE_TYPE_URI": resource_json.get("location", {}).get("identifier", {}).get("system", None)
}
if isinstance(flat_dict["PERSON_FORENAME"], list):
flat_dict["PERSON_FORENAME"] = ' '.join(flat_dict["PERSON_FORENAME"])

Expand Down
17 changes: 9 additions & 8 deletions delta_backend/src/delta.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ def send_message(record):
try:
# Send the record to the queue
sqs_client.send_message(QueueUrl=failure_queue_url, MessageBody=json.dumps(message_body))
print("Record saved successfully to the DLQ")
logger.info("Record saved successfully to the DLQ")
except ClientError as e:
print(f"Error sending record to DLQ: {e}")
logger.info(f"Error sending record to DLQ: {e}")


def get_vaccine_type(patientsk) -> str:
Expand Down Expand Up @@ -62,7 +62,6 @@ def handler(event, context):
operation = str()
if record["eventName"] != "REMOVE":
new_image = record["dynamodb"]["NewImage"]
print(f"new_image:{new_image}")
imms_id = new_image["PK"]["S"].split("#")[1]
vaccine_type = get_vaccine_type(new_image["PatientSK"]["S"])
supplier_system = new_image["SupplierSystem"]["S"]
Expand Down Expand Up @@ -92,11 +91,11 @@ def handler(event, context):
firehose_log["event"] = log_data
firehose_logger.send_log(firehose_log)
logger.info(f"Record from DPS skipped for {imms_id}")
continue
return {"statusCode": 200, "body": f"Record from DPS skipped for {imms_id}"}
else:
operation = "REMOVE"
new_image = record["dynamodb"]["Keys"]
print(f"Record to delta:{new_image}")
logger.info(f"Record to delta:{new_image}")
imms_id = new_image["PK"]["S"].split("#")[1]
response = delta_table.put_item(
Item={
Expand All @@ -121,16 +120,18 @@ def handler(event, context):
log_data["operation_outcome"] = operation_outcome
firehose_log["event"] = log_data
firehose_logger.send_log(firehose_log)
logger.info(log)
return {"statusCode": 200, "body": "Records processed successfully"}
else:
log = f"Record NOT created for {imms_id}"
operation_outcome["statusCode"] = "500"
operation_outcome["statusDesc"] = "Exception"
log_data["operation_outcome"] = operation_outcome
firehose_log["event"] = log_data
firehose_logger.send_log(firehose_log)
logger.info(log)
return {"statusCode": 200, "body": "Records processed successfully"}

logger.info(log)
return {"statusCode": 500, "body": "Records not processed successfully"}
except Exception as e:
operation_outcome["statusCode"] = "500"
operation_outcome["statusDesc"] = "Exception"
Expand Down
Loading