From cddbe45d9f197cd6e1edd4cb57cc8c4b57a60d35 Mon Sep 17 00:00:00 2001 From: Biyeun Buczyk Date: Mon, 3 Jun 2024 11:05:17 +0200 Subject: [PATCH 1/5] optimize case_property_names when domain has a ton of applications, and not all have relevant case type --- corehq/apps/export/models/new.py | 25 +++++++++++++------ .../apps/reports/standard/cases/case_data.py | 3 ++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/corehq/apps/export/models/new.py b/corehq/apps/export/models/new.py index f8be59b3dc5b..3721de1205b2 100644 --- a/corehq/apps/export/models/new.py +++ b/corehq/apps/export/models/new.py @@ -1767,6 +1767,7 @@ def generate_schema_from_builds( only_process_current_builds=False, task=None, for_new_export_instance=False, + is_identifier_case_type=False, ): """ Builds a schema from Application builds for a given identifier @@ -1783,6 +1784,7 @@ def generate_schema_from_builds( processed. :param task: A celery task to update the progress of the build :param for_new_export_instance: Flag to be set if generating schema for a new export instance + :param is_identifier_case_type: boolean, if True, some optimizations are applied specific to case type :returns: Returns a ExportDataSchema instance """ @@ -1805,7 +1807,8 @@ def generate_schema_from_builds( ) app_build_ids.extend(app_ids_for_domain) current_schema = cls._process_apps_for_export(domain, current_schema, identifier, app_build_ids, task, - for_new_export_instance=for_new_export_instance) + for_new_export_instance=for_new_export_instance, + is_identifier_case_type=is_identifier_case_type) inferred_schema = cls._get_inferred_schema(domain, app_id, identifier) if inferred_schema: @@ -1951,7 +1954,7 @@ def _save_export_schema(current_schema, original_id, original_rev): @classmethod def _process_apps_for_export(cls, domain, schema, identifier, app_build_ids, task, - for_new_export_instance=False): + for_new_export_instance=False, is_identifier_case_type=False): apps_processed = 0 for app_doc in iter_docs(Application.get_db(), app_build_ids, chunksize=10): doc_type = app_doc.get('doc_type', '') @@ -1970,6 +1973,9 @@ def _process_apps_for_export(cls, domain, schema, identifier, app_build_ids, tas ) continue + if is_identifier_case_type and not app.case_type_exists(identifier): + continue + try: schema = cls._process_app_build( schema, @@ -2311,14 +2317,15 @@ def _get_stock_items_from_question(question, app_id, app_version, repeats): @classmethod def _process_apps_for_export(cls, domain, schema, identifier, app_build_ids, task, - for_new_export_instance=False): + for_new_export_instance=False, is_identifier_case_type=False): return super(FormExportDataSchema, cls)._process_apps_for_export( domain, schema, identifier, app_build_ids, task, - for_new_export_instance=for_new_export_instance + for_new_export_instance=for_new_export_instance, + is_identifier_case_type=is_identifier_case_type, ) @@ -2512,7 +2519,7 @@ def _add_to_group_schema(group_schema, path_start, prop, app_id, app_version): @classmethod def _process_apps_for_export(cls, domain, schema, identifier, app_build_ids, task, - for_new_export_instance=False): + for_new_export_instance=False, is_identifier_case_type=False): if identifier == ALL_CASE_TYPE_EXPORT: return cls._process_apps_for_bulk_export(domain, schema, app_build_ids, task) else: @@ -2522,7 +2529,8 @@ def _process_apps_for_export(cls, domain, schema, identifier, app_build_ids, tas identifier, app_build_ids, task, - for_new_export_instance=for_new_export_instance + for_new_export_instance=for_new_export_instance, + is_identifier_case_type=is_identifier_case_type ) @classmethod @@ -2591,14 +2599,15 @@ def get_latest_export_schema(domain, include_metadata, identifier=None): return SMSExportDataSchema(domain=domain, include_metadata=include_metadata) def _process_apps_for_export(cls, domain, schema, identifier, app_build_ids, task, - for_new_export_instance=False): + for_new_export_instance=False, is_identifier_case_type=False): return super(SMSExportDataSchema, cls)._process_apps_for_export( domain, schema, identifier, app_build_ids, task, - for_new_export_instance=for_new_export_instance + for_new_export_instance=for_new_export_instance, + is_identifier_case_type=is_identifier_case_type ) diff --git a/corehq/apps/reports/standard/cases/case_data.py b/corehq/apps/reports/standard/cases/case_data.py index 163125e77708..697af14ebe65 100644 --- a/corehq/apps/reports/standard/cases/case_data.py +++ b/corehq/apps/reports/standard/cases/case_data.py @@ -450,7 +450,8 @@ def case_property_names(request, domain, case_id): # We need to look at the export schema in order to remove any case properties that # have been deleted from the app. When the data dictionary is fully public, we can use that # so that users may deprecate those properties manually - export_schema = CaseExportDataSchema.generate_schema_from_builds(domain, None, case.type) + export_schema = CaseExportDataSchema.generate_schema_from_builds(domain, None, case.type, + is_identifier_case_type=True) property_schema = export_schema.group_schemas[0] last_app_ids = get_latest_app_ids_and_versions(domain) all_property_names = { From a0a3e65e71b8b13b33fd66f2d9343084121089cd Mon Sep 17 00:00:00 2001 From: Biyeun Buczyk Date: Thu, 29 Aug 2024 13:58:49 +0200 Subject: [PATCH 2/5] remove extraneous overrides of _process_apps_for_export in FormDataExportSchema and SMSExportDataSchema, which only call super() --- corehq/apps/export/models/new.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/corehq/apps/export/models/new.py b/corehq/apps/export/models/new.py index 5dc5fee78262..dd94f03cd70c 100644 --- a/corehq/apps/export/models/new.py +++ b/corehq/apps/export/models/new.py @@ -2319,19 +2319,6 @@ def _get_stock_items_from_question(question, app_id, app_version, repeats): return items - @classmethod - def _process_apps_for_export(cls, domain, schema, identifier, app_build_ids, task, - for_new_export_instance=False, is_identifier_case_type=False): - return super(FormExportDataSchema, cls)._process_apps_for_export( - domain, - schema, - identifier, - app_build_ids, - task, - for_new_export_instance=for_new_export_instance, - is_identifier_case_type=is_identifier_case_type, - ) - class CaseExportDataSchema(ExportDataSchema): @@ -2631,18 +2618,6 @@ def schema_version(cls): def get_latest_export_schema(domain, include_metadata, identifier=None): return SMSExportDataSchema(domain=domain, include_metadata=include_metadata) - def _process_apps_for_export(cls, domain, schema, identifier, app_build_ids, task, - for_new_export_instance=False, is_identifier_case_type=False): - return super(SMSExportDataSchema, cls)._process_apps_for_export( - domain, - schema, - identifier, - app_build_ids, - task, - for_new_export_instance=for_new_export_instance, - is_identifier_case_type=is_identifier_case_type - ) - def _string_path_to_list(path): return path if path is None else path[1:].split('/') From d9dbbf73c3e00c53f03b04750107e28938fbe181 Mon Sep 17 00:00:00 2001 From: Graham Herceg Date: Thu, 29 Aug 2024 11:08:01 -0400 Subject: [PATCH 3/5] Wrap health check in try/except --- corehq/apps/hqadmin/service_checks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/corehq/apps/hqadmin/service_checks.py b/corehq/apps/hqadmin/service_checks.py index ec32f18cd751..bcdaeb42ed54 100644 --- a/corehq/apps/hqadmin/service_checks.py +++ b/corehq/apps/hqadmin/service_checks.py @@ -123,7 +123,11 @@ def check_kafka(): @change_log_level('urllib3.connectionpool', logging.WARNING) def check_elasticsearch(): - cluster_health = check_es_cluster_health() + try: + cluster_health = check_es_cluster_health() + except Exception: + return ServiceStatus(False, "Something went wrong checking cluster health") + if cluster_health == 'red': return ServiceStatus(False, "Cluster health at %s" % cluster_health) From ccfc4345244a47ead689a16f512a5fa5893f4dbd Mon Sep 17 00:00:00 2001 From: Graham Herceg Date: Thu, 29 Aug 2024 11:09:42 -0400 Subject: [PATCH 4/5] Fix lint errors --- corehq/apps/hqadmin/service_checks.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/corehq/apps/hqadmin/service_checks.py b/corehq/apps/hqadmin/service_checks.py index bcdaeb42ed54..be81a0b19e5f 100644 --- a/corehq/apps/hqadmin/service_checks.py +++ b/corehq/apps/hqadmin/service_checks.py @@ -78,9 +78,7 @@ def check_all_rabbitmq(): return ServiceStatus(True, 'RabbitMQ OK') else: - return ServiceStatus(False, '; '.join(['{}:{}'.format(rabbit[0], rabbit[1]) - for rabbit in unwell_rabbits]) - ) + return ServiceStatus(False, "; ".join(["{}:{}".format(rabbit[0], rabbit[1]) for rabbit in unwell_rabbits])) def check_rabbitmq(broker_url): @@ -180,8 +178,9 @@ def check_celery(): bad_queues.append( f"{queue} has been blocked for {blockage_duration} (max allowed is {threshold})" ) - elif (heartbeat_time_to_start is not None and - heartbeat_time_to_start > max(threshold, datetime.timedelta(minutes=5))): + elif heartbeat_time_to_start is not None and heartbeat_time_to_start > max( + threshold, datetime.timedelta(minutes=5) + ): bad_queues.append( f"{queue} is delayed for {heartbeat_time_to_start} (max allowed is {threshold})" ) @@ -198,7 +197,7 @@ def check_postgres(): for db in settings.DATABASES: db_conn = connections[db] try: - c = db_conn.cursor() + db_conn.cursor() c_status = 'OK' except OperationalError: c_status = 'FAIL' From 8072d469cf260038e2b4ae92694efa2ef903a732 Mon Sep 17 00:00:00 2001 From: Graham Herceg Date: Thu, 29 Aug 2024 13:21:06 -0400 Subject: [PATCH 5/5] Send error to sentry --- corehq/apps/hqadmin/service_checks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/corehq/apps/hqadmin/service_checks.py b/corehq/apps/hqadmin/service_checks.py index be81a0b19e5f..57883a20095b 100644 --- a/corehq/apps/hqadmin/service_checks.py +++ b/corehq/apps/hqadmin/service_checks.py @@ -19,6 +19,9 @@ import attr import gevent + +from dimagi.utils.logging import notify_exception + from corehq.apps.app_manager.models import Application from corehq.apps.change_feed.connection import ( get_kafka_client, @@ -124,6 +127,7 @@ def check_elasticsearch(): try: cluster_health = check_es_cluster_health() except Exception: + notify_exception(None, message="Error while checking elasticsearch cluster health") return ServiceStatus(False, "Something went wrong checking cluster health") if cluster_health == 'red':