mark metric as failure if no output (#1747)

* mark metric as failure if no output * return NaN
Azure · Nov 14, 2023 · 9e4380d · 9e4380d
1 parent 1604d15
commit 9e4380d
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 3 deletions.
diff --git a/...odel_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml b/...odel_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml
@@ -4,7 +4,7 @@ type: spark
 name: gsq_annotation_compute_metrics
 display_name: Annotation - Compute Metrics
 description: Compute annotation metrics given a deployment's model data input.
-version: 0.4.2
+version: 0.4.3
 is_deterministic: True
 inputs:
   annotation_histogram: 

diff --git a/...g/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml b/...g/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml
@@ -137,7 +137,7 @@ jobs:
       type: aml_token
   compute_metrics:
     type: spark
-    component: azureml://registries/azureml/components/gsq_annotation_compute_metrics/versions/0.4.2
+    component: azureml://registries/azureml/components/gsq_annotation_compute_metrics/versions/0.4.3
     inputs:
       annotation_histogram:
         type: mltable

diff --git a/...del_monitoring/components/src/generation_safety_quality/annotation_compute_metrics/run.py b/...del_monitoring/components/src/generation_safety_quality/annotation_compute_metrics/run.py
@@ -63,8 +63,10 @@ def _calculate_passrate(df, metric_name):
         .head()[0]
     )
     total = df_with_buckets.select(sum(METRIC_VALUE_COLUMN)).head()[0]
+    # if there are no metric value, we should mark as fail since there was probably a
+    # parsing error or request error that resulted in no metrics
     if total == 0:
-        return "1"
+        return "NaN"
     passrate = passing / total
     return str(passrate)