Validation for vision results (#3377)

* Validation for vision results. * Changelog and setup updates.
Azure · Sep 16, 2024 · 99c48b3 · 99c48b3
1 parent fefc90c
commit 99c48b3
Show file tree

Hide file tree

Showing 10 changed files with 71 additions and 1 deletion.
diff --git a/scripts/azureml-assets/CHANGELOG.md b/scripts/azureml-assets/CHANGELOG.md
@@ -3,6 +3,10 @@
 
 ### 🐛 Bugs Fixed
 
+## 1.16.58 (2024-09-13)
+### 🐛 Bugs Fixed
+- [#3377](https://github.com/Azure/azureml-assets/pull/3377) Validation for vision results
+
 ## 1.16.57 (2024-08-12)
 ### 🐛 Bugs Fixed
 - [#3253](https://github.com/Azure/azureml-assets/pull/3253) Also accept Path types for populating tags from files

diff --git a/scripts/azureml-assets/azureml/assets/config/evaluationresult/tag_values_shared.yaml b/scripts/azureml-assets/azureml/assets/config/evaluationresult/tag_values_shared.yaml
@@ -5,3 +5,4 @@ evaluation_type:
   values:
     - text_generation
     - text_embeddings
+    - vision
diff --git a/scripts/azureml-assets/azureml/assets/config/evaluationresult/tag_values_vision.yaml b/scripts/azureml-assets/azureml/assets/config/evaluationresult/tag_values_vision.yaml
@@ -0,0 +1,7 @@
+# Valid tag values that can be applied to evaluation results whose evaluation_type is `vision`.
+task:
+  required: True
+  allow_multiple: False
+  values:
+  - image-generation
+  - image-vqa
diff --git a/scripts/azureml-assets/azureml/assets/validate_assets.py b/scripts/azureml-assets/azureml/assets/validate_assets.py
@@ -1077,6 +1077,8 @@ def validate_assets(input_dirs: List[Path],
                     error_count += validate_tags(asset_config, 'evaluationresult/tag_values_text_generation.yaml')
                 elif evaluation_type == 'text_embeddings':
                     error_count += validate_tags(asset_config, 'evaluationresult/tag_values_text_embeddings.yaml')
+                elif evaluation_type == 'vision':
+                    error_count += validate_tags(asset_config, 'evaluationresult/tag_values_vision.yaml')
                 else:
                     _log_error(
                         asset_config.file_name_with_path,

diff --git a/scripts/azureml-assets/setup.py b/scripts/azureml-assets/setup.py
@@ -7,7 +7,7 @@
 
 setup(
    name="azureml-assets",
-   version="1.16.57",
+   version="1.16.58",
    description="Utilities for publishing assets to Azure Machine Learning system registries.",
    author="Microsoft Corp",
    packages=find_packages(),

diff --git a/test/resources/validate/evaluationresult/vision_correct/asset.yaml b/test/resources/validate/evaluationresult/vision_correct/asset.yaml
@@ -0,0 +1,3 @@
+type: evaluationresult
+spec: spec.yaml
+categories: ["EvaluationResult"]
diff --git a/test/resources/validate/evaluationresult/vision_correct/spec.yaml b/test/resources/validate/evaluationresult/vision_correct/spec.yaml
@@ -0,0 +1,24 @@
+type: evaluationresult
+name: vision_correct
+version: 1
+display_name: vision_correct
+description: vision_correct
+dataset_name: dataset1
+dataset_family: family1
+
+model_name: model1
+model_version: "1"
+model_asset_id: azureml://registries/azureml/models/model1/versions/1
+relationships:
+    - relationshipType: Source
+      assetId: azureml://registries/azureml/models/model1/versions/1
+
+tags:
+    evaluation_type: vision
+    task: image-generation
+    primary_metric: fid
+
+metrics:
+    fid: 300.1
+
+properties: {}
diff --git a/test/resources/validate/evaluationresult/vision_incorrect/asset.yaml b/test/resources/validate/evaluationresult/vision_incorrect/asset.yaml
@@ -0,0 +1,3 @@
+type: evaluationresult
+spec: spec.yaml
+categories: ["EvaluationResult"]
diff --git a/test/resources/validate/evaluationresult/vision_incorrect/spec.yaml b/test/resources/validate/evaluationresult/vision_incorrect/spec.yaml
@@ -0,0 +1,24 @@
+type: evaluationresult
+name: vision_incorrect
+version: 1
+display_name: vision_incorrect
+description: vision_incorrect
+dataset_name: dataset1
+dataset_family: family1
+
+model_name: model1
+model_version: "1"
+model_asset_id: azureml://registries/azureml/models/model1/versions/1
+relationships:
+    - relationshipType: Source
+      assetId: azureml://registries/azureml/models/model1/versions/1
+
+tags:
+    evaluation_type: vision
+    task: image-understanding
+    primary_metric: visual_intelligence
+
+metrics:
+    ndcg_at_10: 0.001
+
+properties: {}
diff --git a/test/test_validate_assets.py b/test/test_validate_assets.py
@@ -48,6 +48,8 @@
         ("evaluationresult/text_embeddings_incorrect", False, True, None, False),
         ("evaluationresult/text_generation_correct", False, True, None, True),
         ("evaluationresult/text_generation_incorrect", False, True, None, False),
+        ("evaluationresult/vision_correct", False, True, None, True),
+        ("evaluationresult/vision_incorrect", False, True, None, False),
     ]
 )
 def test_validate_assets(test_subdir: str, check_images: bool, check_names: bool,
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,3 +5,4 @@ evaluation_type: @@
       values:
         - text_generation
         - text_embeddings
+        - vision