Repackage benchmark src code (#1583)

* repackage benchmark src code

* shorten folder name

* fix imports

* fix imports

* fix failing tests

* prompt crafter changes

assets/aml-benchmark/components/batch-benchmark-score/spec.yaml

@@ -67,7 +67,7 @@ task:
   code: ../src
   environment: azureml://registries/azureml/environments/model-evaluation/versions/13
   program_arguments: --append_row_safe_output True --debug_mode ${{inputs.debug_mode}} --online_endpoint_url ${{inputs.online_endpoint_url}} $[[--additional_properties ${{inputs.additional_properties}}]] $[[--additional_headers ${{inputs.additional_headers}}]] $[[--user_agent_segment ${{inputs.user_agent_segment}}]] --metrics_out_directory ${{outputs.metrics_out_directory}} --tally_failed_requests False --tally_exclusions none --run_type parallel --segment_large_requests disabled --segment_max_token_size 600  --ensure_ascii ${{inputs.ensure_ascii}} --output_behavior append_row --initial_worker_count ${{inputs.initial_worker_count}} --max_worker_count ${{inputs.max_worker_count}} $[[--max_retry_time_interval ${{inputs.max_retry_time_interval}}]] --save_mini_batch_results enabled --mini_batch_results_out_directory ${{outputs.mini_batch_results_out_directory}} --connections_name ${{inputs.connections_name}}  $[[--deployment_name ${{inputs.deployment_name}}]]
-  entry_script: batch_benchmark_score.batch_score.main
+  entry_script: aml_benchmark.batch_benchmark_score.batch_score.main
   type: run_function
   append_row_to: ${{outputs.job_out_path}}
 input_data: ${{inputs.data_input_table}}

assets/aml-benchmark/components/batch-inference-preparer/spec.yaml

@@ -58,7 +58,7 @@ outputs:
 code: ../src
 environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
-  python -m batch_inference_preparer.main
+  python -m aml_benchmark.batch_inference_preparer.main
   --batch_input_pattern '${{inputs.batch_input_pattern}}'
   --formatted_data ${{outputs.formatted_data}}
   --output_metadata ${{outputs.ground_truth_metadata}}

assets/aml-benchmark/components/batch-output-formatter/spec.yaml

@@ -51,7 +51,7 @@ resources:
   instance_count: 1
 
 command: >-
-  python -m batch_output_formatter.main
+  python -m aml_benchmark.batch_output_formatter.main
   --batch_inference_output ${{inputs.batch_inference_output}}
   --prediction_data ${{outputs.predictions}}
   --perf_data ${{outputs.performance_metadata}}

assets/aml-benchmark/components/batch-resource-manager/spec.yaml

@@ -67,7 +67,7 @@ outputs:
 code: ../src
 environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
-  python -m batch_resource_manager.main
+  python -m aml_benchmark.batch_resource_manager.main
   --output_metadata ${{outputs.output_metadata}}
   $[[--endpoint_name ${{inputs.endpoint_name}}]]
   $[[--deployment_name ${{inputs.deployment_name}}]]

assets/aml-benchmark/components/benchmark-result-aggregator/spec.yaml

@@ -25,7 +25,7 @@ outputs:
 code: ../src
 environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
-  python -m benchmark_result_aggregator.main 
+  python -m aml_benchmark.result_aggregator.main 
   $[[--quality_metrics_path ${{inputs.quality_metrics}}]] 
   $[[--performance_metrics_path ${{inputs.performance_metrics}}]] 
   --output_dataset_path ${{outputs.benchmark_result}}

assets/aml-benchmark/components/compute-performance-metrics/spec.yaml

@@ -58,7 +58,7 @@ outputs:
 code: ../src
 environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
-  python -m compute_performance_metrics.main
+  python -m aml_benchmark.perf_metrics.main
   --performance_data ${{inputs.performance_data}}
   --batch_size_column_name ${{inputs.batch_size_column_name}}
   --start_time_column_name ${{inputs.start_time_column_name}}

assets/aml-benchmark/components/dataset-downloader/spec.yaml

@@ -36,7 +36,7 @@ outputs:
 code: ../src
 environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
-  python -m dataset_downloader.main
+  python -m aml_benchmark.dataset_downloader.main
   $[[--dataset_name ${{inputs.dataset_name}}]] 
   $[[--configuration ${{inputs.configuration}}]] 
   --split ${{inputs.split}} 

assets/aml-benchmark/components/dataset-preprocessor/spec.yaml

@@ -53,7 +53,7 @@ code: ../src
 environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 
 command: >-
-  python -m dataset_preprocessor.main
+  python -m aml_benchmark.dataset_preprocessor.main
   --dataset ${{inputs.dataset}}
   $[[--template_input '${{inputs.template_input}}']]
   $[[--script_path ${{inputs.script_path}}]]

assets/aml-benchmark/components/dataset-sampler/spec.yaml

@@ -49,7 +49,7 @@ outputs:
 code: ../src
 environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
-  python -m dataset_sampler.main 
+  python -m aml_benchmark.dataset_sampler.main 
   --dataset ${{inputs.dataset}} 
   --sampling_style ${{inputs.sampling_style}} 
   $[[--sampling_ratio ${{inputs.sampling_ratio}}]] 

assets/aml-benchmark/components/inference-postprocessor/spec.yaml

@@ -128,7 +128,7 @@ code: ../src
 environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 
 command: >-
-  python -m inference_postprocessor.main
+  python -m aml_benchmark.inference_postprocessor.main
   --prediction_dataset ${{inputs.prediction_dataset}}
   --prediction_column_name ${{inputs.prediction_column_name}}
   $[[--ground_truth_dataset ${{inputs.ground_truth_dataset}}]]

assets/aml-benchmark/components/prompt_crafter/spec.yaml

@@ -6,7 +6,7 @@ display_name: Prompt Crafter
 description: This component is used to create prompts from a given dataset. From a 
   given jinja prompt template, it will generate prompts. It can also create 
   few-shot prompts given a few-shot dataset and the number of shots.
-version: 0.0.2.2
+version: 0.0.2
 is_deterministic: true
 
 inputs:
@@ -108,7 +108,7 @@ outputs:
 code: ../src
 environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
-  python -m prompt_crafter.main 
+  python -m aml_benchmark.prompt_crafter.main 
   --test_data ${{inputs.test_data}}
   $[[--few_shot_data ${{inputs.few_shot_data}}]]
   --n_shots ${{inputs.n_shots}}

assets/aml-benchmark/components/src/batch_benchmark_score/__init__.py → assets/aml-benchmark/components/src/aml_benchmark/batch_benchmark_score/__init__.py

@@ -7,4 +7,6 @@
 import os
 
 
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+path = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+if path not in sys.path:
+    sys.path.append(path)

assets/aml-benchmark/components/src/batch_benchmark_score/batch_score/header_handlers/oai/oai_header_handler.py → assets/aml-benchmark/components/src/aml_benchmark/batch_benchmark_score/batch_score/header_handlers/oai/oai_header_handler.py

@@ -8,8 +8,8 @@
 from ..header_handler import HeaderHandler
 from ...utils.token_provider import TokenProvider
 
-from utils.online_endpoint.aoai_online_endpoint import AOAIOnlineEndpoint
-from utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
+from aml_benchmark.utils.online_endpoint.aoai_online_endpoint import AOAIOnlineEndpoint
+from aml_benchmark.utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
 
 
 class OAIHeaderHandler(HeaderHandler):

assets/aml-benchmark/components/src/batch_benchmark_score/batch_score/header_handlers/oss/oss_header_handler.py → assets/aml-benchmark/components/src/aml_benchmark/batch_benchmark_score/batch_score/header_handlers/oss/oss_header_handler.py

@@ -9,8 +9,8 @@
 from ...utils.token_provider import TokenProvider
 from ...utils.common import constants
 
-from utils.online_endpoint.oss_online_endpoint import OSSOnlineEndpoint
-from utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
+from aml_benchmark.utils.online_endpoint.oss_online_endpoint import OSSOnlineEndpoint
+from aml_benchmark.utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
 
 
 class OSSHeaderHandler(HeaderHandler):

assets/aml-benchmark/components/src/batch_benchmark_score/batch_score/main.py → assets/aml-benchmark/components/src/aml_benchmark/batch_benchmark_score/batch_score/main.py

@@ -13,8 +13,8 @@
 
 from azureml._common._error_definition.azureml_error import AzureMLError
 
-from utils.online_endpoint.endpoint_utils import EndpointUtilities
-from utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
+from aml_benchmark.utils.online_endpoint.endpoint_utils import EndpointUtilities
+from aml_benchmark.utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
 
 from .utils.exceptions import BenchmarkValidationException
 from .utils.error_definitions import BenchmarkValidationError

assets/aml-benchmark/components/src/batch_inference_preparer/__init__.py → assets/aml-benchmark/components/src/aml_benchmark/batch_inference_preparer/__init__.py

@@ -7,4 +7,6 @@
 import os
 
 
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+path = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+if path not in sys.path:
+    sys.path.append(path)

assets/aml-benchmark/components/src/batch_inference_preparer/endpoint_data_preparer.py → assets/aml-benchmark/components/src/aml_benchmark/batch_inference_preparer/endpoint_data_preparer.py

@@ -7,8 +7,8 @@
 import json
 import re
 
-from utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
-from utils.online_endpoint.endpoint_utils import EndpointUtilities
+from aml_benchmark.utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
+from aml_benchmark.utils.online_endpoint.endpoint_utils import EndpointUtilities
 
 
 class EndpointDataPreparer:

assets/aml-benchmark/components/src/batch_inference_preparer/main.py → assets/aml-benchmark/components/src/aml_benchmark/batch_inference_preparer/main.py

@@ -7,12 +7,12 @@
 import argparse
 import pandas as pd
 
-from utils.io import resolve_io_path, read_jsonl_files
-from utils.logging import get_logger
-from utils.aml_run_utils import str2bool
+from aml_benchmark.utils.io import resolve_io_path, read_jsonl_files
+from aml_benchmark.utils.logging import get_logger
+from aml_benchmark.utils.aml_run_utils import str2bool
 from .endpoint_data_preparer import EndpointDataPreparer
-from utils.exceptions import swallow_all_exceptions
-from utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
+from aml_benchmark.utils.exceptions import swallow_all_exceptions
+from aml_benchmark.utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
 
 
 logger = get_logger(__name__)

assets/aml-benchmark/components/src/batch_output_formatter/__init__.py → assets/aml-benchmark/components/src/aml_benchmark/batch_output_formatter/__init__.py

@@ -7,4 +7,6 @@
 import os
 
 
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+path = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+if path not in sys.path:
+    sys.path.append(path)

assets/aml-benchmark/components/src/batch_output_formatter/main.py → assets/aml-benchmark/components/src/aml_benchmark/batch_output_formatter/main.py

@@ -7,11 +7,11 @@
 import os
 import pandas as pd
 
-from utils.io import resolve_io_path, read_jsonl_files
-from utils.logging import get_logger
-from utils.exceptions import swallow_all_exceptions
-from utils.aml_run_utils import str2bool
-from utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
+from aml_benchmark.utils.io import resolve_io_path, read_jsonl_files
+from aml_benchmark.utils.logging import get_logger
+from aml_benchmark.utils.exceptions import swallow_all_exceptions
+from aml_benchmark.utils.aml_run_utils import str2bool
+from aml_benchmark.utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
 from .result_converters import ResultConverters
 
 

assets/aml-benchmark/components/src/batch_output_formatter/result_converters.py → assets/aml-benchmark/components/src/aml_benchmark/batch_output_formatter/result_converters.py

@@ -8,10 +8,10 @@
 from typing import Any, Dict, Optional, Tuple
 import pandas as pd
 
-from utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
-from utils.logging import get_logger
-from utils.online_endpoint.endpoint_utils import EndpointUtilities
-from batch_inference_preparer.endpoint_data_preparer import EndpointDataPreparer
+from aml_benchmark.utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
+from aml_benchmark.utils.logging import get_logger
+from aml_benchmark.utils.online_endpoint.endpoint_utils import EndpointUtilities
+from aml_benchmark.batch_inference_preparer.endpoint_data_preparer import EndpointDataPreparer
 
 
 logger = get_logger(__name__)

assets/aml-benchmark/components/src/batch_resource_manager/__init__.py → assets/aml-benchmark/components/src/aml_benchmark/batch_resource_manager/__init__.py

@@ -7,4 +7,6 @@
 import os
 
 
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+path = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+if path not in sys.path:
+    sys.path.append(path)

assets/aml-benchmark/components/src/batch_resource_manager/main.py → assets/aml-benchmark/components/src/aml_benchmark/batch_resource_manager/main.py

@@ -7,13 +7,13 @@
 
 import argparse
 
-from utils.logging import get_logger
-from utils.exceptions import swallow_all_exceptions
-from utils.aml_run_utils import str2bool
-from utils.online_endpoint.endpoint_utils import EndpointUtilities
-from utils.online_endpoint.online_endpoint_factory import OnlineEndpointFactory
-from utils.online_endpoint.online_endpoint import OnlineEndpoint, ResourceState
-from utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
+from aml_benchmark.utils.logging import get_logger
+from aml_benchmark.utils.exceptions import swallow_all_exceptions
+from aml_benchmark.utils.aml_run_utils import str2bool
+from aml_benchmark.utils.online_endpoint.endpoint_utils import EndpointUtilities
+from aml_benchmark.utils.online_endpoint.online_endpoint_factory import OnlineEndpointFactory
+from aml_benchmark.utils.online_endpoint.online_endpoint import OnlineEndpoint, ResourceState
+from aml_benchmark.utils.online_endpoint.online_endpoint_model import OnlineEndpointModel
 
 
 logger = get_logger(__name__)

assets/aml-benchmark/components/src/dataset_downloader/__init__.py → assets/aml-benchmark/components/src/aml_benchmark/dataset_downloader/__init__.py

@@ -7,4 +7,6 @@
 import os
 
 
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+path = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+if path not in sys.path:
+    sys.path.append(path)

assets/aml-benchmark/components/src/dataset_downloader/main.py → assets/aml-benchmark/components/src/aml_benchmark/dataset_downloader/main.py

@@ -11,13 +11,13 @@
 import pandas as pd
 from azureml._common._error_definition.azureml_error import AzureMLError
 
-from utils.logging import get_logger, log_mlflow_params
-from utils.exceptions import (
+from aml_benchmark.utils.logging import get_logger, log_mlflow_params
+from aml_benchmark.utils.exceptions import (
     swallow_all_exceptions,
     BenchmarkValidationException,
     DatasetDownloadException,
 )
-from utils.error_definitions import BenchmarkValidationError, DatasetDownloadError
+from aml_benchmark.utils.error_definitions import BenchmarkValidationError, DatasetDownloadError
 
 
 logger = get_logger(__name__)

assets/aml-benchmark/components/src/dataset_preprocessor/__init__.py → assets/aml-benchmark/components/src/aml_benchmark/dataset_preprocessor/__init__.py

@@ -6,4 +6,6 @@
 import sys
 import os
 
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+path = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+if path not in sys.path:
+    sys.path.append(path)