diff --git a/testing/protocol_conformity_1.0/conftest.py b/testing/protocol_conformity_1.0/conftest.py index 190abe9..3cf1fb1 100644 --- a/testing/protocol_conformity_1.0/conftest.py +++ b/testing/protocol_conformity_1.0/conftest.py @@ -2,7 +2,7 @@ def pytest_addoption(parser): parser.addoption("--model_url", action="store", help="what model URL to connect to") - parser.addoption("--input_value", action="store", default=0.0, help="value of constant input vector to be passed to the model, since some models may not handle the default well") + parser.addoption("--input_value", action="store", default=0.0, help="input vector or constant value to fill the vector to be passed to the model, since some models may not handle the default well. No spaces allowed when inputting vector!") def pytest_generate_tests(metafunc): option_value = metafunc.config.option.model_url @@ -11,4 +11,8 @@ def pytest_generate_tests(metafunc): option_value = metafunc.config.option.input_value if 'input_value' in metafunc.fixturenames and option_value is not None: - metafunc.parametrize("input_value", [float(option_value)]) + if isinstance(option_value, str): + option_value = eval(option_value) + metafunc.parametrize("input_value", [option_value]) + else: + metafunc.parametrize("input_value", [float(option_value)]) diff --git a/testing/protocol_conformity_1.0/test_model_schema.py b/testing/protocol_conformity_1.0/test_model_schema.py index de059c0..406fa88 100644 --- a/testing/protocol_conformity_1.0/test_model_schema.py +++ b/testing/protocol_conformity_1.0/test_model_schema.py @@ -29,14 +29,21 @@ def test_evaluate(model_url, input_value): inputSizesJSON = requests.post(f'{model_url}/InputSizes', json=input_model_name).json() inputParams = {"input": [], "name": model_name, "config": {}} - input_value_len = getattr(input_value, '__len__', lambda:1)() - for i in range(0,len(inputSizesJSON["inputSizes"])): - assert input_value_len == 1 or input_value_len == inputSizesJSON["inputSizes"][i] - if input_value_len == 1: - inputParams["input"].append([input_value] * inputSizesJSON["inputSizes"][i]) - else: + for i in range(0, len(inputSizesJSON["inputSizes"])): + inputSizesJSON_i = inputSizesJSON["inputSizes"][i] + inputSizesJSON_len = len(inputSizesJSON["inputSizes"]) + # Handles the case for one input vector + if hasattr(input_value, '__len__') and inputSizesJSON_len == 1: + assert len(input_value) == inputSizesJSON_i + inputParams["input"].append(input_value) + # The case where there are multiple input vector + elif hasattr(input_value, '__len__') and inputSizesJSON_len != 1: + assert len(input_value[i]) == inputSizesJSON_i inputParams["input"].append(input_value[i]) - + # Single number entered, will be expanded if sizes unmatched + else: + inputParams["input"].append([input_value] * inputSizesJSON_i) + assert len(inputParams["input"][i]) == inputSizesJSON_i resp = requests.post(f'{model_url}/Evaluate', headers={}, data=json.dumps(inputParams,indent=4))