Merge pull request #76 from UM-Bridge/testing-multi-input

Modified CI to allow vector input

testing/protocol_conformity_1.0/conftest.py

@@ -2,7 +2,7 @@
 
 def pytest_addoption(parser):
     parser.addoption("--model_url", action="store", help="what model URL to connect to")
-    parser.addoption("--input_value", action="store", default=0.0, help="value of constant input vector to be passed to the model, since some models may not handle the default well")
+    parser.addoption("--input_value", action="store", default=0.0, help="input vector or constant value to fill the vector to be passed to the model, since some models may not handle the default well. No spaces allowed when inputting vector!")
 
 def pytest_generate_tests(metafunc):
     option_value = metafunc.config.option.model_url
@@ -11,4 +11,8 @@ def pytest_generate_tests(metafunc):
 
     option_value = metafunc.config.option.input_value
     if 'input_value' in metafunc.fixturenames and option_value is not None:
-        metafunc.parametrize("input_value", [float(option_value)])
+        if isinstance(option_value, str):
+            option_value = eval(option_value)
+            metafunc.parametrize("input_value", [option_value])
+        else:
+            metafunc.parametrize("input_value", [float(option_value)])

testing/protocol_conformity_1.0/test_model_schema.py

@@ -29,14 +29,21 @@ def test_evaluate(model_url, input_value):
     inputSizesJSON = requests.post(f'{model_url}/InputSizes', json=input_model_name).json()
 
     inputParams = {"input": [], "name": model_name, "config": {}}
-    input_value_len = getattr(input_value, '__len__', lambda:1)()
-    for i in range(0,len(inputSizesJSON["inputSizes"])):
-      assert input_value_len == 1 or input_value_len == inputSizesJSON["inputSizes"][i]
-      if input_value_len == 1:
-        inputParams["input"].append([input_value] * inputSizesJSON["inputSizes"][i])
-      else:
+    for i in range(0, len(inputSizesJSON["inputSizes"])):
+      inputSizesJSON_i = inputSizesJSON["inputSizes"][i]
+      inputSizesJSON_len = len(inputSizesJSON["inputSizes"])
+      # Handles the case for one input vector
+      if hasattr(input_value, '__len__') and inputSizesJSON_len == 1:
+        assert len(input_value) == inputSizesJSON_i
+        inputParams["input"].append(input_value)
+      # The case where there are multiple input vector
+      elif hasattr(input_value, '__len__') and inputSizesJSON_len != 1:
+        assert len(input_value[i]) == inputSizesJSON_i
         inputParams["input"].append(input_value[i])
-
+      # Single number entered, will be expanded if sizes unmatched
+      else:
+        inputParams["input"].append([input_value] * inputSizesJSON_i)
+        assert len(inputParams["input"][i]) == inputSizesJSON_i
 
     resp = requests.post(f'{model_url}/Evaluate', headers={}, data=json.dumps(inputParams,indent=4))