Skip to content

Commit

Permalink
update: Enabled OV devices update parsing on runtime
Browse files Browse the repository at this point in the history
  • Loading branch information
@ankitm3k
ankitm3k committed Jul 12, 2024
1 parent 4573740 commit 526513c
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 21 deletions.
2 changes: 1 addition & 1 deletion onnxruntime/core/providers/openvino/openvino_execution_provider.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv
info.device_type_.find("AUTO") != std::string::npos) {
device_found = true;
} else {
for (auto device : available_devices) {
for (const std::string& device : available_devices) {
if (device.rfind(info.device_type_, 0) == 0) {
if (info.device_type_.find("GPU") != std::string::npos && (info.precision_ == "FP32" ||
info.precision_ == "FP16" ||
Expand Down
51 changes: 39 additions & 12 deletions onnxruntime/core/providers/openvino/openvino_execution_provider.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,29 @@
#include <memory>
#include <vector>
#include <set>
#include <utility>

#include "core/providers/openvino/backend_manager.h"

namespace onnxruntime {

struct OVDevices {
ov::Core core;
std::vector<std::string> get_ov_devices() const {
return core.get_available_devices();
}
};

static void print_build_options() {
std::cout << "[ERROR] INVALID DEVICE BUILD TYPE SPECIFIED" << std::endl;
std::cout << "Specify the keyword HETERO (or) MULTI (or) AUTO followed by the devices in the order of priority "
<< "you want to build"
<< std::endl;
std::cout << "The different hardware devices that can be added with HETERO/MULTI/AUTO build "
<< "are ['CPU','GPU','NPU']"
<< "are ['CPU','GPU','NPU','GPU.x'] where x = 0,1,2 and so on"
<< std::endl;
std::cout << "An example of how to specify the HETERO or MULTI or AUTO build type. "
<< "Ex: HETERO:GPU,CPU Ex: MULTI:GPU,CPU Ex: AUTO:GPU,CPU"
<< "Ex: HETERO:GPU,CPU Ex: MULTI:GPU,CPU Ex: AUTO:GPU,CPU Ex: AUTO:GPU.0,CPU Ex: AUTO:GPU.1,CPU"
<< std::endl;
}

Expand All @@ -39,7 +47,8 @@ static std::vector<std::string> split(const std::string& s, char delim) {
return result;
}

static std::vector<std::string> parseDevices(const std::string& device_string) {
static std::vector<std::string> parseDevices(const std::string& device_string,
const std::vector<std::string>& available_devices) {
std::string comma_separated_devices = device_string;
if (comma_separated_devices.find(":") != std::string::npos) {
comma_separated_devices = comma_separated_devices.substr(comma_separated_devices.find(":") + 1);
Expand All @@ -49,8 +58,15 @@ static std::vector<std::string> parseDevices(const std::string& device_string) {
print_build_options();
ORT_THROW("Invalid device string: " + device_string);
}
std::vector<std::string> dev_options = {"CPU", "GPU", "NPU"};
for (std::string dev : devices) {
std::set<std::string> dev_options = {"CPU", "GPU", "NPU"};

for (auto& device : available_devices) {
if (dev_options.find(device) == dev_options.end()) {
auto dev_options_update = dev_options.emplace(device);
}
}

for (const std::string& dev : devices) {
if (!std::count(dev_options.begin(), dev_options.end(), dev)) {
print_build_options();
ORT_THROW("Invalid device string: " + device_string);
Expand All @@ -75,14 +91,15 @@ struct OpenVINOExecutionProviderInfo {

OpenVINOExecutionProviderInfo() = delete;

explicit OpenVINOExecutionProviderInfo(std::string dev_type, std::string precision, bool enable_npu_fast_compile,
size_t num_of_threads, std::string cache_dir, std::string model_priority,
explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision,
bool enable_npu_fast_compile, size_t num_of_threads,
const std::string& cache_dir, const std::string& model_priority,
int num_streams, void* context, bool enable_opencl_throttling,
bool disable_dynamic_shapes, bool export_ep_ctx_blob)
: precision_(precision),
enable_npu_fast_compile_(enable_npu_fast_compile),
num_of_threads_(num_of_threads),
cache_dir_(cache_dir),
cache_dir_(std::move(cache_dir)),
model_priority_(model_priority),
num_streams_(num_streams),
context_(context),
Expand All @@ -91,6 +108,16 @@ struct OpenVINOExecutionProviderInfo {
export_ep_ctx_blob_(export_ep_ctx_blob) {
std::set<std::string> ov_supported_device_types = {"CPU", "GPU",
"GPU.0", "GPU.1", "NPU"};

OVDevices devices;
std::vector<std::string> available_devices = devices.get_ov_devices();

for (auto& device : available_devices) {
if (ov_supported_device_types.find(device) == ov_supported_device_types.end()) {
auto ov_supported_device_types_update = ov_supported_device_types.emplace(device);
}
}

if (dev_type == "") {
LOGS_DEFAULT(INFO) << "[OpenVINO-EP]"
<< "No runtime device selection option provided.";
Expand All @@ -110,18 +137,18 @@ struct OpenVINOExecutionProviderInfo {
dev_type = DEVICE;

if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0 || dev_type.find("AUTO") == 0) {
std::vector<std::string> devices = parseDevices(dev_type);
std::vector<std::string> devices = parseDevices(dev_type, available_devices);
precision_ = "FP16";
if (devices[0] == "CPU") {
precision_ = "FP32";
}
device_type_ = dev_type;
device_type_ = std::move(dev_type);
}
#endif
} else if (ov_supported_device_types.find(dev_type) != ov_supported_device_types.end()) {
device_type_ = dev_type;
device_type_ = std::move(dev_type);
} else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0 || dev_type.find("AUTO") == 0) {
std::vector<std::string> devices = parseDevices(dev_type);
std::vector<std::string> devices = parseDevices(dev_type, available_devices);
device_type_ = dev_type;
} else {
ORT_THROW("Invalid device string: " + dev_type);
Expand Down
12 changes: 10 additions & 2 deletions onnxruntime/core/providers/openvino/openvino_provider_factory.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ struct OpenVINO_Provider : Provider {
std::set<std::string> deprecated_device_types = {"CPU_FP32", "GPU_FP32",
"GPU.0_FP32", "GPU.1_FP32", "GPU_FP16",
"GPU.0_FP16", "GPU.1_FP16"};
OVDevices devices;
std::vector<std::string> available_devices = devices.get_ov_devices();

for (auto& device : available_devices) {
if (ov_supported_device_types.find(device) == ov_supported_device_types.end()) {
auto ov_supported_device_types_update = ov_supported_device_types.emplace(device);
}
}
if (deprecated_device_types.find(device_type) != deprecated_device_types.end()) {
std::string deprecated_device = device_type;
int delimit = device_type.find("_");
Expand All @@ -118,8 +126,8 @@ struct OpenVINO_Provider : Provider {
(device_type.find("MULTI:") == 0) ||
(device_type.find("AUTO:") == 0))) {
ORT_THROW(
"[ERROR] [OpenVINO] You have selcted wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'GPU.0', 'GPU.1', 'NPU' or from"
"[ERROR] [OpenVINO] You have selected wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'NPU', 'GPU.x' where x = 0,1,2 and so on or from"
" HETERO/MULTI/AUTO options available. \n");
}
}
Expand Down
22 changes: 16 additions & 6 deletions onnxruntime/test/perftest/ort_test_session.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,8 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
}
auto pos = token.find("|");
if (pos == std::string::npos || pos == 0 || pos == token.length()) {
ORT_THROW("[ERROR] [OpenVINO] Use a '|' to separate the key and value for the run-time option you are trying to use.\n");
ORT_THROW("[ERROR] [OpenVINO] Use a '|' to separate the key and value for the run-time option"
" you are trying to use.\n");
}

auto key = token.substr(0, pos);
Expand All @@ -250,6 +251,10 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
std::set<std::string> deprecated_device_types = {"CPU_FP32", "GPU_FP32",
"GPU.0_FP32", "GPU.1_FP32", "GPU_FP16",
"GPU.0_FP16", "GPU.1_FP16"};
size_t num_gpus = 10;
for (size_t i = 0; i <= num_gpus; i++) {
ov_supported_device_types.emplace("GPU." + std::to_string(i));
}
if (ov_supported_device_types.find(value) != ov_supported_device_types.end()) {
ov_options[key] = value;
} else if (deprecated_device_types.find(value) != deprecated_device_types.end()) {
Expand All @@ -262,8 +267,8 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] You have selcted wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'GPU.0', 'GPU.1', 'NPU' or from"
"[ERROR] [OpenVINO] You have selected wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'NPU', 'GPU.x' where x = 0,1,2 and so on or from"
" HETERO/MULTI/AUTO options available. \n");
}
} else if (key == "device_id") {
Expand Down Expand Up @@ -306,14 +311,16 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean i.e. true or false. Default value is false.\n");
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean"
" i.e. true or false. Default value is false.\n");
}
} else if (key == "enable_opencl_throttling") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean i.e. true or false. Default value is false.\n");
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean"
" i.e. true or false. Default value is false.\n");
}
} else if (key == "disable_dynamic_shapes") {
if (value == "true" || value == "True" ||
Expand Down Expand Up @@ -352,7 +359,10 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
"should be a boolean i.e. true or false. Default value is false.\n");
}
} else {
ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n");
ORT_THROW("[ERROR] [OpenVINO] wrong key type entered."
" Choose from the following runtime key options that are available for OpenVINO."
" ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir',"
" 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n");
}
}
session_options.AppendExecutionProvider_OpenVINO_V2(ov_options);
Expand Down

0 comments on commit 526513c

Please sign in to comment.