6 changed files with 465 additions and 28 deletions
@ -0,0 +1,58 @@ |
|||||
|
{ |
||||
|
"files.associations": { |
||||
|
"xiosbase": "cpp", |
||||
|
"algorithm": "cpp", |
||||
|
"array": "cpp", |
||||
|
"atomic": "cpp", |
||||
|
"bit": "cpp", |
||||
|
"cctype": "cpp", |
||||
|
"clocale": "cpp", |
||||
|
"cmath": "cpp", |
||||
|
"compare": "cpp", |
||||
|
"concepts": "cpp", |
||||
|
"cstddef": "cpp", |
||||
|
"cstdint": "cpp", |
||||
|
"cstdio": "cpp", |
||||
|
"cstdlib": "cpp", |
||||
|
"cstring": "cpp", |
||||
|
"ctime": "cpp", |
||||
|
"cwchar": "cpp", |
||||
|
"exception": "cpp", |
||||
|
"functional": "cpp", |
||||
|
"initializer_list": "cpp", |
||||
|
"ios": "cpp", |
||||
|
"iosfwd": "cpp", |
||||
|
"iostream": "cpp", |
||||
|
"istream": "cpp", |
||||
|
"iterator": "cpp", |
||||
|
"limits": "cpp", |
||||
|
"list": "cpp", |
||||
|
"memory": "cpp", |
||||
|
"new": "cpp", |
||||
|
"numeric": "cpp", |
||||
|
"optional": "cpp", |
||||
|
"ostream": "cpp", |
||||
|
"stdexcept": "cpp", |
||||
|
"streambuf": "cpp", |
||||
|
"string": "cpp", |
||||
|
"system_error": "cpp", |
||||
|
"tuple": "cpp", |
||||
|
"type_traits": "cpp", |
||||
|
"typeinfo": "cpp", |
||||
|
"unordered_map": "cpp", |
||||
|
"unordered_set": "cpp", |
||||
|
"utility": "cpp", |
||||
|
"variant": "cpp", |
||||
|
"vector": "cpp", |
||||
|
"xfacet": "cpp", |
||||
|
"xhash": "cpp", |
||||
|
"xlocale": "cpp", |
||||
|
"xlocinfo": "cpp", |
||||
|
"xlocnum": "cpp", |
||||
|
"xmemory": "cpp", |
||||
|
"xstddef": "cpp", |
||||
|
"xstring": "cpp", |
||||
|
"xtr1common": "cpp", |
||||
|
"xutility": "cpp" |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,297 @@ |
|||||
|
#include "ofxOnnxRuntime.h" |
||||
|
|
||||
|
namespace ofxOnnxRuntime |
||||
|
{ |
||||
|
#ifdef _MSC_VER |
||||
|
static std::wstring to_wstring(const std::string &str) |
||||
|
{ |
||||
|
unsigned len = str.size() * 2; |
||||
|
setlocale(LC_CTYPE, ""); |
||||
|
wchar_t *p = new wchar_t[len]; |
||||
|
mbstowcs(p, str.c_str(), len); |
||||
|
std::wstring wstr(p); |
||||
|
delete[] p; |
||||
|
return wstr; |
||||
|
} |
||||
|
#endif |
||||
|
void BaseHandler::setup(const std::string & onnx_path, const BaseSetting & base_setting, const int & batch_size, const bool debug, const bool timestamp) |
||||
|
{ |
||||
|
// Store data types
|
||||
|
this->input_dtype = base_setting.input_dtype; |
||||
|
this->output_dtype = base_setting.output_dtype; |
||||
|
|
||||
|
Ort::SessionOptions session_options; |
||||
|
session_options.SetIntraOpNumThreads(1); |
||||
|
session_options.SetIntraOpNumThreads(1); |
||||
|
session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); |
||||
|
|
||||
|
if (base_setting.infer_type == INFER_CUDA) { |
||||
|
OrtCUDAProviderOptions opts; |
||||
|
opts.device_id = 0; |
||||
|
opts.cudnn_conv_algo_search = OrtCudnnConvAlgoSearchExhaustive; |
||||
|
opts.do_copy_in_default_stream = 0; |
||||
|
opts.arena_extend_strategy = 0; |
||||
|
session_options.AppendExecutionProvider_CUDA(opts); |
||||
|
} |
||||
|
|
||||
|
this->timestamp = timestamp; |
||||
|
this->debug = debug; |
||||
|
this->batch_size = batch_size; |
||||
|
this->setup2(onnx_path, session_options); |
||||
|
} |
||||
|
|
||||
|
void BaseHandler::setup2(const std::string & onnx_path, const Ort::SessionOptions & session_options) |
||||
|
{ |
||||
|
std::string path = ofToDataPath(onnx_path, true); |
||||
|
|
||||
|
std::wstring wpath(path.begin(), path.end()); // basic conversion
|
||||
|
|
||||
|
ort_session = std::make_shared<Ort::Session>(ort_env, wpath.c_str(), session_options); |
||||
|
|
||||
|
setNames(); |
||||
|
} |
||||
|
|
||||
|
void BaseHandler::setNames() |
||||
|
{ |
||||
|
Ort::AllocatorWithDefaultOptions allocator; |
||||
|
|
||||
|
// 1. Gets Input Name/s & Shape ([1, 3, 28, 28]) -- In most cases this is usually just one
|
||||
|
for (std::size_t i = 0; i < ort_session->GetInputCount(); i++) { |
||||
|
input_node_names.emplace_back(ort_session->GetInputNameAllocated(i, allocator).get()); |
||||
|
input_node_dims = ort_session->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); |
||||
|
|
||||
|
// Some models might have negative shape values to indicate dynamic shape, e.g., for variable batch size. (?, 3, 28, 28) -> (1, 3, 28, 28)
|
||||
|
for (auto& s : input_node_dims) if (s < 0) s = batch_size; |
||||
|
|
||||
|
if (debug) std::cout << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl; |
||||
|
} |
||||
|
|
||||
|
// 2. Calculate the product of the dimensions
|
||||
|
for (auto& f : input_node_dims) { |
||||
|
input_node_size *= f; |
||||
|
} |
||||
|
|
||||
|
if (debug) ofLog() << ofToString(input_node_size) + ", Batch Size:" + ofToString(input_node_dims[0]); |
||||
|
|
||||
|
// 2. Clear up output values
|
||||
|
output_node_dims.clear(); |
||||
|
output_values.clear(); |
||||
|
|
||||
|
// 3. Gets Output name/s & Shapes
|
||||
|
for (std::size_t i = 0; i < ort_session->GetOutputCount(); i++) { |
||||
|
output_node_names.emplace_back(ort_session->GetOutputNameAllocated(i, allocator).get()); |
||||
|
auto output_shapes = ort_session->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); |
||||
|
|
||||
|
output_values.emplace_back(nullptr); |
||||
|
|
||||
|
if (debug) std::cout << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
float* BaseHandler::run() |
||||
|
{ |
||||
|
|
||||
|
auto start = std::chrono::high_resolution_clock::now(); // starting timestamp
|
||||
|
|
||||
|
std::vector<Ort::Value> input_tensors; |
||||
|
|
||||
|
size_t num_images = input_imgs.size(); |
||||
|
|
||||
|
if(input_imgs.size() != batch_size) { |
||||
|
ofLog() << "Input images do not match batch size. Inference FAILED."; |
||||
|
return dummy_output_tensor.front().GetTensorMutableData<float>(); |
||||
|
} |
||||
|
|
||||
|
// transform std::string -> const char*
|
||||
|
std::vector<const char*> input_names_char(input_node_names.size(), nullptr); |
||||
|
std::transform(std::begin(input_node_names), std::end(input_node_names), std::begin(input_names_char), |
||||
|
[&](const std::string& str) { return str.c_str(); }); |
||||
|
|
||||
|
std::vector<const char*> output_names_char(output_node_names.size(), nullptr); |
||||
|
std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char), |
||||
|
[&](const std::string& str) { return str.c_str(); }); |
||||
|
|
||||
|
std::vector<float> batch_values_f; |
||||
|
std::vector<int32_t> batch_values_int32; |
||||
|
batch_values_f.reserve(input_node_size * batch_size); // Reserve space but don't initialize
|
||||
|
batch_values_int32.reserve(input_node_size * batch_size); // Reserve space but don't initialize
|
||||
|
|
||||
|
|
||||
|
if (input_dtype == ModelDataType::FLOAT32){ |
||||
|
// I have a list of imgs, these need to be converted from images into input for the model (int or float)
|
||||
|
for(size_t i = 0; i < batch_size; i++) { |
||||
|
convertImageToMatFloat(input_imgs[i], batch_values_f, i); |
||||
|
} |
||||
|
|
||||
|
// 2. Create tensor with batch values { input data, input size, model input dims, model input size}
|
||||
|
input_tensors.emplace_back(Ort::Value::CreateTensor<float>( |
||||
|
memory_info_handler, batch_values_f.data(), input_node_size, |
||||
|
input_node_dims.data(), input_node_dims.size())); |
||||
|
} |
||||
|
else if (input_dtype == ModelDataType::INT32) { |
||||
|
// I have a list of imgs, these need to be converted from images into input for the model (int or float)
|
||||
|
for(size_t i = 0; i < batch_size; i++) { |
||||
|
convertImageToMatInt32(input_imgs[i], batch_values_int32, i); |
||||
|
} |
||||
|
|
||||
|
// 2. Create tensor with batch values { input data, input size, model input dims, model input size}
|
||||
|
input_tensors.emplace_back(Ort::Value::CreateTensor<int32_t>( |
||||
|
memory_info_handler, batch_values_int32.data(), input_node_size, |
||||
|
input_node_dims.data(), input_node_dims.size())); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
|
||||
|
|
||||
|
try { |
||||
|
// 3. Run inference, { in names, input data, num of inputs, output names, num of outputs }
|
||||
|
ofLog() << "run"; |
||||
|
output_values = ort_session->Run(Ort::RunOptions{ nullptr }, |
||||
|
input_names_char.data(), input_tensors.data(), |
||||
|
input_names_char.size(), output_names_char.data(), |
||||
|
output_names_char.size()); |
||||
|
ofLog() << "ran"; |
||||
|
|
||||
|
if (debug) { |
||||
|
// Gets the address of the first value
|
||||
|
auto& out = output_values.front(); |
||||
|
// Get tensor shape information
|
||||
|
Ort::TensorTypeAndShapeInfo info = out.GetTensorTypeAndShapeInfo(); |
||||
|
std::vector<int64_t> output_dims = info.GetShape(); |
||||
|
|
||||
|
// Print the dimensions
|
||||
|
std::cout << "Output tensor dimensions: ["; |
||||
|
for (size_t i = 0; i < output_dims.size(); i++) { |
||||
|
std::cout << output_dims[i]; |
||||
|
if (i < output_dims.size() - 1) { |
||||
|
std::cout << ", "; |
||||
|
} |
||||
|
} |
||||
|
std::cout << "]" << std::endl; |
||||
|
|
||||
|
// Optional: Print total number of elements
|
||||
|
size_t total_elements = 1; |
||||
|
for (auto& dim : output_dims) { |
||||
|
if (dim > 0) { // Handle dynamic dimensions
|
||||
|
total_elements *= static_cast<size_t>(dim); |
||||
|
} |
||||
|
} |
||||
|
std::cout << "Total elements: " << total_elements << std::endl; |
||||
|
} |
||||
|
|
||||
|
// if (timestamp) {
|
||||
|
// auto end = std::chrono::high_resolution_clock::now();
|
||||
|
// std::chrono::duration<double, std::milli> elapsed = end - start;
|
||||
|
// std::cout << "Update loop took " << elapsed.count() << " ms" << std::endl;
|
||||
|
// }
|
||||
|
|
||||
|
return output_values.front().GetTensorMutableData<float>(); |
||||
|
|
||||
|
} catch (const Ort::Exception& ex) { |
||||
|
std::cout << "ERROR running model inference: " << ex.what() << std::endl; |
||||
|
return dummy_output_tensor.front().GetTensorMutableData<float>(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
/*
|
||||
|
* |
||||
|
* Utilties (｡･∀･)ﾉﾞ |
||||
|
* |
||||
|
*/ |
||||
|
|
||||
|
// Add separate methods for float and int32 conversion
|
||||
|
void BaseHandler::convertImageToMatFloat(ofImage* img, std::vector<float>& values, size_t& idx) { |
||||
|
// Your existing conversion code for float
|
||||
|
ofPixels& pix = img->getPixels(); |
||||
|
int width = img->getWidth(); |
||||
|
int height = img->getHeight(); |
||||
|
int channels = pix.getNumChannels(); |
||||
|
|
||||
|
cv::Mat cvImage = cv::Mat(height, width, (channels == 3) ? CV_8UC3 : CV_8UC1, pix.getData()); |
||||
|
cv::InputArray inputArray(cvImage); |
||||
|
image_array = cv::dnn::blobFromImage(inputArray, 1 / 255.0, cv::Size(input_node_dims[2], input_node_dims[3]), (0, 0, 0), false, false); |
||||
|
|
||||
|
std::memcpy( |
||||
|
values.data() + idx * channels * width * height, |
||||
|
image_array.data, |
||||
|
channels * width * height * sizeof(float) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
void BaseHandler::convertImageToMatInt32(ofImage* img, std::vector<int32_t>& values, size_t& idx) { |
||||
|
// New conversion code for int32
|
||||
|
ofPixels& pix = img->getPixels(); |
||||
|
int width = img->getWidth(); |
||||
|
int height = img->getHeight(); |
||||
|
int channels = pix.getNumChannels(); |
||||
|
|
||||
|
cv::Mat cvImage = cv::Mat(height, width, (channels == 3) ? CV_8UC3 : CV_8UC1, pix.getData()); |
||||
|
cv::InputArray inputArray(cvImage); |
||||
|
cv::Mat intMat = cv::dnn::blobFromImage(inputArray, 1 / 255.0, cv::Size(height, width), (0, 0, 0), false, false); |
||||
|
|
||||
|
intMat.convertTo(image_array, CV_32S); |
||||
|
|
||||
|
std::memcpy( |
||||
|
values.data() + idx * channels * width * height, |
||||
|
image_array.data, |
||||
|
channels * width * height * sizeof(int32_t) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
void BaseHandler::setInputs(std::vector<ofImage*>& in) { |
||||
|
this->input_imgs = in; |
||||
|
} |
||||
|
|
||||
|
// Prints the shape of the given tensor (ex. input: (1, 1, 512, 512))
|
||||
|
std::string BaseHandler::PrintShape(const std::vector<int64_t>& v) { |
||||
|
std::stringstream ss; |
||||
|
for (std::size_t i = 0; i < v.size() - 1; i++) ss << v[i] << "x"; |
||||
|
ss << v[v.size() - 1]; |
||||
|
return ss.str(); |
||||
|
} |
||||
|
|
||||
|
Ort::Value BaseHandler::GenerateTensor(int batch_size) { |
||||
|
// Random number generation setup
|
||||
|
std::random_device rd; |
||||
|
std::mt19937 gen(rd()); |
||||
|
std::uniform_real_distribution<float> dis(0.0f, 255.0f); // Random values between 0 and 255
|
||||
|
|
||||
|
// Calculate the total number of elements for a single tensor (without batch dimension) {?, 8} -> 8
|
||||
|
int tensor_size = CalculateProduct(input_node_dims); |
||||
|
|
||||
|
// Create a vector to hold all the values for the batch (8 * (4)batch_size) -> 32
|
||||
|
std::vector<float> batch_values(batch_size * tensor_size); |
||||
|
|
||||
|
// Fill the batch with random values
|
||||
|
std::generate(batch_values.begin(), batch_values.end(), [&]() { |
||||
|
return dis(gen); |
||||
|
}); |
||||
|
|
||||
|
// Fill the batch with random values
|
||||
|
std::generate(batch_values.begin(), batch_values.end(), [&]() { |
||||
|
return dis(gen); |
||||
|
}); |
||||
|
|
||||
|
// Create the batched dimensions by inserting the batch size at the beginning of the original dimensions
|
||||
|
std::vector<int64_t> batched_dims = { }; // Start with batch size
|
||||
|
batched_dims.insert(batched_dims.end(), input_node_dims.begin(), input_node_dims.end()); // Add the remaining dimensions
|
||||
|
batched_dims[0] = batch_size; |
||||
|
|
||||
|
return VectorToTensor(batch_values, batched_dims); |
||||
|
} |
||||
|
|
||||
|
int BaseHandler::CalculateProduct(const std::vector<int64_t>& v) { |
||||
|
int total = 1; |
||||
|
for (auto& i : v) total *= i; |
||||
|
return total; |
||||
|
} |
||||
|
|
||||
|
Ort::Value BaseHandler::VectorToTensor(std::vector<float>& data, const std::vector<int64_t>& shape) { |
||||
|
// Create a tensor from the provided data, shape, and memory info
|
||||
|
auto tensor = Ort::Value::CreateTensor<float>(memory_info_handler, data.data(), data.size(), shape.data(), shape.size()); |
||||
|
|
||||
|
// Return the created tensor
|
||||
|
return tensor; |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue