|
|
|
@ -1,5 +1,4 @@ |
|
|
|
#include "ofxOnnxRuntime.h" |
|
|
|
#include "ofMain.h" |
|
|
|
|
|
|
|
namespace ofxOnnxRuntime |
|
|
|
{ |
|
|
|
@ -16,7 +15,7 @@ namespace ofxOnnxRuntime |
|
|
|
} |
|
|
|
#endif |
|
|
|
|
|
|
|
void BaseHandler::setup(const std::string & onnx_path, const BaseSetting & base_setting, const std::vector<int64_t>& batched_dims, const int & batch_size) |
|
|
|
void BaseHandler::setup(const std::string & onnx_path, const BaseSetting & base_setting, const int & batch_size, const bool debug, const bool timestamp) |
|
|
|
{ |
|
|
|
Ort::SessionOptions session_options; |
|
|
|
session_options.SetIntraOpNumThreads(1); |
|
|
|
@ -32,9 +31,9 @@ namespace ofxOnnxRuntime |
|
|
|
session_options.AppendExecutionProvider_CUDA(opts); |
|
|
|
} |
|
|
|
|
|
|
|
// Sets batch size
|
|
|
|
this->timestamp = timestamp; |
|
|
|
this->debug = debug; |
|
|
|
this->batch_size = batch_size; |
|
|
|
this->batched_dims = batched_dims; |
|
|
|
this->setup2(onnx_path, session_options); |
|
|
|
} |
|
|
|
|
|
|
|
@ -46,7 +45,11 @@ namespace ofxOnnxRuntime |
|
|
|
|
|
|
|
ort_session = std::make_shared<Ort::Session>(ort_env, wpath.c_str(), session_options); |
|
|
|
|
|
|
|
setNames(); |
|
|
|
} |
|
|
|
|
|
|
|
void BaseHandler::setNames() |
|
|
|
{ |
|
|
|
Ort::AllocatorWithDefaultOptions allocator; |
|
|
|
|
|
|
|
// 1. Gets Input Name/s & Shape ([1, 3, 28, 28]) -- In most cases this is usually just one
|
|
|
|
@ -55,16 +58,18 @@ namespace ofxOnnxRuntime |
|
|
|
input_node_dims = ort_session->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); |
|
|
|
|
|
|
|
// Some models might have negative shape values to indicate dynamic shape, e.g., for variable batch size. (?, 3, 28, 28) -> (1, 3, 28, 28)
|
|
|
|
for (auto& s : input_node_dims) if (s < 0) s = 1; |
|
|
|
for (auto& s : input_node_dims) if (s < 0) s = batch_size; |
|
|
|
|
|
|
|
std::cout << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl; |
|
|
|
if (debug) std::cout << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl; |
|
|
|
} |
|
|
|
|
|
|
|
// 2. Calculate the product of the dimensions
|
|
|
|
for (auto& f : batched_dims) { |
|
|
|
for (auto& f : input_node_dims) { |
|
|
|
input_node_size *= f; |
|
|
|
} |
|
|
|
|
|
|
|
if (debug) ofLog() << ofToString(input_node_size) + ", Batch Size:" + ofToString(input_node_dims[0]); |
|
|
|
|
|
|
|
// 3. Resize input values array to match input tensor/s
|
|
|
|
input_values_handler.resize(batch_size); |
|
|
|
|
|
|
|
@ -83,16 +88,20 @@ namespace ofxOnnxRuntime |
|
|
|
|
|
|
|
output_values.emplace_back(nullptr); |
|
|
|
|
|
|
|
std::cout << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl; |
|
|
|
if (debug) std::cout << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
std::vector<Ort::Value>& BaseHandler::run() |
|
|
|
float* BaseHandler::run() |
|
|
|
{ |
|
|
|
|
|
|
|
auto start = std::chrono::high_resolution_clock::now(); // starting timestamp
|
|
|
|
|
|
|
|
std::vector<Ort::Value> input_tensors; |
|
|
|
|
|
|
|
// 1. Create 1-D array for all values to create tensor & push all values from input_vals to batch_vals
|
|
|
|
std::vector<float> batch_values(input_node_size * batch_size); |
|
|
|
std::vector<float> batch_values; |
|
|
|
batch_values.reserve(input_node_size * batch_size); // Reserve space but don't initialize
|
|
|
|
|
|
|
|
for (const auto& inner_vec : input_values_handler) { |
|
|
|
for (float value : inner_vec) { |
|
|
|
@ -122,15 +131,59 @@ namespace ofxOnnxRuntime |
|
|
|
input_names_char.size(), output_names_char.data(), |
|
|
|
output_names_char.size()); |
|
|
|
|
|
|
|
return output_values; |
|
|
|
if (debug) { |
|
|
|
// Gets the address of the first value
|
|
|
|
auto& out = output_values.front(); |
|
|
|
// Get tensor shape information
|
|
|
|
Ort::TensorTypeAndShapeInfo info = out.GetTensorTypeAndShapeInfo(); |
|
|
|
std::vector<int64_t> output_dims = info.GetShape(); |
|
|
|
|
|
|
|
// Print the dimensions
|
|
|
|
std::cout << "Output tensor dimensions: ["; |
|
|
|
for (size_t i = 0; i < output_dims.size(); i++) { |
|
|
|
std::cout << output_dims[i]; |
|
|
|
if (i < output_dims.size() - 1) { |
|
|
|
std::cout << ", "; |
|
|
|
} |
|
|
|
} |
|
|
|
std::cout << "]" << std::endl; |
|
|
|
|
|
|
|
// Optional: Print total number of elements
|
|
|
|
size_t total_elements = 1; |
|
|
|
for (auto& dim : output_dims) { |
|
|
|
if (dim > 0) { // Handle dynamic dimensions
|
|
|
|
total_elements *= static_cast<size_t>(dim); |
|
|
|
} |
|
|
|
} |
|
|
|
catch (const Ort::Exception& ex) { |
|
|
|
std::cout << "Total elements: " << total_elements << std::endl; |
|
|
|
} |
|
|
|
|
|
|
|
if (timestamp) { |
|
|
|
auto end = std::chrono::high_resolution_clock::now(); |
|
|
|
std::chrono::duration<double, std::milli> elapsed = end - start; |
|
|
|
std::cout << "Update loop took " << elapsed.count() << " ms" << std::endl; |
|
|
|
} |
|
|
|
|
|
|
|
return output_values.front().GetTensorMutableData<float>(); |
|
|
|
|
|
|
|
} catch (const Ort::Exception& ex) { |
|
|
|
std::cout << "ERROR running model inference: " << ex.what() << std::endl; |
|
|
|
return dummy_output_tensor; |
|
|
|
return dummy_output_tensor.front().GetTensorMutableData<float>(); |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
/*
|
|
|
|
* |
|
|
|
* Utilties (｡･∀･)ﾉﾞ |
|
|
|
* |
|
|
|
*/ |
|
|
|
|
|
|
|
// Fills the tensor (selected by the idx) with input values from ofFloatPixels array
|
|
|
|
void BaseHandler::setInput(ofFloatPixels &pixels, int tensor_idx, int width, int height, int channels) { |
|
|
|
pixels.setFromExternalPixels(getInputTensorData()->at(tensor_idx).data(), width, height, channels); |
|
|
|
} |
|
|
|
|
|
|
|
// Prints the shape of the given tensor (ex. input: (1, 1, 512, 512))
|
|
|
|
std::string BaseHandler::PrintShape(const std::vector<int64_t>& v) { |
|
|
|
std::stringstream ss; |
|
|
|
@ -176,9 +229,6 @@ namespace ofxOnnxRuntime |
|
|
|
} |
|
|
|
|
|
|
|
Ort::Value BaseHandler::VectorToTensor(std::vector<float>& data, const std::vector<int64_t>& shape) { |
|
|
|
//// Allocate memory using CPU memory allocator
|
|
|
|
//Ort::MemoryInfo mem_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);
|
|
|
|
|
|
|
|
// Create a tensor from the provided data, shape, and memory info
|
|
|
|
auto tensor = Ort::Value::CreateTensor<float>(memory_info_handler, data.data(), data.size(), shape.data(), shape.size()); |
|
|
|
|
|
|
|
|
