Compare commits
2 Commits
eb97a52743
...
302f2f6e97
| Author | SHA1 | Date |
|---|---|---|
 cailean
|
302f2f6e97 | 2 months ago |
|
cailean
|
091374af67 | 2 months ago |
18 changed files with 571 additions and 219 deletions
@ -0,0 +1,58 @@ |
|||||
|
{ |
||||
|
"files.associations": { |
||||
|
"xiosbase": "cpp", |
||||
|
"algorithm": "cpp", |
||||
|
"array": "cpp", |
||||
|
"atomic": "cpp", |
||||
|
"bit": "cpp", |
||||
|
"cctype": "cpp", |
||||
|
"clocale": "cpp", |
||||
|
"cmath": "cpp", |
||||
|
"compare": "cpp", |
||||
|
"concepts": "cpp", |
||||
|
"cstddef": "cpp", |
||||
|
"cstdint": "cpp", |
||||
|
"cstdio": "cpp", |
||||
|
"cstdlib": "cpp", |
||||
|
"cstring": "cpp", |
||||
|
"ctime": "cpp", |
||||
|
"cwchar": "cpp", |
||||
|
"exception": "cpp", |
||||
|
"functional": "cpp", |
||||
|
"initializer_list": "cpp", |
||||
|
"ios": "cpp", |
||||
|
"iosfwd": "cpp", |
||||
|
"iostream": "cpp", |
||||
|
"istream": "cpp", |
||||
|
"iterator": "cpp", |
||||
|
"limits": "cpp", |
||||
|
"list": "cpp", |
||||
|
"memory": "cpp", |
||||
|
"new": "cpp", |
||||
|
"numeric": "cpp", |
||||
|
"optional": "cpp", |
||||
|
"ostream": "cpp", |
||||
|
"stdexcept": "cpp", |
||||
|
"streambuf": "cpp", |
||||
|
"string": "cpp", |
||||
|
"system_error": "cpp", |
||||
|
"tuple": "cpp", |
||||
|
"type_traits": "cpp", |
||||
|
"typeinfo": "cpp", |
||||
|
"unordered_map": "cpp", |
||||
|
"unordered_set": "cpp", |
||||
|
"utility": "cpp", |
||||
|
"variant": "cpp", |
||||
|
"vector": "cpp", |
||||
|
"xfacet": "cpp", |
||||
|
"xhash": "cpp", |
||||
|
"xlocale": "cpp", |
||||
|
"xlocinfo": "cpp", |
||||
|
"xlocnum": "cpp", |
||||
|
"xmemory": "cpp", |
||||
|
"xstddef": "cpp", |
||||
|
"xstring": "cpp", |
||||
|
"xtr1common": "cpp", |
||||
|
"xutility": "cpp" |
||||
|
} |
||||
|
} |
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,297 @@ |
|||||
|
#include "ofxOnnxRuntime.h" |
||||
|
|
||||
|
namespace ofxOnnxRuntime |
||||
|
{ |
||||
|
#ifdef _MSC_VER |
||||
|
static std::wstring to_wstring(const std::string &str) |
||||
|
{ |
||||
|
unsigned len = str.size() * 2; |
||||
|
setlocale(LC_CTYPE, ""); |
||||
|
wchar_t *p = new wchar_t[len]; |
||||
|
mbstowcs(p, str.c_str(), len); |
||||
|
std::wstring wstr(p); |
||||
|
delete[] p; |
||||
|
return wstr; |
||||
|
} |
||||
|
#endif |
||||
|
void BaseHandler::setup(const std::string & onnx_path, const BaseSetting & base_setting, const int & batch_size, const bool debug, const bool timestamp) |
||||
|
{ |
||||
|
// Store data types
|
||||
|
this->input_dtype = base_setting.input_dtype; |
||||
|
this->output_dtype = base_setting.output_dtype; |
||||
|
|
||||
|
Ort::SessionOptions session_options; |
||||
|
session_options.SetIntraOpNumThreads(1); |
||||
|
session_options.SetIntraOpNumThreads(1); |
||||
|
session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); |
||||
|
|
||||
|
if (base_setting.infer_type == INFER_CUDA) { |
||||
|
OrtCUDAProviderOptions opts; |
||||
|
opts.device_id = 0; |
||||
|
opts.cudnn_conv_algo_search = OrtCudnnConvAlgoSearchExhaustive; |
||||
|
opts.do_copy_in_default_stream = 0; |
||||
|
opts.arena_extend_strategy = 0; |
||||
|
session_options.AppendExecutionProvider_CUDA(opts); |
||||
|
} |
||||
|
|
||||
|
this->timestamp = timestamp; |
||||
|
this->debug = debug; |
||||
|
this->batch_size = batch_size; |
||||
|
this->setup2(onnx_path, session_options); |
||||
|
} |
||||
|
|
||||
|
void BaseHandler::setup2(const std::string & onnx_path, const Ort::SessionOptions & session_options) |
||||
|
{ |
||||
|
std::string path = ofToDataPath(onnx_path, true); |
||||
|
|
||||
|
std::wstring wpath(path.begin(), path.end()); // basic conversion
|
||||
|
|
||||
|
ort_session = std::make_shared<Ort::Session>(ort_env, wpath.c_str(), session_options); |
||||
|
|
||||
|
setNames(); |
||||
|
} |
||||
|
|
||||
|
void BaseHandler::setNames() |
||||
|
{ |
||||
|
Ort::AllocatorWithDefaultOptions allocator; |
||||
|
|
||||
|
// 1. Gets Input Name/s & Shape ([1, 3, 28, 28]) -- In most cases this is usually just one
|
||||
|
for (std::size_t i = 0; i < ort_session->GetInputCount(); i++) { |
||||
|
input_node_names.emplace_back(ort_session->GetInputNameAllocated(i, allocator).get()); |
||||
|
input_node_dims = ort_session->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); |
||||
|
|
||||
|
// Some models might have negative shape values to indicate dynamic shape, e.g., for variable batch size. (?, 3, 28, 28) -> (1, 3, 28, 28)
|
||||
|
for (auto& s : input_node_dims) if (s < 0) s = batch_size; |
||||
|
|
||||
|
if (debug) std::cout << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl; |
||||
|
} |
||||
|
|
||||
|
// 2. Calculate the product of the dimensions
|
||||
|
for (auto& f : input_node_dims) { |
||||
|
input_node_size *= f; |
||||
|
} |
||||
|
|
||||
|
if (debug) ofLog() << ofToString(input_node_size) + ", Batch Size:" + ofToString(input_node_dims[0]); |
||||
|
|
||||
|
// 2. Clear up output values
|
||||
|
output_node_dims.clear(); |
||||
|
output_values.clear(); |
||||
|
|
||||
|
// 3. Gets Output name/s & Shapes
|
||||
|
for (std::size_t i = 0; i < ort_session->GetOutputCount(); i++) { |
||||
|
output_node_names.emplace_back(ort_session->GetOutputNameAllocated(i, allocator).get()); |
||||
|
auto output_shapes = ort_session->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); |
||||
|
|
||||
|
output_values.emplace_back(nullptr); |
||||
|
|
||||
|
if (debug) std::cout << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
float* BaseHandler::run() |
||||
|
{ |
||||
|
|
||||
|
auto start = std::chrono::high_resolution_clock::now(); // starting timestamp
|
||||
|
|
||||
|
std::vector<Ort::Value> input_tensors; |
||||
|
|
||||
|
size_t num_images = input_imgs.size(); |
||||
|
|
||||
|
if(input_imgs.size() != batch_size) { |
||||
|
ofLog() << "Input images do not match batch size. Inference FAILED."; |
||||
|
return dummy_output_tensor.front().GetTensorMutableData<float>(); |
||||
|
} |
||||
|
|
||||
|
// transform std::string -> const char*
|
||||
|
std::vector<const char*> input_names_char(input_node_names.size(), nullptr); |
||||
|
std::transform(std::begin(input_node_names), std::end(input_node_names), std::begin(input_names_char), |
||||
|
[&](const std::string& str) { return str.c_str(); }); |
||||
|
|
||||
|
std::vector<const char*> output_names_char(output_node_names.size(), nullptr); |
||||
|
std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char), |
||||
|
[&](const std::string& str) { return str.c_str(); }); |
||||
|
|
||||
|
std::vector<float> batch_values_f; |
||||
|
std::vector<int32_t> batch_values_int32; |
||||
|
batch_values_f.reserve(input_node_size * batch_size); // Reserve space but don't initialize
|
||||
|
batch_values_int32.reserve(input_node_size * batch_size); // Reserve space but don't initialize
|
||||
|
|
||||
|
|
||||
|
if (input_dtype == ModelDataType::FLOAT32){ |
||||
|
// I have a list of imgs, these need to be converted from images into input for the model (int or float)
|
||||
|
for(size_t i = 0; i < batch_size; i++) { |
||||
|
convertImageToMatFloat(input_imgs[i], batch_values_f, i); |
||||
|
} |
||||
|
|
||||
|
// 2. Create tensor with batch values { input data, input size, model input dims, model input size}
|
||||
|
input_tensors.emplace_back(Ort::Value::CreateTensor<float>( |
||||
|
memory_info_handler, batch_values_f.data(), input_node_size, |
||||
|
input_node_dims.data(), input_node_dims.size())); |
||||
|
} |
||||
|
else if (input_dtype == ModelDataType::INT32) { |
||||
|
// I have a list of imgs, these need to be converted from images into input for the model (int or float)
|
||||
|
for(size_t i = 0; i < batch_size; i++) { |
||||
|
convertImageToMatInt32(input_imgs[i], batch_values_int32, i); |
||||
|
} |
||||
|
|
||||
|
// 2. Create tensor with batch values { input data, input size, model input dims, model input size}
|
||||
|
input_tensors.emplace_back(Ort::Value::CreateTensor<int32_t>( |
||||
|
memory_info_handler, batch_values_int32.data(), input_node_size, |
||||
|
input_node_dims.data(), input_node_dims.size())); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
|
||||
|
|
||||
|
try { |
||||
|
// 3. Run inference, { in names, input data, num of inputs, output names, num of outputs }
|
||||
|
ofLog() << "run"; |
||||
|
output_values = ort_session->Run(Ort::RunOptions{ nullptr }, |
||||
|
input_names_char.data(), input_tensors.data(), |
||||
|
input_names_char.size(), output_names_char.data(), |
||||
|
output_names_char.size()); |
||||
|
ofLog() << "ran"; |
||||
|
|
||||
|
if (debug) { |
||||
|
// Gets the address of the first value
|
||||
|
auto& out = output_values.front(); |
||||
|
// Get tensor shape information
|
||||
|
Ort::TensorTypeAndShapeInfo info = out.GetTensorTypeAndShapeInfo(); |
||||
|
std::vector<int64_t> output_dims = info.GetShape(); |
||||
|
|
||||
|
// Print the dimensions
|
||||
|
std::cout << "Output tensor dimensions: ["; |
||||
|
for (size_t i = 0; i < output_dims.size(); i++) { |
||||
|
std::cout << output_dims[i]; |
||||
|
if (i < output_dims.size() - 1) { |
||||
|
std::cout << ", "; |
||||
|
} |
||||
|
} |
||||
|
std::cout << "]" << std::endl; |
||||
|
|
||||
|
// Optional: Print total number of elements
|
||||
|
size_t total_elements = 1; |
||||
|
for (auto& dim : output_dims) { |
||||
|
if (dim > 0) { // Handle dynamic dimensions
|
||||
|
total_elements *= static_cast<size_t>(dim); |
||||
|
} |
||||
|
} |
||||
|
std::cout << "Total elements: " << total_elements << std::endl; |
||||
|
} |
||||
|
|
||||
|
// if (timestamp) {
|
||||
|
// auto end = std::chrono::high_resolution_clock::now();
|
||||
|
// std::chrono::duration<double, std::milli> elapsed = end - start;
|
||||
|
// std::cout << "Update loop took " << elapsed.count() << " ms" << std::endl;
|
||||
|
// }
|
||||
|
|
||||
|
return output_values.front().GetTensorMutableData<float>(); |
||||
|
|
||||
|
} catch (const Ort::Exception& ex) { |
||||
|
std::cout << "ERROR running model inference: " << ex.what() << std::endl; |
||||
|
return dummy_output_tensor.front().GetTensorMutableData<float>(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
/*
|
||||
|
* |
||||
|
* Utilties (｡･∀･)ﾉﾞ |
||||
|
* |
||||
|
*/ |
||||
|
|
||||
|
// Add separate methods for float and int32 conversion
|
||||
|
void BaseHandler::convertImageToMatFloat(ofImage* img, std::vector<float>& values, size_t& idx) { |
||||
|
// Your existing conversion code for float
|
||||
|
ofPixels& pix = img->getPixels(); |
||||
|
int width = img->getWidth(); |
||||
|
int height = img->getHeight(); |
||||
|
int channels = pix.getNumChannels(); |
||||
|
|
||||
|
cv::Mat cvImage = cv::Mat(height, width, (channels == 3) ? CV_8UC3 : CV_8UC1, pix.getData()); |
||||
|
cv::InputArray inputArray(cvImage); |
||||
|
image_array = cv::dnn::blobFromImage(inputArray, 1 / 255.0, cv::Size(input_node_dims[2], input_node_dims[3]), (0, 0, 0), false, false); |
||||
|
|
||||
|
std::memcpy( |
||||
|
values.data() + idx * channels * width * height, |
||||
|
image_array.data, |
||||
|
channels * width * height * sizeof(float) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
void BaseHandler::convertImageToMatInt32(ofImage* img, std::vector<int32_t>& values, size_t& idx) { |
||||
|
// New conversion code for int32
|
||||
|
ofPixels& pix = img->getPixels(); |
||||
|
int width = img->getWidth(); |
||||
|
int height = img->getHeight(); |
||||
|
int channels = pix.getNumChannels(); |
||||
|
|
||||
|
cv::Mat cvImage = cv::Mat(height, width, (channels == 3) ? CV_8UC3 : CV_8UC1, pix.getData()); |
||||
|
cv::InputArray inputArray(cvImage); |
||||
|
cv::Mat intMat = cv::dnn::blobFromImage(inputArray, 1 / 255.0, cv::Size(height, width), (0, 0, 0), false, false); |
||||
|
|
||||
|
intMat.convertTo(image_array, CV_32S); |
||||
|
|
||||
|
std::memcpy( |
||||
|
values.data() + idx * channels * width * height, |
||||
|
image_array.data, |
||||
|
channels * width * height * sizeof(int32_t) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
void BaseHandler::setInputs(std::vector<ofImage*>& in) { |
||||
|
this->input_imgs = in; |
||||
|
} |
||||
|
|
||||
|
// Prints the shape of the given tensor (ex. input: (1, 1, 512, 512))
|
||||
|
std::string BaseHandler::PrintShape(const std::vector<int64_t>& v) { |
||||
|
std::stringstream ss; |
||||
|
for (std::size_t i = 0; i < v.size() - 1; i++) ss << v[i] << "x"; |
||||
|
ss << v[v.size() - 1]; |
||||
|
return ss.str(); |
||||
|
} |
||||
|
|
||||
|
Ort::Value BaseHandler::GenerateTensor(int batch_size) { |
||||
|
// Random number generation setup
|
||||
|
std::random_device rd; |
||||
|
std::mt19937 gen(rd()); |
||||
|
std::uniform_real_distribution<float> dis(0.0f, 255.0f); // Random values between 0 and 255
|
||||
|
|
||||
|
// Calculate the total number of elements for a single tensor (without batch dimension) {?, 8} -> 8
|
||||
|
int tensor_size = CalculateProduct(input_node_dims); |
||||
|
|
||||
|
// Create a vector to hold all the values for the batch (8 * (4)batch_size) -> 32
|
||||
|
std::vector<float> batch_values(batch_size * tensor_size); |
||||
|
|
||||
|
// Fill the batch with random values
|
||||
|
std::generate(batch_values.begin(), batch_values.end(), [&]() { |
||||
|
return dis(gen); |
||||
|
}); |
||||
|
|
||||
|
// Fill the batch with random values
|
||||
|
std::generate(batch_values.begin(), batch_values.end(), [&]() { |
||||
|
return dis(gen); |
||||
|
}); |
||||
|
|
||||
|
// Create the batched dimensions by inserting the batch size at the beginning of the original dimensions
|
||||
|
std::vector<int64_t> batched_dims = { }; // Start with batch size
|
||||
|
batched_dims.insert(batched_dims.end(), input_node_dims.begin(), input_node_dims.end()); // Add the remaining dimensions
|
||||
|
batched_dims[0] = batch_size; |
||||
|
|
||||
|
return VectorToTensor(batch_values, batched_dims); |
||||
|
} |
||||
|
|
||||
|
int BaseHandler::CalculateProduct(const std::vector<int64_t>& v) { |
||||
|
int total = 1; |
||||
|
for (auto& i : v) total *= i; |
||||
|
return total; |
||||
|
} |
||||
|
|
||||
|
Ort::Value BaseHandler::VectorToTensor(std::vector<float>& data, const std::vector<int64_t>& shape) { |
||||
|
// Create a tensor from the provided data, shape, and memory info
|
||||
|
auto tensor = Ort::Value::CreateTensor<float>(memory_info_handler, data.data(), data.size(), shape.data(), shape.size()); |
||||
|
|
||||
|
// Return the created tensor
|
||||
|
return tensor; |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue