Browse Source

working

mingw
cailean 2 weeks ago
parent
commit
091374af67
  1. 47
      libs/onnxruntime/include/onnxruntime_c_api.h
  2. 41
      libs/onnxruntime/include/onnxruntime_cxx_api.h
  3. 4
      libs/onnxruntime/include/onnxruntime_cxx_inline.h
  4. 55
      libs/onnxruntime/include/onnxruntime_session_options_config_keys.h
  5. BIN
      libs/onnxruntime/lib/msys2/onnxruntime.dll
  6. BIN
      libs/onnxruntime/lib/msys2/onnxruntime.lib
  7. BIN
      libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.dll
  8. BIN
      libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.lib
  9. BIN
      libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.dll
  10. BIN
      libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.lib
  11. BIN
      libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.dll
  12. BIN
      libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.lib
  13. 84
      src/ofxOnnxRuntime.cpp
  14. 66
      src/ofxOnnxRuntime.h

47
libs/onnxruntime/include/onnxruntime_c_api.h

@ -38,7 +38,7 @@
*
* This value is used by some API functions to behave as this version of the header expects.
*/
#define ORT_API_VERSION 21
#define ORT_API_VERSION 20
#ifdef __cplusplus
extern "C" {
@ -46,7 +46,7 @@ extern "C" {
//! @}
// SAL2 Definitions
#ifndef _MSC_VER
#ifndef _WIN32
#define _In_
#define _In_z_
#define _In_opt_
@ -626,13 +626,8 @@ typedef struct OrtMIGraphXProviderOptions {
} OrtMIGraphXProviderOptions;
/** \brief OpenVINO Provider Options
* \brief This Struct is frozen since ORT 1.13.0. Its maintained part of Legacy API for compatibility.
* \brief For latest OpenVINO Provider Options update to the ProviderOptions map.
* \brief Latest OpenVINO Provider Options are listed in the
* \htmlonly
* <a href="https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options">onnxruntime document.</a>
* \endhtmlonly
* \see OrtApi::SessionOptionsAppendExecutionProvider()
*
* \see OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO
*/
typedef struct OrtOpenVINOProviderOptions {
#ifdef __cplusplus
@ -650,7 +645,7 @@ typedef struct OrtOpenVINOProviderOptions {
* Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16"
*/
const char* device_type;
unsigned char enable_npu_fast_compile; ///< 0 = disabled, nonzero = enabled
unsigned char enable_npu_fast_compile;
const char* device_id;
size_t num_of_threads; ///< 0 = Use default number of threads
const char* cache_dir; // path is set to empty by default
@ -3665,19 +3660,8 @@ struct OrtApi {
* - "1": Enabled.
* "offload_graph_io_quantization": Offload graph input quantization and graph output dequantization to another
* execution provider (typically CPU EP).
* - "0": Disabled. QNN EP will handle quantization and dequantization of graph I/O.
* - "1": Enabled. This is the default value.
* "enable_htp_spill_fill_buffer": Enable HTP spill fill buffer setting. The flag is used while generating context binary.
* - "0": Default. Disabled.
* - "1": Enabled.
* "enable_htp_shared_memory_allocator": Enable the QNN HTP shared memory allocator. Requires libcdsprpc.so/dll to
* be available.
* - "0": Default. Disabled.
* - "0": Default. Disabled. QNN EP will handle quantization and dequantization of graph I/O.
* - "1": Enabled.
* "dump_json_qnn_graph": Set to "1" to dump QNN graphs generated by QNN EP as JSON files. Each graph partition
* assigned to QNN EP is dumped to a separate file.
* "json_qnn_graph_dir": Directory in which to dump QNN JSON graphs. If not specified, QNN graphs are dumped in the
* program's current working directory. Ignored if "dump_json_qnn_graph" is not set.
*
* SNPE supported keys:
* "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16",
@ -4623,8 +4607,6 @@ struct OrtApi {
* \param[in] num_keys
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.17.
*/
ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO_V2,
_In_ OrtSessionOptions* options,
@ -4642,8 +4624,6 @@ struct OrtApi {
* \param[in] num_keys
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.18.
*/
ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_VitisAI,
_In_ OrtSessionOptions* options,
@ -4657,10 +4637,7 @@ struct OrtApi {
* \param[in] mem_info OrtMemoryInfo instance
* \param[in] count_or_bytes How many bytes is this scratch buffer
* \param[out] out A pointer to the scrach buffer
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.18.
*/
ORT_API2_STATUS(KernelContext_GetScratchBuffer, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _In_ size_t count_or_bytes, _Outptr_ void** out);
@ -4671,8 +4648,6 @@ struct OrtApi {
* \param[out] out A pointer to OrtAllocator
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.18.
*/
ORT_API2_STATUS(KernelInfoGetAllocator, _In_ const OrtKernelInfo* info, _In_ OrtMemType mem_type, _Outptr_ OrtAllocator** out);
@ -4694,8 +4669,6 @@ struct OrtApi {
* \param[in] num_external_initializer_files Number of external files
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.18.
*/
ORT_API2_STATUS(AddExternalInitializersFromFilesInMemory, _In_ OrtSessionOptions* options,
_In_reads_(num_external_initializer_files) const ORTCHAR_T* const* external_initializer_file_names,
@ -4718,8 +4691,6 @@ struct OrtApi {
* OrtApi::ReleaseLoraAdapter.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.20.
*/
ORT_API2_STATUS(CreateLoraAdapter, const ORTCHAR_T* adapter_file_path, _In_ OrtAllocator* allocator,
_Outptr_ OrtLoraAdapter** out);
@ -4738,8 +4709,6 @@ struct OrtApi {
* OrtApi::ReleaseLoraAdapter.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.20.
*/
ORT_API2_STATUS(CreateLoraAdapterFromArray, _In_ const void* bytes, size_t num_bytes, _In_ OrtAllocator* allocator,
_Outptr_ OrtLoraAdapter** out);
@ -4761,8 +4730,6 @@ struct OrtApi {
* \param[in] adapter OrtLoraAdapter instance
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.20.
*/
ORT_API2_STATUS(RunOptionsAddActiveLoraAdapter, _Inout_ OrtRunOptions* options, _In_ const OrtLoraAdapter* adapter);
@ -4781,8 +4748,6 @@ struct OrtApi {
* \param[in] kv_len Number of elements in the keys and values arrays
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.20.
*/
ORT_API2_STATUS(SetEpDynamicOptions, _Inout_ OrtSession* sess, _In_reads_(kv_len) const char* const* keys,
_In_reads_(kv_len) const char* const* values, _In_ size_t kv_len);

41
libs/onnxruntime/include/onnxruntime_cxx_api.h

@ -650,9 +650,6 @@ using AllocatedStringPtr = std::unique_ptr<char, detail::AllocatedFree>;
* constructors to construct an instance of a Status object from exceptions.
*/
struct Status : detail::Base<OrtStatus> {
using Base = detail::Base<OrtStatus>;
using Base::Base;
explicit Status(std::nullptr_t) noexcept {} ///< Create an empty object, must be assigned a valid one to be used
explicit Status(OrtStatus* status) noexcept; ///< Takes ownership of OrtStatus instance returned from the C API.
explicit Status(const Exception&) noexcept; ///< Creates status instance out of exception
@ -731,9 +728,6 @@ struct Env : detail::Base<OrtEnv> {
*
*/
struct CustomOpDomain : detail::Base<OrtCustomOpDomain> {
using Base = detail::Base<OrtCustomOpDomain>;
using Base::Base;
explicit CustomOpDomain(std::nullptr_t) {} ///< Create an empty CustomOpDomain object, must be assigned a valid one to be used
/// \brief Wraps OrtApi::CreateCustomOpDomain
@ -969,10 +963,8 @@ struct SessionOptions : detail::SessionOptionsImpl<OrtSessionOptions> {
*
*/
struct ModelMetadata : detail::Base<OrtModelMetadata> {
using Base = detail::Base<OrtModelMetadata>;
using Base::Base;
explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used
explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used
explicit ModelMetadata(OrtModelMetadata* p) : Base<OrtModelMetadata>{p} {} ///< Used for interop with the C API
/** \brief Returns a copy of the producer name.
*
@ -1245,9 +1237,6 @@ using ConstTensorTypeAndShapeInfo = detail::TensorTypeAndShapeInfoImpl<detail::U
*
*/
struct TensorTypeAndShapeInfo : detail::TensorTypeAndShapeInfoImpl<OrtTensorTypeAndShapeInfo> {
using Base = detail::TensorTypeAndShapeInfoImpl<OrtTensorTypeAndShapeInfo>;
using Base::Base;
explicit TensorTypeAndShapeInfo(std::nullptr_t) {} ///< Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used
explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {} ///< Used for interop with the C API
ConstTensorTypeAndShapeInfo GetConst() const { return ConstTensorTypeAndShapeInfo{this->p_}; }
@ -1269,9 +1258,6 @@ using ConstSequenceTypeInfo = detail::SequenceTypeInfoImpl<detail::Unowned<const
*
*/
struct SequenceTypeInfo : detail::SequenceTypeInfoImpl<OrtSequenceTypeInfo> {
using Base = detail::SequenceTypeInfoImpl<OrtSequenceTypeInfo>;
using Base::Base;
explicit SequenceTypeInfo(std::nullptr_t) {} ///< Create an empty SequenceTypeInfo object, must be assigned a valid one to be used
explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : SequenceTypeInfoImpl<OrtSequenceTypeInfo>{p} {} ///< Used for interop with the C API
ConstSequenceTypeInfo GetConst() const { return ConstSequenceTypeInfo{this->p_}; }
@ -1307,9 +1293,6 @@ using ConstMapTypeInfo = detail::MapTypeInfoImpl<detail::Unowned<const OrtMapTyp
*
*/
struct MapTypeInfo : detail::MapTypeInfoImpl<OrtMapTypeInfo> {
using Base = detail::MapTypeInfoImpl<OrtMapTypeInfo>;
using Base::Base;
explicit MapTypeInfo(std::nullptr_t) {} ///< Create an empty MapTypeInfo object, must be assigned a valid one to be used
explicit MapTypeInfo(OrtMapTypeInfo* p) : MapTypeInfoImpl<OrtMapTypeInfo>{p} {} ///< Used for interop with the C API
ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; }
@ -1341,9 +1324,6 @@ using ConstTypeInfo = detail::TypeInfoImpl<detail::Unowned<const OrtTypeInfo>>;
/// the information about contained sequence or map depending on the ONNXType.
/// </summary>
struct TypeInfo : detail::TypeInfoImpl<OrtTypeInfo> {
using Base = detail::TypeInfoImpl<OrtTypeInfo>;
using Base::Base;
explicit TypeInfo(std::nullptr_t) {} ///< Create an empty TypeInfo object, must be assigned a valid one to be used
explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl<OrtTypeInfo>{p} {} ///< C API Interop
@ -1681,11 +1661,11 @@ using UnownedValue = detail::ValueImpl<detail::Unowned<OrtValue>>;
*/
struct Value : detail::ValueImpl<OrtValue> {
using Base = detail::ValueImpl<OrtValue>;
using Base::Base;
using OrtSparseValuesParam = detail::OrtSparseValuesParam;
using Shape = detail::Shape;
explicit Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used
explicit Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used
explicit Value(OrtValue* p) : Base{p} {} ///< Used for interop with the C API
Value(Value&&) = default;
Value& operator=(Value&&) = default;
@ -1961,10 +1941,6 @@ struct ArenaCfg : detail::Base<OrtArenaCfg> {
/// This struct provides life time management for custom op attribute
/// </summary>
struct OpAttr : detail::Base<OrtOpAttr> {
using Base = detail::Base<OrtOpAttr>;
using Base::Base;
explicit OpAttr(std::nullptr_t) {}
OpAttr(const char* name, const void* data, int len, OrtOpAttrType type);
};
@ -2130,10 +2106,10 @@ struct KernelContext {
explicit KernelContext(OrtKernelContext* context);
size_t GetInputCount() const;
size_t GetOutputCount() const;
// If input is optional and is not present, the method returns an empty ConstValue
// If input is optional and is not present, the method returns en empty ConstValue
// which can be compared to nullptr.
ConstValue GetInput(size_t index) const;
// If output is optional and is not present, the method returns an empty UnownedValue
// If outout is optional and is not present, the method returns en empty UnownedValue
// which can be compared to nullptr.
UnownedValue GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const;
UnownedValue GetOutput(size_t index, const std::vector<int64_t>& dims) const;
@ -2207,8 +2183,6 @@ using ConstKernelInfo = detail::KernelInfoImpl<detail::Unowned<const OrtKernelIn
/// so it does not destroy the pointer the kernel does not own.
/// </summary>
struct KernelInfo : detail::KernelInfoImpl<OrtKernelInfo> {
using Base = detail::KernelInfoImpl<OrtKernelInfo>;
using Base::Base;
explicit KernelInfo(std::nullptr_t) {} ///< Create an empty instance to initialize later
explicit KernelInfo(OrtKernelInfo* info); ///< Take ownership of the instance
ConstKernelInfo GetConst() const { return ConstKernelInfo{this->p_}; }
@ -2218,9 +2192,6 @@ struct KernelInfo : detail::KernelInfoImpl<OrtKernelInfo> {
/// Create and own custom defined operation.
/// </summary>
struct Op : detail::Base<OrtOp> {
using Base = detail::Base<OrtOp>;
using Base::Base;
explicit Op(std::nullptr_t) {} ///< Create an empty Operator object, must be assigned a valid one to be used
explicit Op(OrtOp*); ///< Take ownership of the OrtOp

4
libs/onnxruntime/include/onnxruntime_cxx_inline.h

@ -51,7 +51,7 @@ inline void ThrowOnError(const Status& st) {
}
}
inline Status::Status(OrtStatus* status) noexcept : detail::Base<OrtStatus>{status} {
inline Status::Status(OrtStatus* status) noexcept : Base<OrtStatus>{status} {
}
inline Status::Status(const std::exception& e) noexcept {
@ -1908,7 +1908,7 @@ inline void attr_utils::GetAttrs(const OrtKernelInfo* p, const char* name, std::
inline KernelInfo::KernelInfo(OrtKernelInfo* info) : detail::KernelInfoImpl<OrtKernelInfo>{info} {}
inline Op::Op(OrtOp* p) : detail::Base<OrtOp>(p) {}
inline Op::Op(OrtOp* p) : Base<OrtOp>(p) {}
inline Op Op::Create(const OrtKernelInfo* info, const char* op_name, const char* domain, int version,
const char** type_constraint_names,

55
libs/onnxruntime/include/onnxruntime_session_options_config_keys.h

@ -250,51 +250,6 @@ static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersFil
static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersMinSizeInBytes =
"session.optimized_model_external_initializers_min_size_in_bytes";
// When loading model from memory buffer and the model has external initializers
// Use this config to set the external data file folder path
// All external data files should be in the same folder
static const char* const kOrtSessionOptionsModelExternalInitializersFileFolderPath =
"session.model_external_initializers_file_folder_path";
// Use this config when saving pre-packed constant initializers to an external data file.
// This allows you to memory map pre-packed initializers on model load and leave it to
// to the OS the amount of memory consumed by the pre-packed initializers. Otherwise,
// pre-packed data resides on the heap.
//
// - "0": Default is not save pre-packed initializers to a data file.
// - "1": Save pre-packed constant initializers to an external data file.
// Sample usage: sess_options.add_session_config_entry(kOrtSessionOptionsSavePrePackedConstantInitializers, "1")
static const char* const kOrtSessionOptionsSavePrePackedConstantInitializers =
"session.save_external_prepacked_constant_initializers";
// Use this config when you want to collect memory stats for each node in the graph.
// The file format is a CSV file with the following columns:
// The file will be created if it does not exist, and will be overwritten if it does.
//
// The content of the file can be used to estimate memory requirements at run time including
// the temporary allocations. This operation is preferably done on a CPU device, as the model may exceed
// device memory limits in constrained environments. When enabling this option, it is important to disable
// memory patterns, as they tend to allocate large blocks to avoid fragmentation and accommodate needs of multiple
// kernels. Memory patterns may make it difficult to allocate on a device with limited memory.
//
// The collected stats then can be used to partition the graph among the devices in a way that only the
// required memory is allocated on each device.
//
// node_name, initializers_memory, dynamic_outputs_sizes, temp_allocations_size
//
// - "full path to file": there is not a default for this option. If the file can not be opened for writing, an error will be returned.
static const char* const kOrtSessionOptionsCollectNodeMemoryStatsToFile = "session.collect_node_memory_stats_to_file";
/// This is a composite CSV setting formatted as "memory limit in kb,file name for collected stats"
/// "limit > 0": enables Capacity Aware Partitioning for Cuda EP. `limit` is optional and when absent
/// the provider may attempt to figure out the memory available automatically.
/// The setting with no limit is expected to look like: ",file name for collected stats"
/// The EP will place nodes on device "file name" :
/// this file is expected to be found at the same folder with the model. The file contains
/// pre-recorded stats collected when running with kOrtSessionOptionsCollectNodeMemoryStatsToFile enforce (see above)
static const char* const kOrtSessionOptionsResourceCudaPartitioningSettings =
"session.resource_cuda_partitioning_settings";
// Enable EP context feature to dump the partitioned graph which includes the EP context into Onnx file.
// The dumped Onnx model with EP context can be used for future inference to avoid the EP graph partitioning/compile overhead.
// "0": disable. (default)
@ -303,12 +258,11 @@ static const char* const kOrtSessionOptionEpContextEnable = "ep.context_enable";
// Specify the file path for the Onnx model which has EP context.
// Default to original_file_name_ctx.onnx if not specified
// Folder is not a valid option
static const char* const kOrtSessionOptionEpContextFilePath = "ep.context_file_path";
// Flag to specify whether to dump the EP context into the Onnx model.
// "0": dump the EP context into separate file, keep the file name in the Onnx model. (default).
// "1": dump the EP context into the Onnx model.
// "0": dump the EP context into separate file, keep the file name in the Onnx model.
// "1": dump the EP context into the Onnx model. (default).
static const char* const kOrtSessionOptionEpContextEmbedMode = "ep.context_embed_mode";
// Specify the EPContext node name prefix to make it unique
@ -318,11 +272,6 @@ static const char* const kOrtSessionOptionEpContextNodeNamePrefix = "ep.context_
// Share EP related resources across EPs
static const char* const kOrtSessionOptionShareEpContexts = "ep.share_ep_contexts";
// Use this config when dumping EP context model with an external initializers file
// All initializers will be inside the external data file if specified, otherwise all in Onnx file
static const char* const kOrtSessionOptionsEpContextModelExternalInitializersFileName =
"ep.context_model_external_initializers_file_name";
// Gemm fastmath mode provides fp32 gemm acceleration with bfloat16 based matmul.
// Option values:
// - "0": Gemm FastMath mode is not enabled. [DEFAULT]

BIN
libs/onnxruntime/lib/msys2/onnxruntime.dll (Stored with Git LFS)

Binary file not shown.

BIN
libs/onnxruntime/lib/msys2/onnxruntime.lib (Stored with Git LFS)

Binary file not shown.

BIN
libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.dll (Stored with Git LFS)

Binary file not shown.

BIN
libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.lib (Stored with Git LFS)

Binary file not shown.

BIN
libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.dll (Stored with Git LFS)

Binary file not shown.

BIN
libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.lib (Stored with Git LFS)

Binary file not shown.

BIN
libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.dll (Stored with Git LFS)

Binary file not shown.

BIN
libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.lib (Stored with Git LFS)

Binary file not shown.

84
src/ofxOnnxRuntime.cpp

@ -1,5 +1,4 @@
#include "ofxOnnxRuntime.h"
#include "ofMain.h"
namespace ofxOnnxRuntime
{
@ -16,7 +15,7 @@ namespace ofxOnnxRuntime
}
#endif
void BaseHandler::setup(const std::string & onnx_path, const BaseSetting & base_setting, const std::vector<int64_t>& batched_dims, const int & batch_size)
void BaseHandler::setup(const std::string & onnx_path, const BaseSetting & base_setting, const int & batch_size, const bool debug, const bool timestamp)
{
Ort::SessionOptions session_options;
session_options.SetIntraOpNumThreads(1);
@ -32,9 +31,9 @@ namespace ofxOnnxRuntime
session_options.AppendExecutionProvider_CUDA(opts);
}
// Sets batch size
this->timestamp = timestamp;
this->debug = debug;
this->batch_size = batch_size;
this->batched_dims = batched_dims;
this->setup2(onnx_path, session_options);
}
@ -46,7 +45,11 @@ namespace ofxOnnxRuntime
ort_session = std::make_shared<Ort::Session>(ort_env, wpath.c_str(), session_options);
setNames();
}
void BaseHandler::setNames()
{
Ort::AllocatorWithDefaultOptions allocator;
// 1. Gets Input Name/s & Shape ([1, 3, 28, 28]) -- In most cases this is usually just one
@ -55,16 +58,18 @@ namespace ofxOnnxRuntime
input_node_dims = ort_session->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();
// Some models might have negative shape values to indicate dynamic shape, e.g., for variable batch size. (?, 3, 28, 28) -> (1, 3, 28, 28)
for (auto& s : input_node_dims) if (s < 0) s = 1;
for (auto& s : input_node_dims) if (s < 0) s = batch_size;
std::cout << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl;
if (debug) std::cout << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl;
}
// 2. Calculate the product of the dimensions
for (auto& f : batched_dims) {
for (auto& f : input_node_dims) {
input_node_size *= f;
}
if (debug) ofLog() << ofToString(input_node_size) + ", Batch Size:" + ofToString(input_node_dims[0]);
// 3. Resize input values array to match input tensor/s
input_values_handler.resize(batch_size);
@ -83,16 +88,20 @@ namespace ofxOnnxRuntime
output_values.emplace_back(nullptr);
std::cout << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl;
if (debug) std::cout << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl;
}
}
std::vector<Ort::Value>& BaseHandler::run()
float* BaseHandler::run()
{
auto start = std::chrono::high_resolution_clock::now(); // starting timestamp
std::vector<Ort::Value> input_tensors;
// 1. Create 1-D array for all values to create tensor & push all values from input_vals to batch_vals
std::vector<float> batch_values(input_node_size * batch_size);
std::vector<float> batch_values;
batch_values.reserve(input_node_size * batch_size); // Reserve space but don't initialize
for (const auto& inner_vec : input_values_handler) {
for (float value : inner_vec) {
@ -121,16 +130,60 @@ namespace ofxOnnxRuntime
input_names_char.data(), input_tensors.data(),
input_names_char.size(), output_names_char.data(),
output_names_char.size());
if (debug) {
// Gets the address of the first value
auto& out = output_values.front();
// Get tensor shape information
Ort::TensorTypeAndShapeInfo info = out.GetTensorTypeAndShapeInfo();
std::vector<int64_t> output_dims = info.GetShape();
// Print the dimensions
std::cout << "Output tensor dimensions: [";
for (size_t i = 0; i < output_dims.size(); i++) {
std::cout << output_dims[i];
if (i < output_dims.size() - 1) {
std::cout << ", ";
}
}
std::cout << "]" << std::endl;
// Optional: Print total number of elements
size_t total_elements = 1;
for (auto& dim : output_dims) {
if (dim > 0) { // Handle dynamic dimensions
total_elements *= static_cast<size_t>(dim);
}
}
std::cout << "Total elements: " << total_elements << std::endl;
}
if (timestamp) {
auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> elapsed = end - start;
std::cout << "Update loop took " << elapsed.count() << " ms" << std::endl;
}
return output_values;
}
catch (const Ort::Exception& ex) {
return output_values.front().GetTensorMutableData<float>();
} catch (const Ort::Exception& ex) {
std::cout << "ERROR running model inference: " << ex.what() << std::endl;
return dummy_output_tensor;
return dummy_output_tensor.front().GetTensorMutableData<float>();
}
}
/*
*
* Utilties ()
*
*/
// Fills the tensor (selected by the idx) with input values from ofFloatPixels array
void BaseHandler::setInput(ofFloatPixels &pixels, int tensor_idx, int width, int height, int channels) {
pixels.setFromExternalPixels(getInputTensorData()->at(tensor_idx).data(), width, height, channels);
}
// Prints the shape of the given tensor (ex. input: (1, 1, 512, 512))
std::string BaseHandler::PrintShape(const std::vector<int64_t>& v) {
std::stringstream ss;
@ -176,9 +229,6 @@ namespace ofxOnnxRuntime
}
Ort::Value BaseHandler::VectorToTensor(std::vector<float>& data, const std::vector<int64_t>& shape) {
//// Allocate memory using CPU memory allocator
//Ort::MemoryInfo mem_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);
// Create a tensor from the provided data, shape, and memory info
auto tensor = Ort::Value::CreateTensor<float>(memory_info_handler, data.data(), data.size(), shape.data(), shape.size());

66
src/ofxOnnxRuntime.h

@ -1,6 +1,7 @@
#pragma once
#include <onnxruntime_cxx_api.h>
#include "ofMain.h"
namespace ofxOnnxRuntime
{
@ -19,45 +20,48 @@ namespace ofxOnnxRuntime
class BaseHandler
{
public:
BaseHandler() {}
public:
BaseHandler() {}
void setup(const std::string& onnx_path, const BaseSetting& base_setting = BaseSetting{ INFER_CPU, 0 }, const std::vector<int64_t>& batched_dims = {}, const int& batch_size = 1);
void setup2(const std::string& onnx_path, const Ort::SessionOptions& session_options);
void setup(const std::string& onnx_path, const BaseSetting& base_setting = BaseSetting{ INFER_CPU, 0 }, const int& batch_size = 1, const bool debug = false, const bool timestamp = false);
void setup2(const std::string& onnx_path, const Ort::SessionOptions& session_options);
void setNames();
void setInput(ofFloatPixels &pixels, int tensor_idx, int width, int height, int channels);
float* run();
std::vector<std::vector<float>>* getInputTensorData() {
return &this->input_values_handler;
}
// Utilities ╰(‵□′)╯
std::string PrintShape(const std::vector<int64_t>& v);
Ort::Value GenerateTensor(int batch_size);
int CalculateProduct(const std::vector<int64_t>& v);
Ort::Value VectorToTensor(std::vector<float>& data, const std::vector<int64_t>& shape);
std::vector<Ort::Value>& run();
std::vector<std::vector<float>>* getInputTensorData() {
return &this->input_values_handler;
}
protected:
bool debug = false;
bool timestamp = false;
Ort::Env ort_env;
std::shared_ptr<Ort::Session> ort_session;
// Utilities
std::string PrintShape(const std::vector<int64_t>& v);
Ort::Value GenerateTensor(int batch_size);
int CalculateProduct(const std::vector<int64_t>& v);
Ort::Value VectorToTensor(std::vector<float>& data, const std::vector<int64_t>& shape);
std::vector<std::string> input_node_names;
std::vector<int64_t> input_node_dims; // 1 input only.
protected:
Ort::Env ort_env;
std::shared_ptr<Ort::Session> ort_session;
Ort::MemoryInfo memory_info_handler = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
std::vector<std::string> input_node_names;
std::vector<int64_t> input_node_dims; // 1 input only.
std::vector<std::string> output_node_names;
std::vector<std::vector<int64_t>> output_node_dims; // >=1 outputs
std::vector<Ort::Value> output_values;
Ort::MemoryInfo memory_info_handler = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
Ort::Value dummy_tensor{ nullptr };
std::vector<Ort::Value> dummy_output_tensor;
std::vector<std::string> output_node_names;
std::vector<std::vector<int64_t>> output_node_dims; // >=1 outputs
std::vector<Ort::Value> output_values;
std::vector<std::vector<float>> input_values_handler;
Ort::Value dummy_tensor{ nullptr };
std::vector<Ort::Value> dummy_output_tensor;
std::vector<std::vector<float>> input_values_handler;
size_t input_node_size = 1;
std::vector<int64_t> batched_dims;
int batch_size;
int num_outputs = 1;
size_t input_node_size = 1;
std::vector<int64_t> batched_dims;
int batch_size;
int num_outputs = 1;
};
}

Loading…
Cancel
Save