diff --git a/libs/onnxruntime/include/onnxruntime_c_api.h b/libs/onnxruntime/include/onnxruntime_c_api.h index 7c334a2..fcf3239 100644 --- a/libs/onnxruntime/include/onnxruntime_c_api.h +++ b/libs/onnxruntime/include/onnxruntime_c_api.h @@ -38,7 +38,7 @@ * * This value is used by some API functions to behave as this version of the header expects. */ -#define ORT_API_VERSION 21 +#define ORT_API_VERSION 20 #ifdef __cplusplus extern "C" { @@ -46,7 +46,7 @@ extern "C" { //! @} // SAL2 Definitions -#ifndef _MSC_VER +#ifndef _WIN32 #define _In_ #define _In_z_ #define _In_opt_ @@ -626,13 +626,8 @@ typedef struct OrtMIGraphXProviderOptions { } OrtMIGraphXProviderOptions; /** \brief OpenVINO Provider Options - * \brief This Struct is frozen since ORT 1.13.0. Its maintained part of Legacy API for compatibility. - * \brief For latest OpenVINO Provider Options update to the ProviderOptions map. - * \brief Latest OpenVINO Provider Options are listed in the - * \htmlonly - * onnxruntime document. - * \endhtmlonly - * \see OrtApi::SessionOptionsAppendExecutionProvider() + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO */ typedef struct OrtOpenVINOProviderOptions { #ifdef __cplusplus @@ -650,7 +645,7 @@ typedef struct OrtOpenVINOProviderOptions { * Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16" */ const char* device_type; - unsigned char enable_npu_fast_compile; ///< 0 = disabled, nonzero = enabled + unsigned char enable_npu_fast_compile; const char* device_id; size_t num_of_threads; ///< 0 = Use default number of threads const char* cache_dir; // path is set to empty by default @@ -3665,19 +3660,8 @@ struct OrtApi { * - "1": Enabled. * "offload_graph_io_quantization": Offload graph input quantization and graph output dequantization to another * execution provider (typically CPU EP). - * - "0": Disabled. QNN EP will handle quantization and dequantization of graph I/O. - * - "1": Enabled. This is the default value. - * "enable_htp_spill_fill_buffer": Enable HTP spill fill buffer setting. The flag is used while generating context binary. - * - "0": Default. Disabled. - * - "1": Enabled. - * "enable_htp_shared_memory_allocator": Enable the QNN HTP shared memory allocator. Requires libcdsprpc.so/dll to - * be available. - * - "0": Default. Disabled. + * - "0": Default. Disabled. QNN EP will handle quantization and dequantization of graph I/O. * - "1": Enabled. - * "dump_json_qnn_graph": Set to "1" to dump QNN graphs generated by QNN EP as JSON files. Each graph partition - * assigned to QNN EP is dumped to a separate file. - * "json_qnn_graph_dir": Directory in which to dump QNN JSON graphs. If not specified, QNN graphs are dumped in the - * program's current working directory. Ignored if "dump_json_qnn_graph" is not set. * * SNPE supported keys: * "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16", @@ -4623,8 +4607,6 @@ struct OrtApi { * \param[in] num_keys * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.17. */ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO_V2, _In_ OrtSessionOptions* options, @@ -4642,8 +4624,6 @@ struct OrtApi { * \param[in] num_keys * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.18. */ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_VitisAI, _In_ OrtSessionOptions* options, @@ -4657,10 +4637,7 @@ struct OrtApi { * \param[in] mem_info OrtMemoryInfo instance * \param[in] count_or_bytes How many bytes is this scratch buffer * \param[out] out A pointer to the scrach buffer - * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.18. */ ORT_API2_STATUS(KernelContext_GetScratchBuffer, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _In_ size_t count_or_bytes, _Outptr_ void** out); @@ -4671,8 +4648,6 @@ struct OrtApi { * \param[out] out A pointer to OrtAllocator * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.18. */ ORT_API2_STATUS(KernelInfoGetAllocator, _In_ const OrtKernelInfo* info, _In_ OrtMemType mem_type, _Outptr_ OrtAllocator** out); @@ -4694,8 +4669,6 @@ struct OrtApi { * \param[in] num_external_initializer_files Number of external files * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.18. */ ORT_API2_STATUS(AddExternalInitializersFromFilesInMemory, _In_ OrtSessionOptions* options, _In_reads_(num_external_initializer_files) const ORTCHAR_T* const* external_initializer_file_names, @@ -4718,8 +4691,6 @@ struct OrtApi { * OrtApi::ReleaseLoraAdapter. * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.20. */ ORT_API2_STATUS(CreateLoraAdapter, const ORTCHAR_T* adapter_file_path, _In_ OrtAllocator* allocator, _Outptr_ OrtLoraAdapter** out); @@ -4738,8 +4709,6 @@ struct OrtApi { * OrtApi::ReleaseLoraAdapter. * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.20. */ ORT_API2_STATUS(CreateLoraAdapterFromArray, _In_ const void* bytes, size_t num_bytes, _In_ OrtAllocator* allocator, _Outptr_ OrtLoraAdapter** out); @@ -4761,8 +4730,6 @@ struct OrtApi { * \param[in] adapter OrtLoraAdapter instance * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.20. */ ORT_API2_STATUS(RunOptionsAddActiveLoraAdapter, _Inout_ OrtRunOptions* options, _In_ const OrtLoraAdapter* adapter); @@ -4781,8 +4748,6 @@ struct OrtApi { * \param[in] kv_len Number of elements in the keys and values arrays * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.20. */ ORT_API2_STATUS(SetEpDynamicOptions, _Inout_ OrtSession* sess, _In_reads_(kv_len) const char* const* keys, _In_reads_(kv_len) const char* const* values, _In_ size_t kv_len); diff --git a/libs/onnxruntime/include/onnxruntime_cxx_api.h b/libs/onnxruntime/include/onnxruntime_cxx_api.h index 82169c3..ff196cf 100644 --- a/libs/onnxruntime/include/onnxruntime_cxx_api.h +++ b/libs/onnxruntime/include/onnxruntime_cxx_api.h @@ -650,9 +650,6 @@ using AllocatedStringPtr = std::unique_ptr; * constructors to construct an instance of a Status object from exceptions. */ struct Status : detail::Base { - using Base = detail::Base; - using Base::Base; - explicit Status(std::nullptr_t) noexcept {} ///< Create an empty object, must be assigned a valid one to be used explicit Status(OrtStatus* status) noexcept; ///< Takes ownership of OrtStatus instance returned from the C API. explicit Status(const Exception&) noexcept; ///< Creates status instance out of exception @@ -731,9 +728,6 @@ struct Env : detail::Base { * */ struct CustomOpDomain : detail::Base { - using Base = detail::Base; - using Base::Base; - explicit CustomOpDomain(std::nullptr_t) {} ///< Create an empty CustomOpDomain object, must be assigned a valid one to be used /// \brief Wraps OrtApi::CreateCustomOpDomain @@ -969,10 +963,8 @@ struct SessionOptions : detail::SessionOptionsImpl { * */ struct ModelMetadata : detail::Base { - using Base = detail::Base; - using Base::Base; - - explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used + explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used + explicit ModelMetadata(OrtModelMetadata* p) : Base{p} {} ///< Used for interop with the C API /** \brief Returns a copy of the producer name. * @@ -1245,9 +1237,6 @@ using ConstTensorTypeAndShapeInfo = detail::TensorTypeAndShapeInfoImpl { - using Base = detail::TensorTypeAndShapeInfoImpl; - using Base::Base; - explicit TensorTypeAndShapeInfo(std::nullptr_t) {} ///< Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {} ///< Used for interop with the C API ConstTensorTypeAndShapeInfo GetConst() const { return ConstTensorTypeAndShapeInfo{this->p_}; } @@ -1269,9 +1258,6 @@ using ConstSequenceTypeInfo = detail::SequenceTypeInfoImpl { - using Base = detail::SequenceTypeInfoImpl; - using Base::Base; - explicit SequenceTypeInfo(std::nullptr_t) {} ///< Create an empty SequenceTypeInfo object, must be assigned a valid one to be used explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : SequenceTypeInfoImpl{p} {} ///< Used for interop with the C API ConstSequenceTypeInfo GetConst() const { return ConstSequenceTypeInfo{this->p_}; } @@ -1307,9 +1293,6 @@ using ConstMapTypeInfo = detail::MapTypeInfoImpl { - using Base = detail::MapTypeInfoImpl; - using Base::Base; - explicit MapTypeInfo(std::nullptr_t) {} ///< Create an empty MapTypeInfo object, must be assigned a valid one to be used explicit MapTypeInfo(OrtMapTypeInfo* p) : MapTypeInfoImpl{p} {} ///< Used for interop with the C API ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; } @@ -1341,9 +1324,6 @@ using ConstTypeInfo = detail::TypeInfoImpl>; /// the information about contained sequence or map depending on the ONNXType. /// struct TypeInfo : detail::TypeInfoImpl { - using Base = detail::TypeInfoImpl; - using Base::Base; - explicit TypeInfo(std::nullptr_t) {} ///< Create an empty TypeInfo object, must be assigned a valid one to be used explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl{p} {} ///< C API Interop @@ -1681,11 +1661,11 @@ using UnownedValue = detail::ValueImpl>; */ struct Value : detail::ValueImpl { using Base = detail::ValueImpl; - using Base::Base; using OrtSparseValuesParam = detail::OrtSparseValuesParam; using Shape = detail::Shape; - explicit Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used + explicit Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used + explicit Value(OrtValue* p) : Base{p} {} ///< Used for interop with the C API Value(Value&&) = default; Value& operator=(Value&&) = default; @@ -1961,10 +1941,6 @@ struct ArenaCfg : detail::Base { /// This struct provides life time management for custom op attribute /// struct OpAttr : detail::Base { - using Base = detail::Base; - using Base::Base; - - explicit OpAttr(std::nullptr_t) {} OpAttr(const char* name, const void* data, int len, OrtOpAttrType type); }; @@ -2130,10 +2106,10 @@ struct KernelContext { explicit KernelContext(OrtKernelContext* context); size_t GetInputCount() const; size_t GetOutputCount() const; - // If input is optional and is not present, the method returns an empty ConstValue + // If input is optional and is not present, the method returns en empty ConstValue // which can be compared to nullptr. ConstValue GetInput(size_t index) const; - // If output is optional and is not present, the method returns an empty UnownedValue + // If outout is optional and is not present, the method returns en empty UnownedValue // which can be compared to nullptr. UnownedValue GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const; UnownedValue GetOutput(size_t index, const std::vector& dims) const; @@ -2207,8 +2183,6 @@ using ConstKernelInfo = detail::KernelInfoImpl struct KernelInfo : detail::KernelInfoImpl { - using Base = detail::KernelInfoImpl; - using Base::Base; explicit KernelInfo(std::nullptr_t) {} ///< Create an empty instance to initialize later explicit KernelInfo(OrtKernelInfo* info); ///< Take ownership of the instance ConstKernelInfo GetConst() const { return ConstKernelInfo{this->p_}; } @@ -2218,9 +2192,6 @@ struct KernelInfo : detail::KernelInfoImpl { /// Create and own custom defined operation. /// struct Op : detail::Base { - using Base = detail::Base; - using Base::Base; - explicit Op(std::nullptr_t) {} ///< Create an empty Operator object, must be assigned a valid one to be used explicit Op(OrtOp*); ///< Take ownership of the OrtOp diff --git a/libs/onnxruntime/include/onnxruntime_cxx_inline.h b/libs/onnxruntime/include/onnxruntime_cxx_inline.h index b2b2e5c..f1f4904 100644 --- a/libs/onnxruntime/include/onnxruntime_cxx_inline.h +++ b/libs/onnxruntime/include/onnxruntime_cxx_inline.h @@ -51,7 +51,7 @@ inline void ThrowOnError(const Status& st) { } } -inline Status::Status(OrtStatus* status) noexcept : detail::Base{status} { +inline Status::Status(OrtStatus* status) noexcept : Base{status} { } inline Status::Status(const std::exception& e) noexcept { @@ -1908,7 +1908,7 @@ inline void attr_utils::GetAttrs(const OrtKernelInfo* p, const char* name, std:: inline KernelInfo::KernelInfo(OrtKernelInfo* info) : detail::KernelInfoImpl{info} {} -inline Op::Op(OrtOp* p) : detail::Base(p) {} +inline Op::Op(OrtOp* p) : Base(p) {} inline Op Op::Create(const OrtKernelInfo* info, const char* op_name, const char* domain, int version, const char** type_constraint_names, diff --git a/libs/onnxruntime/include/onnxruntime_session_options_config_keys.h b/libs/onnxruntime/include/onnxruntime_session_options_config_keys.h index ddfb665..a9216c3 100644 --- a/libs/onnxruntime/include/onnxruntime_session_options_config_keys.h +++ b/libs/onnxruntime/include/onnxruntime_session_options_config_keys.h @@ -250,51 +250,6 @@ static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersFil static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersMinSizeInBytes = "session.optimized_model_external_initializers_min_size_in_bytes"; -// When loading model from memory buffer and the model has external initializers -// Use this config to set the external data file folder path -// All external data files should be in the same folder -static const char* const kOrtSessionOptionsModelExternalInitializersFileFolderPath = - "session.model_external_initializers_file_folder_path"; - -// Use this config when saving pre-packed constant initializers to an external data file. -// This allows you to memory map pre-packed initializers on model load and leave it to -// to the OS the amount of memory consumed by the pre-packed initializers. Otherwise, -// pre-packed data resides on the heap. -// -// - "0": Default is not save pre-packed initializers to a data file. -// - "1": Save pre-packed constant initializers to an external data file. -// Sample usage: sess_options.add_session_config_entry(kOrtSessionOptionsSavePrePackedConstantInitializers, "1") -static const char* const kOrtSessionOptionsSavePrePackedConstantInitializers = - "session.save_external_prepacked_constant_initializers"; - -// Use this config when you want to collect memory stats for each node in the graph. -// The file format is a CSV file with the following columns: -// The file will be created if it does not exist, and will be overwritten if it does. -// -// The content of the file can be used to estimate memory requirements at run time including -// the temporary allocations. This operation is preferably done on a CPU device, as the model may exceed -// device memory limits in constrained environments. When enabling this option, it is important to disable -// memory patterns, as they tend to allocate large blocks to avoid fragmentation and accommodate needs of multiple -// kernels. Memory patterns may make it difficult to allocate on a device with limited memory. -// -// The collected stats then can be used to partition the graph among the devices in a way that only the -// required memory is allocated on each device. -// -// node_name, initializers_memory, dynamic_outputs_sizes, temp_allocations_size -// -// - "full path to file": there is not a default for this option. If the file can not be opened for writing, an error will be returned. -static const char* const kOrtSessionOptionsCollectNodeMemoryStatsToFile = "session.collect_node_memory_stats_to_file"; - -/// This is a composite CSV setting formatted as "memory limit in kb,file name for collected stats" -/// "limit > 0": enables Capacity Aware Partitioning for Cuda EP. `limit` is optional and when absent -/// the provider may attempt to figure out the memory available automatically. -/// The setting with no limit is expected to look like: ",file name for collected stats" -/// The EP will place nodes on device "file name" : -/// this file is expected to be found at the same folder with the model. The file contains -/// pre-recorded stats collected when running with kOrtSessionOptionsCollectNodeMemoryStatsToFile enforce (see above) -static const char* const kOrtSessionOptionsResourceCudaPartitioningSettings = - "session.resource_cuda_partitioning_settings"; - // Enable EP context feature to dump the partitioned graph which includes the EP context into Onnx file. // The dumped Onnx model with EP context can be used for future inference to avoid the EP graph partitioning/compile overhead. // "0": disable. (default) @@ -303,12 +258,11 @@ static const char* const kOrtSessionOptionEpContextEnable = "ep.context_enable"; // Specify the file path for the Onnx model which has EP context. // Default to original_file_name_ctx.onnx if not specified -// Folder is not a valid option static const char* const kOrtSessionOptionEpContextFilePath = "ep.context_file_path"; // Flag to specify whether to dump the EP context into the Onnx model. -// "0": dump the EP context into separate file, keep the file name in the Onnx model. (default). -// "1": dump the EP context into the Onnx model. +// "0": dump the EP context into separate file, keep the file name in the Onnx model. +// "1": dump the EP context into the Onnx model. (default). static const char* const kOrtSessionOptionEpContextEmbedMode = "ep.context_embed_mode"; // Specify the EPContext node name prefix to make it unique @@ -318,11 +272,6 @@ static const char* const kOrtSessionOptionEpContextNodeNamePrefix = "ep.context_ // Share EP related resources across EPs static const char* const kOrtSessionOptionShareEpContexts = "ep.share_ep_contexts"; -// Use this config when dumping EP context model with an external initializers file -// All initializers will be inside the external data file if specified, otherwise all in Onnx file -static const char* const kOrtSessionOptionsEpContextModelExternalInitializersFileName = - "ep.context_model_external_initializers_file_name"; - // Gemm fastmath mode provides fp32 gemm acceleration with bfloat16 based matmul. // Option values: // - "0": Gemm FastMath mode is not enabled. [DEFAULT] diff --git a/libs/onnxruntime/lib/msys2/onnxruntime.dll b/libs/onnxruntime/lib/msys2/onnxruntime.dll deleted file mode 100644 index bda8a2a..0000000 --- a/libs/onnxruntime/lib/msys2/onnxruntime.dll +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b78eb3f4e65513c904bc6810cd7ed596399e689f026a19d096c7653644190752 -size 13426720 diff --git a/libs/onnxruntime/lib/msys2/onnxruntime.lib b/libs/onnxruntime/lib/msys2/onnxruntime.lib deleted file mode 100644 index 965a554..0000000 --- a/libs/onnxruntime/lib/msys2/onnxruntime.lib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5deed87357eb953489b9abfb13b43539e71543c790974f8db0b1aedc0438dfd3 -size 2852 diff --git a/libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.dll b/libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.dll deleted file mode 100644 index a57170b..0000000 --- a/libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.dll +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:48611eac97db0561b5dbc92d982823ab9d399871f093ec5c0b99b70126917c01 -size 320433712 diff --git a/libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.lib b/libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.lib deleted file mode 100644 index a84a862..0000000 --- a/libs/onnxruntime/lib/msys2/onnxruntime_providers_cuda.lib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6b46ad7c335fb0ef1abf25ab5b28aaba0ebfb58fe094e1ad83bbcc7eabe53aec -size 2038 diff --git a/libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.dll b/libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.dll deleted file mode 100644 index fffa43c..0000000 --- a/libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.dll +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d5ef034019ec9e3a3710404f66336bbe893f2d4593f0e40840e079095756be97 -size 22056 diff --git a/libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.lib b/libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.lib deleted file mode 100644 index fc404d6..0000000 --- a/libs/onnxruntime/lib/msys2/onnxruntime_providers_shared.lib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e3de923bee2994db06e0d01ce8c0b62ce9b05c25501b3efc837a039d3eef88f1 -size 2314 diff --git a/libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.dll b/libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.dll deleted file mode 100644 index c81fe0f..0000000 --- a/libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.dll +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b5a23ef47b63165dbe579ae80be4feb4bf04b0473b7be13b69b9ad8825a726c5 -size 780320 diff --git a/libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.lib b/libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.lib deleted file mode 100644 index 673d5a3..0000000 --- a/libs/onnxruntime/lib/msys2/onnxruntime_providers_tensorrt.lib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:20870322214db089be606eacfb535372d7c50f1166e2d613740777bf0d5e8037 -size 2090 diff --git a/src/ofxOnnxRuntime.cpp b/src/ofxOnnxRuntime.cpp index b91b0e2..e26e3a2 100644 --- a/src/ofxOnnxRuntime.cpp +++ b/src/ofxOnnxRuntime.cpp @@ -1,5 +1,4 @@ #include "ofxOnnxRuntime.h" -#include "ofMain.h" namespace ofxOnnxRuntime { @@ -16,7 +15,7 @@ namespace ofxOnnxRuntime } #endif - void BaseHandler::setup(const std::string & onnx_path, const BaseSetting & base_setting, const std::vector& batched_dims, const int & batch_size) + void BaseHandler::setup(const std::string & onnx_path, const BaseSetting & base_setting, const int & batch_size, const bool debug, const bool timestamp) { Ort::SessionOptions session_options; session_options.SetIntraOpNumThreads(1); @@ -32,9 +31,9 @@ namespace ofxOnnxRuntime session_options.AppendExecutionProvider_CUDA(opts); } - // Sets batch size + this->timestamp = timestamp; + this->debug = debug; this->batch_size = batch_size; - this->batched_dims = batched_dims; this->setup2(onnx_path, session_options); } @@ -46,7 +45,11 @@ namespace ofxOnnxRuntime ort_session = std::make_shared(ort_env, wpath.c_str(), session_options); + setNames(); + } + void BaseHandler::setNames() + { Ort::AllocatorWithDefaultOptions allocator; // 1. Gets Input Name/s & Shape ([1, 3, 28, 28]) -- In most cases this is usually just one @@ -55,16 +58,18 @@ namespace ofxOnnxRuntime input_node_dims = ort_session->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); // Some models might have negative shape values to indicate dynamic shape, e.g., for variable batch size. (?, 3, 28, 28) -> (1, 3, 28, 28) - for (auto& s : input_node_dims) if (s < 0) s = 1; + for (auto& s : input_node_dims) if (s < 0) s = batch_size; - std::cout << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl; + if (debug) std::cout << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl; } // 2. Calculate the product of the dimensions - for (auto& f : batched_dims) { + for (auto& f : input_node_dims) { input_node_size *= f; } + if (debug) ofLog() << ofToString(input_node_size) + ", Batch Size:" + ofToString(input_node_dims[0]); + // 3. Resize input values array to match input tensor/s input_values_handler.resize(batch_size); @@ -83,16 +88,20 @@ namespace ofxOnnxRuntime output_values.emplace_back(nullptr); - std::cout << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl; + if (debug) std::cout << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl; } } - std::vector& BaseHandler::run() + float* BaseHandler::run() { + + auto start = std::chrono::high_resolution_clock::now(); // starting timestamp + std::vector input_tensors; // 1. Create 1-D array for all values to create tensor & push all values from input_vals to batch_vals - std::vector batch_values(input_node_size * batch_size); + std::vector batch_values; + batch_values.reserve(input_node_size * batch_size); // Reserve space but don't initialize for (const auto& inner_vec : input_values_handler) { for (float value : inner_vec) { @@ -121,16 +130,60 @@ namespace ofxOnnxRuntime input_names_char.data(), input_tensors.data(), input_names_char.size(), output_names_char.data(), output_names_char.size()); + + if (debug) { + // Gets the address of the first value + auto& out = output_values.front(); + // Get tensor shape information + Ort::TensorTypeAndShapeInfo info = out.GetTensorTypeAndShapeInfo(); + std::vector output_dims = info.GetShape(); + + // Print the dimensions + std::cout << "Output tensor dimensions: ["; + for (size_t i = 0; i < output_dims.size(); i++) { + std::cout << output_dims[i]; + if (i < output_dims.size() - 1) { + std::cout << ", "; + } + } + std::cout << "]" << std::endl; + + // Optional: Print total number of elements + size_t total_elements = 1; + for (auto& dim : output_dims) { + if (dim > 0) { // Handle dynamic dimensions + total_elements *= static_cast(dim); + } + } + std::cout << "Total elements: " << total_elements << std::endl; + } + + if (timestamp) { + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = end - start; + std::cout << "Update loop took " << elapsed.count() << " ms" << std::endl; + } - return output_values; - } - catch (const Ort::Exception& ex) { + return output_values.front().GetTensorMutableData(); + + } catch (const Ort::Exception& ex) { std::cout << "ERROR running model inference: " << ex.what() << std::endl; - return dummy_output_tensor; + return dummy_output_tensor.front().GetTensorMutableData(); } } + /* + * + * Utilties (。・∀・)ノ゙ + * + */ + + // Fills the tensor (selected by the idx) with input values from ofFloatPixels array + void BaseHandler::setInput(ofFloatPixels &pixels, int tensor_idx, int width, int height, int channels) { + pixels.setFromExternalPixels(getInputTensorData()->at(tensor_idx).data(), width, height, channels); + } + // Prints the shape of the given tensor (ex. input: (1, 1, 512, 512)) std::string BaseHandler::PrintShape(const std::vector& v) { std::stringstream ss; @@ -176,9 +229,6 @@ namespace ofxOnnxRuntime } Ort::Value BaseHandler::VectorToTensor(std::vector& data, const std::vector& shape) { - //// Allocate memory using CPU memory allocator - //Ort::MemoryInfo mem_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); - // Create a tensor from the provided data, shape, and memory info auto tensor = Ort::Value::CreateTensor(memory_info_handler, data.data(), data.size(), shape.data(), shape.size()); diff --git a/src/ofxOnnxRuntime.h b/src/ofxOnnxRuntime.h index ec54088..a513a7c 100644 --- a/src/ofxOnnxRuntime.h +++ b/src/ofxOnnxRuntime.h @@ -1,6 +1,7 @@ #pragma once #include +#include "ofMain.h" namespace ofxOnnxRuntime { @@ -19,45 +20,48 @@ namespace ofxOnnxRuntime class BaseHandler { - public: - BaseHandler() {} + public: + BaseHandler() {} - void setup(const std::string& onnx_path, const BaseSetting& base_setting = BaseSetting{ INFER_CPU, 0 }, const std::vector& batched_dims = {}, const int& batch_size = 1); - void setup2(const std::string& onnx_path, const Ort::SessionOptions& session_options); + void setup(const std::string& onnx_path, const BaseSetting& base_setting = BaseSetting{ INFER_CPU, 0 }, const int& batch_size = 1, const bool debug = false, const bool timestamp = false); + void setup2(const std::string& onnx_path, const Ort::SessionOptions& session_options); + void setNames(); + void setInput(ofFloatPixels &pixels, int tensor_idx, int width, int height, int channels); + float* run(); - std::vector>* getInputTensorData() { - return &this->input_values_handler; - } + // Utilities ╰(‵□′)╯ + std::string PrintShape(const std::vector& v); + Ort::Value GenerateTensor(int batch_size); + int CalculateProduct(const std::vector& v); + Ort::Value VectorToTensor(std::vector& data, const std::vector& shape); - std::vector& run(); + std::vector>* getInputTensorData() { + return &this->input_values_handler; + } + + protected: + bool debug = false; + bool timestamp = false; + Ort::Env ort_env; + std::shared_ptr ort_session; - // Utilities - std::string PrintShape(const std::vector& v); - Ort::Value GenerateTensor(int batch_size); - int CalculateProduct(const std::vector& v); - Ort::Value VectorToTensor(std::vector& data, const std::vector& shape); + std::vector input_node_names; + std::vector input_node_dims; // 1 input only. - protected: - Ort::Env ort_env; - std::shared_ptr ort_session; + Ort::MemoryInfo memory_info_handler = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); - std::vector input_node_names; - std::vector input_node_dims; // 1 input only. + std::vector output_node_names; + std::vector> output_node_dims; // >=1 outputs + std::vector output_values; - Ort::MemoryInfo memory_info_handler = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + Ort::Value dummy_tensor{ nullptr }; + std::vector dummy_output_tensor; - std::vector output_node_names; - std::vector> output_node_dims; // >=1 outputs - std::vector output_values; + std::vector> input_values_handler; - Ort::Value dummy_tensor{ nullptr }; - std::vector dummy_output_tensor; - - std::vector> input_values_handler; - - size_t input_node_size = 1; - std::vector batched_dims; - int batch_size; - int num_outputs = 1; + size_t input_node_size = 1; + std::vector batched_dims; + int batch_size; + int num_outputs = 1; }; }