commit 98afb8ebf6d8d352b09a127bd92205b935554b30 Author: cailean Date: Tue Sep 10 16:55:46 2024 +0100 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b52ee82 --- /dev/null +++ b/.gitignore @@ -0,0 +1,90 @@ +#.gitignore is a file which makes git ignore files which should +# not go into version control in the first place +# Questions? See +# http://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository#Ignoring-Files +# http://git-scm.com/docs/gitignore + +########################### +# ignore generated binaries +# but not the data folder +########################### + +/bin/* + +######### +# general +######### + +[Bb]uild/ +[Oo]bj/ +*.o +[Dd]ebug*/ +[Rr]elease*/ +*.mode* +*.app/ +*.pyc +.svn/ +*.log + +######################## +# IDE files which should +# be ignored +######################## + +# XCode +*.pbxuser +*.perspective +*.perspectivev3 +*.mode1v3 +*.mode2v3 +# XCode 4 +*.xccheckout +xcuserdata/ + +# Visual Studio +*.sdf +*.opensdf +*.suo +*.pdb +*.ilk +*.aps +ipch/ + +# Eclipse +.metadata +local.properties +.externalToolBuilders + +# Android Studio +.gradle +/local.properties +/.idea/workspace.xml +/.idea/libraries + +################## +# operating system +################## + +# Linux +*~ +# KDE +.directory +.AppleDouble + +# OSX +.DS_Store +*.swp +*~.nib +# Thumbnails +._* + +# Windows +# Image file caches +Thumbs.db +# Folder config file +Desktop.ini + +# Android +.csettings + +/bin/data/ \ No newline at end of file diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 0000000..b9ff5a2 --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,181 @@ +{ + "configurations": [ + { + "name": "Mac", + "includePath": [ + "/usr/include", + "/usr/local/include", + "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/", + "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/4.2.1", + "${workspaceRoot}/../../../libs/openFrameworks", + "${workspaceRoot}/../../../libs/openFrameworks/3d", + "${workspaceRoot}/../../../libs/openFrameworks/app", + "${workspaceRoot}/../../../libs/openFrameworks/communication", + "${workspaceRoot}/../../../libs/openFrameworks/events", + "${workspaceRoot}/../../../libs/openFrameworks/gl", + "${workspaceRoot}/../../../libs/openFrameworks/graphics", + "${workspaceRoot}/../../../libs/openFrameworks/math", + "${workspaceRoot}/../../../libs/openFrameworks/sound", + "${workspaceRoot}/../../../libs/openFrameworks/types", + "${workspaceRoot}/../../../libs/openFrameworks/utils", + "${workspaceRoot}/../../../libs/openFrameworks/video", + "${workspaceRoot}/../../../libs/boost/include", + "${workspaceRoot}/../../../libs/cairo/include", + "${workspaceRoot}/../../../libs/curl/include", + "${workspaceRoot}/../../../libs/fmod/include", + "${workspaceRoot}/../../../libs/FreeImage/include", + "${workspaceRoot}/../../../libs/freetype/include", + "${workspaceRoot}/../../../libs/glew/include", + "${workspaceRoot}/../../../libs/glfw/include", + "${workspaceRoot}/../../../libs/glm/include", + "${workspaceRoot}/../../../libs/json/include", + "${workspaceRoot}/../../../libs/pugixml/include", + "${workspaceRoot}/../../../libs/rtAudio/include", + "${workspaceRoot}/../../../libs/tess2/include", + "${workspaceRoot}/../../../libs/uriparser/include", + "${workspaceRoot}/../../../libs/utf8/include" + ], + "browse": { + "limitSymbolsToIncludedHeaders": true, + "databaseFilename": "${workspaceRoot}/.vscode/browse.db", + "path": [ + "/usr/include", + "/usr/local/include", + "/System/Library/Frameworks", + "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/", + "${workspaceRoot}/../../../libs", + "${workspaceRoot}/../../../addons", + "${workspaceRoot}/../../../libs/openFrameworks" + ] + }, + "intelliSenseMode": "clang-x64", + "macFrameworkPath": [ + "/System/Library/Frameworks", + "/Library/Frameworks" + ], + "compilerPath": "/usr/local/bin/gcc-8", + "cStandard": "c11", + "cppStandard": "c++17" + }, + { + "name": "Linux", + "includePath": [ + "/usr/include", + "/usr/local/include", + "/usr/include/c++/5", + "/usr/include/c++/5/backward", + "/usr/lib/gcc/x86_64-linux-gnu/5/include", + "/usr/lib/x86_64-linux-gnu/glib-2.0/include", + "/usr/lib/x86_64-linux-gnu/gstreamer-1.0/include", + "/usr/local/include", + "/usr/lib/gcc/x86_64-linux-gnu/5/include-fixed", + "/usr/include/x86_64-linux-gnu", + "${workspaceRoot}/../../../libs/openFrameworks", + "${workspaceRoot}/../../../libs/openFrameworks/3d", + "${workspaceRoot}/../../../libs/openFrameworks/app", + "${workspaceRoot}/../../../libs/openFrameworks/communication", + "${workspaceRoot}/../../../libs/openFrameworks/events", + "${workspaceRoot}/../../../libs/openFrameworks/gl", + "${workspaceRoot}/../../../libs/openFrameworks/graphics", + "${workspaceRoot}/../../../libs/openFrameworks/math", + "${workspaceRoot}/../../../libs/openFrameworks/sound", + "${workspaceRoot}/../../../libs/openFrameworks/types", + "${workspaceRoot}/../../../libs/openFrameworks/utils", + "${workspaceRoot}/../../../libs/openFrameworks/video", + "${workspaceRoot}/../../../libs/boost/include", + "${workspaceRoot}/../../../libs/cairo/include", + "${workspaceRoot}/../../../libs/curl/include", + "${workspaceRoot}/../../../libs/fmod/include", + "${workspaceRoot}/../../../libs/FreeImage/include", + "${workspaceRoot}/../../../libs/freetype/include", + "${workspaceRoot}/../../../libs/glew/include", + "${workspaceRoot}/../../../libs/glfw/include", + "${workspaceRoot}/../../../libs/glm/include", + "${workspaceRoot}/../../../libs/json/include", + "${workspaceRoot}/../../../libs/pugixml/include", + "${workspaceRoot}/../../../libs/rtAudio/include", + "${workspaceRoot}/../../../libs/tess2/include", + "${workspaceRoot}/../../../libs/uriparser/include", + "${workspaceRoot}/../../../libs/utf8/include", + "${workspaceRoot}/../../../libs/poco/include", + "${workspaceRoot}/../../../libs/kiss/include", + "/usr/include/x86_64-linux-gnu/c++/5", + "/usr/include/pulse", + "/usr/include/cairo", + "/usr/include/gstreamer-1.0", + "/usr/include/glib-2.0", + "${workspaceRoot}", + "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxOpenCv/src", + "/usr/local/onnxruntime-linux-x64-gpu-1.19.2/include", + "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxOpenCv/libs/opencv/include/opencv4" + ], + "browse": { + "limitSymbolsToIncludedHeaders": true, + "databaseFilename": "", + "path": [ + "/usr/include", + "/usr/local/include", + "${workspaceRoot}/../../../libs", + "${workspaceRoot}/../../../addons", + "${workspaceRoot}/../../../libs/openFrameworks", + "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxOpenCv/src", + "/usr/local/onnxruntime-linux-x64-gpu-1.19.2/include" + ] + }, + "intelliSenseMode": "clang-x64", + "compilerPath": "/usr/bin/g++", + "cStandard": "c17", + "cppStandard": "c++17" + }, + { + "name": "Win32", + "includePath": [ + "C:/msys64/mingw32/include/c++/7.4.0", + "C:/msys64/mingw32/i686-w64-mingw32/include", + "${workspaceRoot}/../../../libs", + "${workspaceRoot}/../../../addons", + "${workspaceRoot}/../../../libs/openFrameworks", + "${workspaceRoot}/../../../libs/openFrameworks/3d", + "${workspaceRoot}/../../../libs/openFrameworks/app", + "${workspaceRoot}/../../../libs/openFrameworks/communication", + "${workspaceRoot}/../../../libs/openFrameworks/events", + "${workspaceRoot}/../../../libs/openFrameworks/gl", + "${workspaceRoot}/../../../libs/openFrameworks/graphics", + "${workspaceRoot}/../../../libs/openFrameworks/math", + "${workspaceRoot}/../../../libs/openFrameworks/sound", + "${workspaceRoot}/../../../libs/openFrameworks/types", + "${workspaceRoot}/../../../libs/openFrameworks/utils", + "${workspaceRoot}/../../../libs/openFrameworks/video", + "${workspaceRoot}/../../../libs/boost/include", + "${workspaceRoot}/../../../libs/cairo/include", + "${workspaceRoot}/../../../libs/curl/include", + "${workspaceRoot}/../../../libs/fmod/include", + "${workspaceRoot}/../../../libs/FreeImage/include", + "${workspaceRoot}/../../../libs/freetype/include", + "${workspaceRoot}/../../../libs/glew/include", + "${workspaceRoot}/../../../libs/glfw/include", + "${workspaceRoot}/../../../libs/glm/include", + "${workspaceRoot}/../../../libs/json/include", + "${workspaceRoot}/../../../libs/pugixml/include", + "${workspaceRoot}/../../../libs/rtAudio/include", + "${workspaceRoot}/../../../libs/tess2/include", + "${workspaceRoot}/../../../libs/uriparser/include", + "${workspaceRoot}/../../../libs/utf8/include" + ], + "browse": { + "limitSymbolsToIncludedHeaders": true, + "databaseFilename": "", + "path": [ + "${workspaceRoot}/../../../libs", + "${workspaceRoot}/../../../addons", + "${workspaceRoot}/../../../libs/openFrameworks" + ] + }, + "intelliSenseMode": "clang-x64", + "compilerPath": "C:/msys64/mingw32/bin/g++.exe", + "cStandard": "c11", + "cppStandard": "c++17" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..e535136 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,72 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "C++ Launch", + "type": "cppdbg", + "request": "launch", + "args": [], + "stopAtEntry": false, + "cwd": "${workspaceRoot}", + "environment": [], + "externalConsole": false, + "linux": { + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + "program": "${workspaceRoot}/bin/${workspaceFolderBasename}_debug" + }, + "osx": { + "MIMode": "lldb", + "program": "${workspaceRoot}/bin/${workspaceFolderBasename}_debug.app/Contents/MacOS/${workspaceFolderBasename}_debug" + }, + "windows": { + "MIMode": "gdb", + "miDebuggerPath": "gdb.exe", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + "program": "${workspaceRoot}/bin/${workspaceFolderBasename}_debug" + } + }, + { + "name": "C++ Attach", + "type": "cppdbg", + "request": "attach", + "program": "enter program name, for example ${workspaceRoot}/a.out", + "processId": "${command:pickProcess}", + "linux": { + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + "osx": { + "MIMode": "lldb" + }, + "windows": { + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + } + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..2de3ce0 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,133 @@ +{ + "version": "2.0.0", + "shell":{ + "task": true + }, + "tasks": [ + { + "type": "shell", + "label": "Build RELEASE", + "presentation": { + "reveal": "always", + "panel": "shared" + }, + "command": "make -j -s 2>&1 && make RunRelease", + "group": { + "kind": "build", + "isDefault": true + }, + "problemMatcher": { + "owner": "cpp", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + } + }, + { + "type": "shell", + "label": "Build DEBUG", + "presentation": { + "reveal": "always", + "panel": "shared" + }, + "command": "make Debug -j -s 2>&1 || exit 1", + "group": { + "kind": "build", + "isDefault": true + }, + "problemMatcher": { + "owner": "cpp", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + } + }, + { + "type": "shell", + "label": "Clean DEBUG", + "presentation": { + "reveal": "always", + "panel": "shared" + }, + "command": "make CleanDebug", + "group": { + "kind": "build", + "isDefault": true + }, + "problemMatcher": { + "owner": "cpp", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + } + }, + { + "type": "shell", + "label": "Clean RELEASE", + "presentation": { + "reveal": "always", + "panel": "shared" + }, + "command": "make CleanRelease", + "group": { + "kind": "build", + "isDefault": true + }, + "problemMatcher": { + "owner": "cpp", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + } + }, + { + "type": "shell", + "label": "Clean ALL", + "presentation": { + "reveal": "always", + "panel": "shared" + }, + "command": "make clean", + "group": { + "kind": "build", + "isDefault": true + }, + "problemMatcher": { + "owner": "cpp", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + } + } + ] +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..177e172 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +# Attempt to load a config.make file. +# If none is found, project defaults in config.project.make will be used. +ifneq ($(wildcard config.make),) + include config.make +endif + +# make sure the the OF_ROOT location is defined +ifndef OF_ROOT + OF_ROOT=$(realpath ../../..) +endif + +# call the project makefile! +include $(OF_ROOT)/libs/openFrameworksCompiled/project/makefileCommon/compile.project.mk diff --git a/addons.make b/addons.make new file mode 100644 index 0000000..ecff515 --- /dev/null +++ b/addons.make @@ -0,0 +1,2 @@ +ofxOpenCv +ofxOpenCv diff --git a/config.make b/config.make new file mode 100644 index 0000000..920d2f1 --- /dev/null +++ b/config.make @@ -0,0 +1,142 @@ +################################################################################ +# CONFIGURE PROJECT MAKEFILE (optional) +# This file is where we make project specific configurations. +################################################################################ + +################################################################################ +# OF ROOT +# The location of your root openFrameworks installation +# (default) OF_ROOT = ../../.. +################################################################################ +OF_ROOT = ../../.. + +################################################################################ +# PROJECT ROOT +# The location of the project - a starting place for searching for files +# (default) PROJECT_ROOT = . (this directory) +# +################################################################################ +# PROJECT_ROOT = . + +################################################################################ +# PROJECT SPECIFIC CHECKS +# This is a project defined section to create internal makefile flags to +# conditionally enable or disable the addition of various features within +# this makefile. For instance, if you want to make changes based on whether +# GTK is installed, one might test that here and create a variable to check. +################################################################################ +# None + +################################################################################ +# PROJECT EXTERNAL SOURCE PATHS +# These are fully qualified paths that are not within the PROJECT_ROOT folder. +# Like source folders in the PROJECT_ROOT, these paths are subject to +# exlclusion via the PROJECT_EXLCUSIONS list. +# +# (default) PROJECT_EXTERNAL_SOURCE_PATHS = (blank) +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_EXTERNAL_SOURCE_PATHS = + +################################################################################ +# PROJECT EXCLUSIONS +# These makefiles assume that all folders in your current project directory +# and any listed in the PROJECT_EXTERNAL_SOURCH_PATHS are are valid locations +# to look for source code. The any folders or files that match any of the +# items in the PROJECT_EXCLUSIONS list below will be ignored. +# +# Each item in the PROJECT_EXCLUSIONS list will be treated as a complete +# string unless teh user adds a wildcard (%) operator to match subdirectories. +# GNU make only allows one wildcard for matching. The second wildcard (%) is +# treated literally. +# +# (default) PROJECT_EXCLUSIONS = (blank) +# +# Will automatically exclude the following: +# +# $(PROJECT_ROOT)/bin% +# $(PROJECT_ROOT)/obj% +# $(PROJECT_ROOT)/%.xcodeproj +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_EXCLUSIONS = + +################################################################################ +# PROJECT LINKER FLAGS +# These flags will be sent to the linker when compiling the executable. +# +# (default) PROJECT_LDFLAGS = -Wl,-rpath=./libs +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ + +# Currently, shared libraries that are needed are copied to the +# $(PROJECT_ROOT)/bin/libs directory. The following LDFLAGS tell the linker to +# add a runtime path to search for those shared libraries, since they aren't +# incorporated directly into the final executable application binary. +# TODO: should this be a default setting? +# PROJECT_LDFLAGS=-Wl,-rpath=./libs + +################################################################################ +# PROJECT DEFINES +# Create a space-delimited list of DEFINES. The list will be converted into +# CFLAGS with the "-D" flag later in the makefile. +# +# (default) PROJECT_DEFINES = (blank) +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_DEFINES = + +################################################################################ +# PROJECT CFLAGS +# This is a list of fully qualified CFLAGS required when compiling for this +# project. These CFLAGS will be used IN ADDITION TO the PLATFORM_CFLAGS +# defined in your platform specific core configuration files. These flags are +# presented to the compiler BEFORE the PROJECT_OPTIMIZATION_CFLAGS below. +# +# (default) PROJECT_CFLAGS = (blank) +# +# Note: Before adding PROJECT_CFLAGS, note that the PLATFORM_CFLAGS defined in +# your platform specific configuration file will be applied by default and +# further flags here may not be needed. +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_CFLAGS = + +################################################################################ +# PROJECT OPTIMIZATION CFLAGS +# These are lists of CFLAGS that are target-specific. While any flags could +# be conditionally added, they are usually limited to optimization flags. +# These flags are added BEFORE the PROJECT_CFLAGS. +# +# PROJECT_OPTIMIZATION_CFLAGS_RELEASE flags are only applied to RELEASE targets. +# +# (default) PROJECT_OPTIMIZATION_CFLAGS_RELEASE = (blank) +# +# PROJECT_OPTIMIZATION_CFLAGS_DEBUG flags are only applied to DEBUG targets. +# +# (default) PROJECT_OPTIMIZATION_CFLAGS_DEBUG = (blank) +# +# Note: Before adding PROJECT_OPTIMIZATION_CFLAGS, please note that the +# PLATFORM_OPTIMIZATION_CFLAGS defined in your platform specific configuration +# file will be applied by default and further optimization flags here may not +# be needed. +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_OPTIMIZATION_CFLAGS_RELEASE = +# PROJECT_OPTIMIZATION_CFLAGS_DEBUG = + +################################################################################ +# PROJECT COMPILERS +# Custom compilers can be set for CC and CXX +# (default) PROJECT_CXX = (blank) +# (default) PROJECT_CC = (blank) +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_CXX = +# PROJECT_CC = diff --git a/onnx-test.code-workspace b/onnx-test.code-workspace new file mode 100644 index 0000000..b91d338 --- /dev/null +++ b/onnx-test.code-workspace @@ -0,0 +1,107 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "${workspaceRoot}/../../../../libs/openFrameworks" + }, + { + "path": "${workspaceRoot}/../../../../addons" + } + ], + "settings": { + "files.associations": { + "cctype": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "csignal": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "any": "cpp", + "array": "cpp", + "atomic": "cpp", + "strstream": "cpp", + "bit": "cpp", + "*.tcc": "cpp", + "bitset": "cpp", + "cfenv": "cpp", + "charconv": "cpp", + "chrono": "cpp", + "codecvt": "cpp", + "compare": "cpp", + "complex": "cpp", + "concepts": "cpp", + "condition_variable": "cpp", + "cstdint": "cpp", + "deque": "cpp", + "forward_list": "cpp", + "list": "cpp", + "map": "cpp", + "set": "cpp", + "string": "cpp", + "unordered_map": "cpp", + "unordered_set": "cpp", + "vector": "cpp", + "exception": "cpp", + "expected": "cpp", + "algorithm": "cpp", + "functional": "cpp", + "iterator": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "optional": "cpp", + "random": "cpp", + "ratio": "cpp", + "regex": "cpp", + "source_location": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "utility": "cpp", + "fstream": "cpp", + "future": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "mutex": "cpp", + "new": "cpp", + "numbers": "cpp", + "ostream": "cpp", + "ranges": "cpp", + "semaphore": "cpp", + "shared_mutex": "cpp", + "span": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "stop_token": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "cinttypes": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "valarray": "cpp", + "variant": "cpp", + "format": "cpp", + "stdfloat": "cpp", + "__nullptr": "cpp", + "*.ipp": "cpp", + "__bit_reference": "cpp", + "__hash_table": "cpp", + "__split_buffer": "cpp", + "__tree": "cpp", + "filesystem": "cpp" + } + } +} \ No newline at end of file diff --git a/onnx-test.qbs b/onnx-test.qbs new file mode 100644 index 0000000..0c42546 --- /dev/null +++ b/onnx-test.qbs @@ -0,0 +1,69 @@ +import qbs +import qbs.Process +import qbs.File +import qbs.FileInfo +import qbs.TextFile +import "../../../libs/openFrameworksCompiled/project/qtcreator/ofApp.qbs" as ofApp + +Project{ + property string of_root: "../../.." + + ofApp { + name: { return FileInfo.baseName(sourceDirectory) } + + files: [ + 'src/main.cpp', + 'src/ofApp.cpp', + 'src/ofApp.h', + ] + + of.addons: [ + 'ofxOpenCv', + 'ofxOpenCv', + ] + + // additional flags for the project. the of module sets some + // flags by default to add the core libraries, search paths... + // this flags can be augmented through the following properties: + of.pkgConfigs: [] // list of additional system pkgs to include + of.includePaths: [] // include search paths + of.cFlags: [] // flags passed to the c compiler + of.cxxFlags: [] // flags passed to the c++ compiler + of.linkerFlags: [] // flags passed to the linker + of.defines: [] // defines are passed as -D to the compiler + // and can be checked with #ifdef or #if in the code + of.frameworks: [] // osx only, additional frameworks to link with the project + of.staticLibraries: [] // static libraries + of.dynamicLibraries: [] // dynamic libraries + + // other flags can be set through the cpp module: http://doc.qt.io/qbs/cpp-module.html + // eg: this will enable ccache when compiling + // + // cpp.compilerWrapper: 'ccache' + + Depends{ + name: "cpp" + } + + // common rules that parse the include search paths, core libraries... + Depends{ + name: "of" + } + + // dependency with the OF library + Depends{ + name: "openFrameworks" + } + } + + property bool makeOF: true // use makfiles to compile the OF library + // will compile OF only once for all your projects + // otherwise compiled per project with qbs + + + property bool precompileOfMain: false // precompile ofMain.h + // faster to recompile when including ofMain.h + // but might use a lot of space per project + + references: [FileInfo.joinPaths(of_root, "/libs/openFrameworksCompiled/project/qtcreator/openFrameworks.qbs")] +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..799e13c --- /dev/null +++ b/readme.md @@ -0,0 +1 @@ +## Affection Mapping Beta ## diff --git a/src/Onnx.cpp b/src/Onnx.cpp new file mode 100644 index 0000000..4a543e3 --- /dev/null +++ b/src/Onnx.cpp @@ -0,0 +1,287 @@ +#include "Onnx.h" + +// Setups the model. CUDA is enabled by default +void Onnx::Setup(ORTCHAR_T* modelPath, bool isLog, bool useCuda){ + session_options.SetIntraOpNumThreads(1); + session_options.SetIntraOpNumThreads(1); + session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + log = isLog; + + // cuda setup + if(useCuda){ + OrtCUDAProviderOptions opts; + opts.device_id = 0; + opts.cudnn_conv_algo_search = OrtCudnnConvAlgoSearchExhaustive; + opts.do_copy_in_default_stream = 0; + opts.arena_extend_strategy = 0; + session_options.AppendExecutionProvider_CUDA(opts); + } + + ort_session = std::make_shared(ort_env, modelPath, session_options); + + Ort::AllocatorWithDefaultOptions allocator; + for (std::size_t i = 0; i < ort_session->GetInputCount(); i++) { + input_node_names.emplace_back(ort_session->GetInputNameAllocated(i, allocator).get()); + input_node_dims = ort_session->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); + if(log) + std::cout << "\t" << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl; + } + + // some models might have negative shape values to indicate dynamic shape, e.g., for variable batch size. + for (auto& s : input_node_dims) { + if (s < 0) { + s = 1; + if(log) + std::cout << "transfromed!" << std::endl; + } + } + + + + if(log) + std::cout << "Output Node Name/Shape (" << output_node_names.size() << "):" << std::endl; + for (std::size_t i = 0; i < ort_session->GetOutputCount(); i++) { + output_node_names.emplace_back(ort_session->GetOutputNameAllocated(i, allocator).get()); + auto output_shapes = ort_session->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); + if(log) + std::cout << "\t" << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl; + } + +} + +// Runs the model, given an image +std::vector Onnx::Run(ofImage &img){ + + auto start = std::chrono::high_resolution_clock::now(); + + TransformImage(img); + size_t input_tensor_size = image_array.total(); + std::vector input_tensors; + Ort::MemoryInfo mem_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); + input_tensors.emplace_back(Ort::Value::CreateTensor(mem_info, (float*)image_array.data, + input_tensor_size, input_node_dims.data(), input_node_dims.size())); + + // double-check the dimensions of the input tensor + assert(input_tensors[0].IsTensor() && input_tensors[0].GetTensorTypeAndShapeInfo().GetShape() == input_node_dims); + + //std::cout << "\ninput_tensor shape: " << PrintShape(input_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << std::endl; + + // pass data through model + std::vector input_names_char(input_node_names.size(), nullptr); + std::transform(std::begin(input_node_names), std::end(input_node_names), std::begin(input_names_char), + [&](const std::string& str) { return str.c_str(); }); + + std::vector output_names_char(output_node_names.size(), nullptr); + std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char), + [&](const std::string& str) { return str.c_str(); }); + + std::cout << "Running model... "; + + try { + auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(), + input_names_char.size(), output_names_char.data(), output_names_char.size()); + + std::cout << "Done!" << std::endl; + + if (timeStamp) { + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = end - start; + std::cout << "Update loop took " << elapsed.count() << " ms" << std::endl; + } + + return output_tensors; + + } catch (const Ort::Exception& exception) { + std::cout << "ERROR running model inference: " << exception.what() << std::endl; + return input_tensors; + + } + +} + +/* + Runs a model, with a batch of images as input (emotion.onnx) + (1) Creates a 1-dim float array based on the batch * channels * width * height) + (2) Transforms each image into a cv::Mat, and appends it to the array + (3) Sends that information to the model to be processed. +*/ +std::vector Onnx::RunBatch(std::vector& images){ + + auto start = std::chrono::high_resolution_clock::now(); + + // Number of images in the batch + size_t batchSize = images.size(); + + std::vector batch_node_dims = {static_cast(batchSize), 3, 260, 260}; + + std::vector batch_image_array(static_cast(batchSize) * 3 * 260 * 260); + + for(size_t i = 0; i < batchSize; i++){ + TransformImage(images[i]); + // Copy the image data into the batch_image_array + std::memcpy( + batch_image_array.data() + i * 3 * 260 * 260, // Destination: starting point for this image in the batch array + image_array.data, // Source: pixel data in the cv::Mat + 3 * 260 * 260 * sizeof(float) // Size: number of bytes to copy (3 channels * 260 * 260 pixels) + ); + } + + size_t input_tensor_size = batch_image_array.size(); + std::vector input_tensors; + Ort::MemoryInfo mem_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); + input_tensors.emplace_back(Ort::Value::CreateTensor(mem_info, batch_image_array.data(), + input_tensor_size, batch_node_dims.data(), batch_node_dims.size())); + + // double-check the dimensions of the input tensor + assert(input_tensors[0].IsTensor() && input_tensors[0].GetTensorTypeAndShapeInfo().GetShape() == batch_node_dims); + + std::cout << "\ninput_tensor shape: " << PrintShape(input_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << std::endl; + + // pass data through model + std::vector input_names_char(input_node_names.size(), nullptr); + std::transform(std::begin(input_node_names), std::end(input_node_names), std::begin(input_names_char), + [&](const std::string& str) { return str.c_str(); }); + + std::vector output_names_char(output_node_names.size(), nullptr); + std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char), + [&](const std::string& str) { return str.c_str(); }); + + std::cout << "Running model... "; + + try { + auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(), + input_names_char.size(), output_names_char.data(), output_names_char.size()); + + std::cout << "Done!" << std::endl; + + if (timeStamp) { + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = end - start; + std::cout << "Update loop took " << elapsed.count() << " ms" << std::endl; + } + + return output_tensors; + + } catch (const Ort::Exception& exception) { + std::cout << "ERROR running model inference: " << exception.what() << std::endl; + return input_tensors; + + } + +} + +// Transforms an ofImage into a cv::ImageArray +void Onnx::TransformImage(ofImage &img){ + // Convert ofImage to cv::Mat + ofPixels &pixels = img.getPixels(); // Get pixels from ofImage + int width = img.getWidth(); + int height = img.getHeight(); + int channels = img.getPixels().getNumChannels(); + + // Create a cv::Mat from the pixel data + cv::Mat cvImg = cv::Mat(height, width, (channels == 3) ? CV_8UC3 : CV_8UC1, pixels.getData()); + + // Convert cv::Mat to cv::InputArray + cv::InputArray inputArray(cvImg); + + image_array = cv::dnn::blobFromImage(inputArray, 1 / 255.0, cv::Size(input_node_dims[2] , input_node_dims[3] ), (0, 0, 0), false, false); +} + +/* Generates a random 1 dimensional float array, creating values between 0->255. + Returns a tensor. */ +Ort::Value Onnx::GenerateTensor(){ + std::vector random_input_tensor_values(CalculateProduct(input_node_dims)); + std::generate(random_input_tensor_values.begin(), random_input_tensor_values.end(), [&] { return rand() % 255; }); + return VectorToTensor(random_input_tensor_values, input_node_dims); +} + +// Calculates the product of the vector, how many values. +int Onnx::CalculateProduct(const std::vector& v){ + int total = 1; + for (auto& i : v) total *= i; + return total; +} + +// Creates a tensor from a given vector input. +Ort::Value Onnx::VectorToTensor(std::vector& data, const std::vector& shape){ + Ort::MemoryInfo mem_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); + auto tensor = Ort::Value::CreateTensor(mem_info, data.data(), data.size(), shape.data(), shape.size()); + return tensor; +} + +// Prints the shape of the given tensor (ex. input: (1, 1, 512, 512)) +std::string Onnx::PrintShape(const std::vector& v){ + std::stringstream ss(""); + for (std::size_t i = 0; i < v.size() - 1; i++) ss << v[i] << "x"; + ss << v[v.size() - 1]; + return ss.str(); +} + +// Function to calculate the minimum value in an array +float Onnx::ReduceMin(const float* data, size_t size) { + return *std::min_element(data, data + size); +} + +// Function to calculate the maximum value in an array +float Onnx::ReduceMax(const float* data, size_t size) { + return *std::max_element(data, data + size); +} + +// Function to normalize an array between 0 and 1 +void Onnx::Normalize(float* data, size_t size, float min_value, float max_value) { + for (size_t i = 0; i < size; ++i) { + data[i] = (data[i] - min_value) / (max_value - min_value); + } +} + +// Coverts the output tensor data to a texture of a given ofFbo. +void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo){ + // Convert data into opencv mat + cv::Mat inputMat(height, width, CV_32FC1, const_cast(data)); + + // Convert to 8-bit grayscale Mat + cv::Mat inputMat8U; + inputMat.convertTo(inputMat8U, CV_8UC1, 255.0); // Convert float to 8-bit grayscale + + // Resize the image using OpenCV + cv::Mat resizedMat; + cv::resize(inputMat8U, resizedMat, cv::Size(fbo.getWidth(), fbo.getHeight()), 0, 0, cv::INTER_LINEAR); + + // Convert OpenCV Mat to ofPixels + ofPixels pixels; + pixels.allocate(fbo.getWidth(), fbo.getHeight(), OF_PIXELS_GRAY); + + // Copy data from resizedMat to ofPixels + for (size_t y = 0; y < fbo.getHeight(); ++y) { + for (size_t x = 0; x < fbo.getWidth(); ++x) { + unsigned char value = resizedMat.at(y, x); + pixels.setColor(x, y, ofColor(value)); + } + } + + // Update FBO with new pixels + fbo.begin(); + ofTexture& texture = fbo.getTexture(); + texture.loadData(pixels); + fbo.end(); + + +} + +void Onnx::Softmax(float* data, size_t size) { + std::vector logits(data, data + size); + std::vector expValues(size); + float maxLogit = *std::max_element(logits.begin(), logits.end()); + + // Calculate exp(logit - maxLogit) for numerical stability + std::transform(logits.begin(), logits.end(), expValues.begin(), + [maxLogit](float logit) { return std::exp(logit - maxLogit); }); + + float sumExp = std::accumulate(expValues.begin(), expValues.end(), 0.0f); + + // Normalize to get probabilities + std::transform(expValues.begin(), expValues.end(), data, + [sumExp](float expValue) { return expValue / sumExp; }); +} + + diff --git a/src/Onnx.h b/src/Onnx.h new file mode 100644 index 0000000..8e17c51 --- /dev/null +++ b/src/Onnx.h @@ -0,0 +1,57 @@ +#ifndef ONNX +#define ONNX + +#include +#include // std::generate +#include +#include +#include +#include +#include +#include +#include +#include // For std::rand and std::srand +#include "ofMain.h" +#include "ofxOpenCv.h" +#include +#include + + class Onnx { + + public: + Onnx() {} + void Setup(ORTCHAR_T* modelPath, bool isLog, bool useCuda); + std::vector Run(ofImage &img); + std::vector RunBatch(std::vector &images); + std::string PrintShape(const std::vector& v); + Ort::Value VectorToTensor(std::vector& data, const std::vector& shape); + Ort::Value GenerateTensor(); + int CalculateProduct(const std::vector& v); + void TransformImage(ofImage &img); + float ReduceMin(const float* data, size_t size); + float ReduceMax(const float* data, size_t size); + void Normalize(float* data, size_t size, float min_value, float max_value); + void DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo); + void Softmax(float* data, size_t size); + bool timeStamp = true; + bool log = false; + + protected: + Ort::Env ort_env; + Ort::SessionOptions session_options; + cv::Mat image_array; + std::shared_ptr ort_session; + std::vector input_node_names; + std::vector input_node_dims; // 1 input only. + std::size_t input_tensor_size = 1; + std::vector input_values_handler; + Ort::MemoryInfo memory_info_handler = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + std::vector output_node_names; + std::vector> output_node_dims; // >=1 outputs + std::vector output_values; + Ort::Value dummy_tensor{ nullptr }; + std::vector imageBatch; + int num_outputs = 1; + }; + + #endif \ No newline at end of file diff --git a/src/Player.cpp b/src/Player.cpp new file mode 100644 index 0000000..7672811 --- /dev/null +++ b/src/Player.cpp @@ -0,0 +1,55 @@ +#include "Player.h" + +Player::Player(){ + hasVideo = false; +} + +/* Basic ofVideoPlayer setup */ +void Player::Setup(){ + videoPlayer.setLoopState(OF_LOOP_NONE); + videoPlayer.setVolume(0); +} + +/* Updated the video player: + (1) Allocates the required W x H for the model input + (2) Updates the video texture, and sets the current frame value */ +void Player::Update(ofImage &img){ + + if(!img.isAllocated() || img.getWidth() != videoPlayer.getWidth() || img.getHeight() != videoPlayer.getHeight()){ + img.allocate(videoPlayer.getWidth(), videoPlayer.getHeight(), OF_IMAGE_COLOR); + std::cout << "allocating new ofImage" << std::endl; + } + + if(videoPlayer.isLoaded()){ + hasVideo = true; + playerCurrentFrame = videoPlayer.getCurrentFrame(); + videoPlayer.update(); + videoPlayer.play(); + } +} + +void Player::Draw(){ + if(videoPlayer.isLoaded()){ + videoPlayer.draw(0, 0); + } +} + +ofPixels Player::GetVideoPixels(){ + return videoPlayer.getPixels(); +} + +/* Loads the video from path: + (1) Sets the frame + (2) Allocates the fbo dims for final render */ +void Player::SetVideo(std::string path, ofFbo &fbo){ + videoPlayer.load(path); + videoPlayer.setFrame(800); + fbo.allocate(videoPlayer.getWidth(), videoPlayer.getHeight(), GL_RGB); +} + +// Sets a random frame in the active video +void Player::SetRandomFrame(){ + int randomFrame = ofRandom(0, videoPlayer.getTotalNumFrames()); + std::cout << "setting frame: " << randomFrame << std::endl; + videoPlayer.setFrame(randomFrame); +} \ No newline at end of file diff --git a/src/Player.h b/src/Player.h new file mode 100644 index 0000000..98baf26 --- /dev/null +++ b/src/Player.h @@ -0,0 +1,39 @@ +#ifndef _PLAYER +#define _PLAYER + +#include "ofMain.h" + +class Player { + + public: + + void Setup(); + void Update(ofImage &img); + void Draw(); + void SetVideo(std::string path, ofFbo &fbo); + ofPixels GetVideoPixels(); + void SetVideoPosition(); + void SetRandomFrame(); + void SetupGUI(); + void UpdateGUI(); + void UpdateVector(); + void CheckNewFrame(); + void GetCurrentFrame(); + + ofVideoPlayer videoPlayer; + + int playerCurrentFrame; + int playerTotalFrameNum; + int playerVideoIndex; + std::string videoPath; + bool hasVideo; + + glm::vec2 centerPosition; + + ofFbo fbo; + + Player(); + +}; + +#endif \ No newline at end of file diff --git a/src/Yolo.cpp b/src/Yolo.cpp new file mode 100644 index 0000000..33e7535 --- /dev/null +++ b/src/Yolo.cpp @@ -0,0 +1,155 @@ +#include "Yolo.h" + +// Takes output tensor data, processes that data, and returns an array of BoxfWithLandmarks (detected_faces) +void Yolo::ParseOutput(float* &output_tensors, std::vector &sorted_faces, unsigned int num_anchors) { + + std::vector detected_faces; + + for (unsigned int i = 0; i < num_anchors; ++i) { + const float *row_ptr = output_tensors + i * 16; // Each row contains 16 values + float obj_conf = row_ptr[4]; // Objectness confidence + + if (obj_conf < 0.5) continue; // Filter out low-confidence detections + + // Extract bounding box, confidence, and landmarks + float cx = row_ptr[0]; + float cy = row_ptr[1]; + float w = row_ptr[2]; + float h = row_ptr[3]; + float cls_conf = row_ptr[15]; // Face confidence score + + if (cls_conf < 0.5) continue; // Filter by class confidence + + types::BoxfWithLandmarks face; + face.box.x1 = cx - w / 2; + face.box.y1 = cy - h / 2; + face.box.x2 = cx + w / 2; + face.box.y2 = cy + h / 2; + face.box.score = cls_conf; + face.box.label_text = "face"; + + // Extract landmarks + for (int j = 0; j < 10; j += 2) { + face.landmarks.points.push_back(cv::Point2f(row_ptr[5 + j], row_ptr[5 + j + 1])); + } + + detected_faces.push_back(face); // Store the detected face with landmarks + } + + // Apply NMS to detected faces list to remove any overlapping bounding boxes. + NonMaximumSuppression(detected_faces, sorted_faces, 0.5); + + // Sort faces based on confidence value + SortDetectedFaces(sorted_faces); +} + +// Simple helper for drawing boxes given x1, y1, x2, y2 coordinates. +void Yolo::DrawBox(std::vector &detected_faces){ + for (const auto &face : detected_faces) { + ofNoFill(); + ofDrawRectangle(face.box.x1, face.box.y1, face.box.x2 - face.box.x1, face.box.y2 - face.box.y1); + } +} + +// Simple helper to draw boxes at the center of the detected face. +void Yolo::DrawCenter(std::vector &detected_faces){ + ofNoFill(); + ofDrawCircle(detected_faces[0].box.center, 5); + +} + +// Applies NMS to an array of BoxfWithLandmarks.face.boxes, to remove any overlapping bounding boxes. +void Yolo::NonMaximumSuppression(std::vector &input_faces, std::vector &output_faces, float iou_threshold) +{ + // Sort the boxes by their confidence scores (highest to lowest) + std::sort(input_faces.begin(), input_faces.end(), + [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) { + return a.box.score > b.box.score; + }); + + std::vector suppressed(input_faces.size(), 0); // Suppression mask + + // Iterate through the boxes + for (size_t i = 0; i < input_faces.size(); ++i) { + if (suppressed[i]) continue; // Skip already suppressed boxes + + // Add this box to the output + output_faces.push_back(input_faces[i]); + + for (size_t j = i + 1; j < input_faces.size(); ++j) { + if (suppressed[j]) continue; + + // Calculate IoU between box i and box j + float iou = input_faces[i].box.iou_of(input_faces[j].box); + + // Suppress box j if IoU is greater than the threshold + if (iou > iou_threshold) { + suppressed[j] = 1; + } + } + } +} + +// Scales the coordinates of detected faces from the model output dimensions -> the original input image dimensions. +void Yolo::ConvertBoxCoordsToOriginalSize(std::vector &detected_faces, size_t original_width, size_t original_height){ + float width_scale = static_cast(original_width) / modelSize; + float height_scale = static_cast(original_height) / modelSize; + + for (auto &face : detected_faces) { + // Convert bounding box coordinates + face.box.x1 *= width_scale; + face.box.y1 *= height_scale; + face.box.x2 *= width_scale; + face.box.y2 *= height_scale; + face.box.UpdateCenter(); + + // Convert landmarks + for (size_t j = 0; j < face.landmarks.points.size(); ++j) { + face.landmarks.points[j].x *= width_scale; + face.landmarks.points[j].y *= height_scale; + } + } +} + +void Yolo::CropFaceToImage(ofImage &inputImage, types::BoxfWithLandmarks &face, ofxCvColorImage &colorImage){ + + colorImage.resetROI(); + + // Calculate the coordinates and dimensions of the face box + float x1 = face.box.x1; + float y1 = face.box.y1; + float x2 = face.box.x2; + float y2 = face.box.y2; + + // Ensure coordinates are within the input image bounds + x1 = ofClamp(x1, 0.0f, (float)inputImage.getWidth()); + y1 = ofClamp(y1, 0.0f, (float)inputImage.getHeight()); + x2 = ofClamp(x2, 0.0f, (float)inputImage.getWidth()); + y2 = ofClamp(y2, 0.0f, (float)inputImage.getHeight()); + + // Calculate width and height of the cropped area + float cropWidth = x2 - x1; + float cropHeight = y2 - y1; + + // Create cropped section, defined by the box coords + ofFbo tempFbo; + tempFbo.allocate(cropWidth, cropHeight, GL_RGB); + + tempFbo.begin(); + ofClear(0); + inputImage.getTexture().drawSubsection(0, 0, cropWidth, cropHeight, x1, y1); + tempFbo.end(); + + ofFloatPixels pix; + tempFbo.readToPixels(pix); + colorImage.setFromPixels(pix); + + colorImage.resize(260, 260); +} + +void Yolo::SortDetectedFaces(std::vector &detectedFaces){ + std::sort(detectedFaces.begin(), detectedFaces.end(), + [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) { + return a.box.score > b.box.score; // Sort in descending order + }); +} \ No newline at end of file diff --git a/src/Yolo.h b/src/Yolo.h new file mode 100644 index 0000000..808cc50 --- /dev/null +++ b/src/Yolo.h @@ -0,0 +1,81 @@ +#ifndef YOLO +#define YOLO + +#include "ofMain.h" +#include "ofxOpenCv.h" +#include +#include + +struct Emotef{ + float emotions[7]; +}; + +namespace types { + + /* + Struct for storing information about detetced faces. + */ + struct Boxf { + float x1, y1, x2, y2; // Coordinates of the bounding box + float score; // Confidence score + glm::vec2 center; + int label; // Class label (e.g., "face") + std::string label_text; + Emotef emotional_state; + + // Calculate Intersection over Union (IoU) with another box + float iou_of(const Boxf &other) const { + float intersection_x1 = std::max(x1, other.x1); + float intersection_y1 = std::max(y1, other.y1); + float intersection_x2 = std::min(x2, other.x2); + float intersection_y2 = std::min(y2, other.y2); + + float intersection_area = std::max(0.0f, intersection_x2 - intersection_x1) * + std::max(0.0f, intersection_y2 - intersection_y1); + + float this_area = (x2 - x1) * (y2 - y1); + float other_area = (other.x2 - other.x1) * (other.y2 - other.y1); + + float union_area = this_area + other_area - intersection_area; + + return intersection_area / union_area; + } + + void UpdateCenter(){ + center.x = (x1 + x2) / 2; + center.y = (y1 + y2) / 2; + } + + void SetEmotionState(float* emotional_data){ + std::copy(emotional_data, emotional_data + 7, emotional_state.emotions); + } + }; + + struct Landmarks { + std::vector points; // Facial landmarks points (e.g., eyes, nose, mouth) + bool flag = false; // Indicator if landmarks are available + }; + + struct BoxfWithLandmarks { + Boxf box; // Bounding box for the face + Landmarks landmarks; // Landmark points for the face + bool flag = false; // Indicator if this detection is valid + }; +} + +class Yolo{ + public: + Yolo(){}; + void ParseOutput(float* &out_ptr, std::vector &sorted_faces, unsigned int num_anchors); + void DrawBox(std::vector &detected_faces); + void DrawCenter(std::vector &detected_faces); + void NonMaximumSuppression(std::vector &input_faces, std::vector &output_faces, float iou_threshold); + void ConvertBoxCoordsToOriginalSize(std::vector &detected_faces, size_t original_width, size_t original_height); + void CropFaceToImage(ofImage &inputImage, types::BoxfWithLandmarks &face, ofxCvColorImage &colorImage); + void SortDetectedFaces(std::vector &detectedFaces); + private: + // Input dimenions of the model -- used for coordinate scaling. + size_t modelSize = 640; +}; + +#endif \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..e2540ed --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,17 @@ +#include "ofMain.h" +#include "ofApp.h" + +//======================================================================== +int main( ){ + + //Use ofGLFWWindowSettings for more options like multi-monitor fullscreen + ofGLWindowSettings settings; + settings.setSize(1280, 720); + settings.windowMode = OF_WINDOW; //can also be OF_FULLSCREEN + + auto window = ofCreateWindow(settings); + + ofRunApp(window, make_shared()); + ofRunMainLoop(); + +} diff --git a/src/ofApp.cpp b/src/ofApp.cpp new file mode 100644 index 0000000..66212f3 --- /dev/null +++ b/src/ofApp.cpp @@ -0,0 +1,210 @@ +#include "ofApp.h" + +//-------------------------------------------------------------- +void ofApp::setup(){ + ofSetFrameRate(24); + ofSetVerticalSync(true); + + player.Setup(); + player.SetVideo("videos/demo.mp4", fbo); + + emoteImage.allocate(260, 260); + tempImage.allocate(emoteImage.getWidth(), emoteImage.getHeight(), OF_IMAGE_COLOR); + + ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/depth_anything_v2_vitb.onnx"; + ORTCHAR_T* modelPath2 = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/yolov5s-face.onnx"; + ORTCHAR_T* modelPath3 = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/rgb_emotion.onnx"; + + /* Setup Models (modelPath, log, useCuda) */ + yolo.Setup(modelPath2, false, true); + depth.Setup(modelPath, false, true); + emotion.Setup(modelPath3, false, true); +} + + + +//-------------------------------------------------------------- +void ofApp::update(){ + /* Check to see if the application has moved to the first frame + As the models need to load first, as the first inference is quite slow */ + if(ofGetFrameNum() > 0) + firstRun = false; + + /* Clear detetced face list */ + detected_faces.clear(); + + /* Setup model input using ofImage */ + player.Update(img); + img.setFromPixels(player.GetVideoPixels()); + + /* Run Models */ + try{ + auto output_tensors = depth.Run(img); + float* output_ptr = output_tensors.front().GetTensorMutableData(); + size_t num_elements = output_tensors.front().GetTensorTypeAndShapeInfo().GetElementCount(); + + float min_value = depth.ReduceMin(output_ptr, num_elements); + float max_value = depth.ReduceMax(output_ptr, num_elements); + + depth.Normalize(output_ptr, num_elements, min_value, max_value); + + depth.DataToFbo(output_ptr, 518, 518, fbo); + + auto output_tensors_face = yolo.Run(img); + + auto output_faces = output_tensors_face.front().GetTensorTypeAndShapeInfo().GetShape(); + + unsigned int num_anchors = output_faces[1]; // Number of anchors + + float* output_face_ptr = output_tensors_face.front().GetTensorMutableData(); + + faceDetector.ParseOutput(output_face_ptr, detected_faces, num_anchors); + + faceDetector.ConvertBoxCoordsToOriginalSize(detected_faces, fbo.getWidth(), fbo.getHeight()); + + /* As no input is generated for the emotion recognition model, run a dummy vector through the model + So it can load */ + if(firstRun){ + + /* + Create a dummy initial input of batch_size = 5, as + when initialising the model, it will attempt to create a space in memory for this array. + If the batch_size does change it will completely slow down inference, due to how the cudnn_search_algo is set. + None of the other search alogithms bar EXHAUSTIVE will work.. no idead why. + */ + for(int i = 0; i < emotionImageMaxBatchSize; i++){ + tempImage.setFromPixels(emoteImage.getPixels()); + croppedFaces.push_back(tempImage); + } + + // Run model to warmup + auto emotion_output_tensor = emotion.RunBatch(croppedFaces); + + } else { + inferEmotionalState(); + } + + /* Run emotion inference */ + //inferEmotionalState(); + + } catch (exception e){ + + std::cout << "Model did not run" << std::endl; + + } + +} + + +//-------------------------------------------------------------- +void ofApp::draw(){ + fbo.draw(0, 0); + + if(!firstRun){ + faceDetector.DrawBox(detected_faces); + faceDetector.DrawCenter(detected_faces); + } + +// emoteImage.draw(640, 0); +// for(auto& face : detected_faces){ +// ofDrawBitmapString(std::to_string(face.box.emotional_state.emotions[0]), 700, 300); +// } + +} + +//-------------------------------------------------------------- +void ofApp::inferEmotionalState(){ + + /* + Max faces to process with the model (5) + */ + int max_faces_to_process = std::min(emotionImageMaxBatchSize, static_cast(detected_faces.size())); + + /* + For the detetced faces, set the cropped image to a location in the image batch array + */ + for (size_t i = 0; i < max_faces_to_process; i++){ + auto& face = detected_faces[i]; + faceDetector.CropFaceToImage(img, face, emoteImage); + tempImage.setFromPixels(emoteImage.getPixels()); + croppedFaces[i] = tempImage; + } + + + auto emotion_output_tensor = emotion.RunBatch(croppedFaces); + + auto& output_tensor = emotion_output_tensor.front(); + auto output_shap = output_tensor.GetTensorTypeAndShapeInfo().GetShape(); + size_t batch_size = output_shap[0]; // Number of images in the batch + size_t num_classes = output_shap[1]; // Number of emotion classes + std::cout << batch_size << " : " << num_classes << std::endl; + + // for(int i = 0; i < max_faces_to_process; ++i){ + // auto& face = detected_faces[i]; + // faceDetector.CropFaceToImage(img, face, emoteImage); + // tempImage.setFromPixels(emoteImage.getPixels()); + // auto emotion_output_tensor = emotion.Run(tempImage); + // float* emotional_data = emotion_output_tensor.front().GetTensorMutableData(); + // emotion.Softmax(emotional_data, 7); + // face.box.SetEmotionState(emotional_data); + // } + + std::cout << croppedFaces.size() << std::endl; +} + +//-------------------------------------------------------------- +void ofApp::keyPressed(int key){ + if (key=OF_KEY_LEFT){ + player.SetRandomFrame(); + } +} + +//-------------------------------------------------------------- +void ofApp::keyReleased(int key){ + +} + +//-------------------------------------------------------------- +void ofApp::mouseMoved(int x, int y ){ + +} + +//-------------------------------------------------------------- +void ofApp::mouseDragged(int x, int y, int button){ + +} + +//-------------------------------------------------------------- +void ofApp::mousePressed(int x, int y, int button){ + +} + +//-------------------------------------------------------------- +void ofApp::mouseReleased(int x, int y, int button){ + +} + +//-------------------------------------------------------------- +void ofApp::mouseEntered(int x, int y){ + +} + +//-------------------------------------------------------------- +void ofApp::mouseExited(int x, int y){ + +} + +//-------------------------------------------------------------- +void ofApp::windowResized(int w, int h){ + +} + +//-------------------------------------------------------------- +void ofApp::gotMessage(ofMessage msg){ + +} + +//-------------------------------------------------------------- +void ofApp::dragEvent(ofDragInfo dragInfo){ + +} diff --git a/src/ofApp.h b/src/ofApp.h new file mode 100644 index 0000000..3970985 --- /dev/null +++ b/src/ofApp.h @@ -0,0 +1,49 @@ +#pragma once + +#include "ofMain.h" +#include +#include "ofxOpenCv.h" +#include "Onnx.h" +#include "Yolo.h" +#include +#include "Player.h" + +class ofApp : public ofBaseApp{ + + public: + void setup(); + void update(); + void draw(); + + void keyPressed(int key); + void keyReleased(int key); + void mouseMoved(int x, int y ); + void mouseDragged(int x, int y, int button); + void mousePressed(int x, int y, int button); + void mouseReleased(int x, int y, int button); + void mouseEntered(int x, int y); + void mouseExited(int x, int y); + void windowResized(int w, int h); + void dragEvent(ofDragInfo dragInfo); + void gotMessage(ofMessage msg); + void inferEmotionalState(); + + ofImage img; + ofFbo fbo; + cv::Mat cvImg; + ofVideoGrabber webcam; + Player player; + bool firstRun = true; + + Onnx depth; + Onnx yolo; + Onnx emotion; + ofxCvColorImage emoteImage; + ofImage tempImage; + std::vector croppedFaces; + int emotionImageMaxBatchSize = 5; + + Emotef emo; + Yolo faceDetector; + std::vector detected_faces; +};