From b78446bcf773547fc84d7a81cea1787d751a90d9 Mon Sep 17 00:00:00 2001 From: cailean Date: Sat, 5 Oct 2024 00:56:38 +0100 Subject: [PATCH] unsure where to go! --- src/ModelThread.h | 4 ++++ src/Onnx.cpp | 1 - src/Onnx.h | 2 +- src/main.cpp | 2 +- src/ofApp.cpp | 53 +++++++++++++++++++---------------------------- src/ofApp.h | 2 ++ 6 files changed, 29 insertions(+), 35 deletions(-) diff --git a/src/ModelThread.h b/src/ModelThread.h index 9d1dd26..6267438 100644 --- a/src/ModelThread.h +++ b/src/ModelThread.h @@ -12,6 +12,10 @@ class ModelThread : public ofThread std::vector* detected_faces; std::string model_type; + // emotional recognition model + std::vector* croppedFaces; + float* emotional_data; + ~ModelThread(){ stop(); diff --git a/src/Onnx.cpp b/src/Onnx.cpp index a89ebcd..f93d2ee 100644 --- a/src/Onnx.cpp +++ b/src/Onnx.cpp @@ -52,7 +52,6 @@ void Onnx::Setup(ORTCHAR_T* modelPath, bool isLog, bool useCuda){ // Runs the model, given an image std::vector Onnx::Run(ofImage &img){ - std::cout << "hallo" << std::endl; auto start = std::chrono::high_resolution_clock::now(); TransformImage(img); diff --git a/src/Onnx.h b/src/Onnx.h index e99e4b7..c9d785c 100644 --- a/src/Onnx.h +++ b/src/Onnx.h @@ -34,7 +34,7 @@ void DataToFbo(float* data, size_t width, size_t height, ofFbo& fbo); void Softmax(float* data, size_t size); void SetPixels(ofFbo& fbo); - bool timeStamp = true; + bool timeStamp = false; bool log = false; ofPixels pixels; diff --git a/src/main.cpp b/src/main.cpp index f9818e4..0afab0b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6,7 +6,7 @@ int main( ){ //Use ofGLFWWindowSettings for more options like multi-monitor fullscreen ofGLWindowSettings settings; - settings.setSize(2000, 1000); + settings.setSize(1920, 720); settings.setGLVersion(3, 2); settings.windowMode = OF_WINDOW; //can also be OF_FULLSCREEN diff --git a/src/ofApp.cpp b/src/ofApp.cpp index e66cbc0..aae8e29 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -30,7 +30,7 @@ void ofApp::setup(){ yolo.Setup(modelPath2, false, true); depth.Setup(modelPath, false, true); depth_small.Setup(modelPath4, false, true); - emotion.Setup(modelPath3, false, true); + //emotion.Setup(modelPath3, false, true); /* Depth output fbo */ model_output_fbo.allocate(window_width / 2, window_height, GL_RGB); @@ -46,6 +46,18 @@ void ofApp::setup(){ threadMap.setup(&map.fboImage, &model_output_fbo, &depth); threadVideo.setup(&img, &model_output_fbo_1, &depth_small); threadYolo.setupYolo(&img, &detected_faces, &yolo, &faceDetector); + + /* + Create a dummy initial input of batch_size = 5, as + when initialising the model, it will attempt to create a space in memory for this array. + If the batch_size does change it will completely slow down inference, due to how the cudnn_search_algo is set. + None of the other search alogithms bar EXHAUSTIVE will work.. no idea why. + + for(int i = 0; i < emotionImageMaxBatchSize; i++){ + tempImage.setFromPixels(emoteImage.getPixels()); + croppedFaces.push_back(tempImage); + } + */ } @@ -78,40 +90,18 @@ void ofApp::update(){ try{ threadMap.update(); - depth.SetPixels(model_output_fbo); - threadVideo.update(); - depth_small.SetPixels(model_output_fbo_1); - threadYolo.update(); + depth.SetPixels(model_output_fbo); + depth_small.SetPixels(model_output_fbo_1); faceDetector.ConvertBoxCoordsToOriginalSize(detected_faces, model_output_fbo_1.getWidth(), model_output_fbo_1.getHeight()); /* As no input is generated for the emotion recognition model, run a dummy vector through the model - So it can load */ - if(firstRun){ - /* - Create a dummy initial input of batch_size = 5, as - when initialising the model, it will attempt to create a space in memory for this array. - If the batch_size does change it will completely slow down inference, due to how the cudnn_search_algo is set. - None of the other search alogithms bar EXHAUSTIVE will work.. no idea why. - */ - // for(int i = 0; i < emotionImageMaxBatchSize; i++){ - // tempImage.setFromPixels(emoteImage.getPixels()); - // croppedFaces.push_back(tempImage); - // } - - // Run model to warmup - // auto emotion_output_tensor = emotion.RunBatch(croppedFaces); - - } else { - //inferEmotionalState(); - - } - - /* Run emotion inference */ - //inferEmotionalState(); + So it can load. + auto emotion_output_tensor = emotion.RunBatch(croppedFaces); + */ } catch (exception e){ std::cout << "Model did not run" << std::endl; @@ -167,7 +157,6 @@ void ofApp::inferDepthImage(ofFbo& fbo, ofImage& img, Onnx& model){ //-------------------------------------------------------------- void ofApp::inferEmotionalState(){ - /* Max faces to process with the model (5) */ @@ -188,13 +177,13 @@ void ofApp::inferEmotionalState(){ for each image -> set emotional state in detected_faces array */ auto emotion_output_tensor = emotion.RunBatch(croppedFaces); - + auto& output_tensor = emotion_output_tensor.front(); auto output_shap = output_tensor.GetTensorTypeAndShapeInfo().GetShape(); size_t batch_size = output_shap[0]; // Number of images in the batch - size_t num_classes = output_shap[1]; // Number of emotion classes + size_t num_classes = 7; //output_shap[1]; // Number of emotion classes - float* emotional_data = output_tensor.GetTensorMutableData(); + emotional_data = output_tensor.GetTensorMutableData(); for (size_t i = 0; i < max_faces_to_process; i++){ diff --git a/src/ofApp.h b/src/ofApp.h index bba6113..63f89fe 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -55,6 +55,7 @@ class ofApp : public ofBaseApp{ Emotef emo; Yolo faceDetector; + float* emotional_data; std::vector detected_faces; Map map; @@ -72,4 +73,5 @@ class ofApp : public ofBaseApp{ ModelThread threadMap; ModelThread threadVideo; ModelThread threadYolo; + ModelThread threadEmotion; };