frustum culling added & fixed low fps

src/Map.cpp

@@ -42,16 +42,25 @@ void Map::Draw(){
 
     ofClear(0, 0, 0, 1);
 
+    ofMatrix4x4 projectionMatrix = cam.getProjectionMatrix();
+    ofMatrix4x4 viewMatrix = cam.getModelViewMatrix();
+    ofMatrix4x4 mvpMatrix = projectionMatrix * viewMatrix;
+
     for (auto& n :nodes){
+        glm::vec3 node_position = n.position + n.offset;
+
+        if(isFrustum(node_position, 50)){
             n.texture.getTexture().bind();
             ofPushMatrix();
                 ofFill();
-                n.geom.setPosition(n.position + n.offset);
+                n.geom.setPosition(node_position);
                 n.geom.draw();
             ofPopMatrix();
             n.texture.getTexture().unbind();
         }
         
+    }
+
     cam.end();
 
     mapFbo.end();
@@ -189,3 +198,21 @@ void Map::SortNodes(){
     });
     std::cout << "Finished Sorting!" << std::endl;
 }
+
+bool Map::isFrustum(const glm::vec3& position, float radius){
+    float box_w = 2000;
+    float box_h = 2000;
+    glm::vec3 cam_position = cam.getPosition();
+
+    float left = cam_position.x - box_w / 2.0f;
+    float right = cam_position.x + box_w / 2.0f;
+    float top = cam_position.y + box_h / 2.0f;
+    float bottom = cam_position.y - box_h / 2.0f;
+
+    // Check if the object (with radius) is within the bounding box
+    if (position.x + radius < left || position.x - radius > right) return false;
+    if (position.y + radius < bottom || position.y - radius > top) return false;
+
+    // Z-depth doesn't matter in this approach, so we're ignoring it
+    return true;
+}

src/Map.h

@@ -37,6 +37,8 @@ class Map {
     void SearchMap();
     void SortNodes();
 
+    bool isFrustum(const glm::vec3& position, float radius);
+
     /*
         Variables
     */

src/Onnx.cpp

@@ -245,12 +245,7 @@ void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo)
     pixels.allocate(fbo.getWidth(), fbo.getHeight(), OF_PIXELS_GRAY);
 
     // Copy data from resizedMat to ofPixels
-    for (size_t y = 0; y < fbo.getHeight(); ++y) {
-        for (size_t x = 0; x < fbo.getWidth(); ++x) {
-            unsigned char value = resizedMat.at<uchar>(y, x);
-            pixels.setColor(x, y, ofColor(value));
-        }
-    }
+    memcpy(pixels.getData(), resizedMat.data, fbo.getWidth() * fbo.getHeight());
     
     // Update FBO with new pixels
     fbo.begin();

src/Onnx.h

@@ -33,7 +33,7 @@
         void Normalize(float* data, size_t size, float min_value, float max_value);
         void DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo);
         void Softmax(float* data, size_t size);
-        bool timeStamp = false;
+        bool timeStamp = true;
         bool log = false;
     
     protected:

src/ofApp.cpp

@@ -3,28 +3,31 @@
 //--------------------------------------------------------------
 void ofApp::setup(){
     ofDisableArbTex();
-    ofSetFrameRate(24);
-    ofSetVerticalSync(true);
+    ofSetFrameRate(60);
+    // ofSetVerticalSync(true);
+
+    tf.load("data/fonts/jetbrainsmono-regular.ttf", 20);
 
     map.Setup();
 
-    player.Setup();
-    player.SetVideo("videos/demo.mp4", fbo);
+    //player.Setup();
+    //player.SetVideo("videos/demo.mp4", fbo);
 
-    emoteImage.allocate(260, 260);
-    tempImage.allocate(emoteImage.getWidth(), emoteImage.getHeight(), OF_IMAGE_COLOR);
+    //emoteImage.allocate(260, 260);
+    //tempImage.allocate(emoteImage.getWidth(), emoteImage.getHeight(), OF_IMAGE_COLOR);
 
     ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/depth_anything_v2_vitb.onnx";
     ORTCHAR_T* modelPath2 = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/yolov5s-face.onnx";
     ORTCHAR_T* modelPath3 = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/rgb_emotion.onnx";
 
     /* Setup Models (modelPath, log, useCuda)   */
-    yolo.Setup(modelPath2, false, true);
+    //yolo.Setup(modelPath2, false, true);
     depth.Setup(modelPath, false, true);
-    emotion.Setup(modelPath3, false, true);
+    //emotion.Setup(modelPath3, false, true);
 
     /* Load shader, allocated rampedFbo */
     depthToColourShader.load("data/shader/rampShader.vert", "data/shader/rampShader.frag");
+    fbo.allocate(1600, 800, GL_RGB);
     rampedFbo.allocate(1600, 800);
 }
 
@@ -34,6 +37,7 @@ void ofApp::setup(){
 void ofApp::update(){
     /*  Check to see if the application has moved to the first frame    
         As the models need to load first, as the first inference is quite slow  */ 
+    auto start = std::chrono::high_resolution_clock::now();
     if(ofGetFrameNum() > 0)
         firstRun = false;
 
@@ -48,11 +52,13 @@ void ofApp::update(){
     }
     
     /* Setup model input using ofImage, allocated fbo */
-    player.Update(img);
-    img.setFromPixels(player.GetVideoPixels());
+    //player.Update(img);
+    //img.setFromPixels(player.GetVideoPixels());
 
     /* Run Models */
     try{
+
+        
         auto output_tensors = depth.Run(map.fboImage);
         float* output_ptr = output_tensors.front().GetTensorMutableData<float>();
         size_t num_elements = output_tensors.front().GetTensorTypeAndShapeInfo().GetElementCount();
@@ -64,17 +70,17 @@ void ofApp::update(){
 
         depth.DataToFbo(output_ptr, 518, 518, fbo);
 
-        auto output_tensors_face = yolo.Run(map.fboImage);
+        // auto output_tensors_face = yolo.Run(map.fboImage);
 
-        auto output_faces = output_tensors_face.front().GetTensorTypeAndShapeInfo().GetShape();
+        // auto output_faces = output_tensors_face.front().GetTensorTypeAndShapeInfo().GetShape();
 
-        unsigned int num_anchors = output_faces[1];  // Number of anchors
+        // unsigned int num_anchors = output_faces[1];  // Number of anchors
 
-        float* output_face_ptr = output_tensors_face.front().GetTensorMutableData<float>();
+        // float* output_face_ptr = output_tensors_face.front().GetTensorMutableData<float>();
 
-        faceDetector.ParseOutput(output_face_ptr, detected_faces, num_anchors);
+        // faceDetector.ParseOutput(output_face_ptr, detected_faces, num_anchors);
 
-        faceDetector.ConvertBoxCoordsToOriginalSize(detected_faces, fbo.getWidth(), fbo.getHeight());
+        // faceDetector.ConvertBoxCoordsToOriginalSize(detected_faces, fbo.getWidth(), fbo.getHeight());
 
         /* As no input is generated for the emotion recognition model, run a dummy vector through the model
             So it can load */
@@ -86,13 +92,13 @@ void ofApp::update(){
                 If the batch_size does change it will completely slow down inference, due to how the cudnn_search_algo is set.
                 None of the other search alogithms bar EXHAUSTIVE will work.. no idea why.
             */
-            for(int i = 0; i < emotionImageMaxBatchSize; i++){
-                tempImage.setFromPixels(emoteImage.getPixels());
-                croppedFaces.push_back(tempImage);
-            }
+            // for(int i = 0; i < emotionImageMaxBatchSize; i++){
+            //     tempImage.setFromPixels(emoteImage.getPixels());
+            //     croppedFaces.push_back(tempImage);
+            // }
 
             // Run model to warmup
-            auto emotion_output_tensor = emotion.RunBatch(croppedFaces);
+            // auto emotion_output_tensor = emotion.RunBatch(croppedFaces);
 
         } else {
             //inferEmotionalState();
@@ -107,23 +113,39 @@ void ofApp::update(){
 
     }
 
+    auto end = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<float> duration = end - start;
+    std::cout << "Time taken for Update: " << duration.count() << " seconds" << std::endl;
+
 }
 
 
 //--------------------------------------------------------------
 void ofApp::draw(){
+    auto start = std::chrono::high_resolution_clock::now();
     map.Draw();
+
     renderDepthMap();
 
-    if(!firstRun){
-        faceDetector.DrawBox(detected_faces);
-        faceDetector.DrawCenter(detected_faces);
-    }
+    // if(!firstRun){
+    //     faceDetector.DrawBox(detected_faces);
+    //     faceDetector.DrawCenter(detected_faces);
+    // }
+
+    ofPushMatrix();
+    ofSetColor(255);
+    ofSetBackgroundColor(0);
+    tf.drawString(std::to_string(ofGetFrameRate()), 10, 30);
+    ofPopMatrix();
 //   emoteImage.draw(640, 0);
 //   for(auto& face : detected_faces){
 //     ofDrawBitmapString(std::to_string(face.box.emotional_state.emotions[0]), 700, 300);
 //   }
 
+    auto end = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<float> duration = end - start;
+    std::cout << "Time taken for Draw: " << duration.count() << " seconds" << std::endl;
+
 }
 
 //--------------------------------------------------------------
@@ -185,9 +207,9 @@ void ofApp::renderDepthMap(){
 
 //--------------------------------------------------------------
 void ofApp::keyPressed(int key){
-    if (key=OF_KEY_LEFT){
-        player.SetRandomFrame();
-    }
+    // if (key=OF_KEY_LEFT){
+    //     player.SetRandomFrame();
+    // }
 }
 
 //--------------------------------------------------------------

src/ofApp.h

@@ -8,6 +8,8 @@
 #include <vector>
 #include "Player.h"
 #include "Map.h"
+#include <chrono>
+#include <iostream>
 
 class ofApp : public ofBaseApp{
 
@@ -53,4 +55,6 @@ class ofApp : public ofBaseApp{
 
 		ofShader depthToColourShader;
 		ofFbo rampedFbo;
+
+		ofTrueTypeFont tf;
 };