diff --git a/src/Map.cpp b/src/Map.cpp index daa8118..d3a59eb 100644 --- a/src/Map.cpp +++ b/src/Map.cpp @@ -42,14 +42,23 @@ void Map::Draw(){ ofClear(0, 0, 0, 1); + ofMatrix4x4 projectionMatrix = cam.getProjectionMatrix(); + ofMatrix4x4 viewMatrix = cam.getModelViewMatrix(); + ofMatrix4x4 mvpMatrix = projectionMatrix * viewMatrix; + for (auto& n :nodes){ + glm::vec3 node_position = n.position + n.offset; + + if(isFrustum(node_position, 50)){ n.texture.getTexture().bind(); ofPushMatrix(); ofFill(); - n.geom.setPosition(n.position + n.offset); + n.geom.setPosition(node_position); n.geom.draw(); ofPopMatrix(); n.texture.getTexture().unbind(); + } + } cam.end(); @@ -188,4 +197,22 @@ void Map::SortNodes(){ return a.position.z < b.position.z; }); std::cout << "Finished Sorting!" << std::endl; +} + +bool Map::isFrustum(const glm::vec3& position, float radius){ + float box_w = 2000; + float box_h = 2000; + glm::vec3 cam_position = cam.getPosition(); + + float left = cam_position.x - box_w / 2.0f; + float right = cam_position.x + box_w / 2.0f; + float top = cam_position.y + box_h / 2.0f; + float bottom = cam_position.y - box_h / 2.0f; + + // Check if the object (with radius) is within the bounding box + if (position.x + radius < left || position.x - radius > right) return false; + if (position.y + radius < bottom || position.y - radius > top) return false; + + // Z-depth doesn't matter in this approach, so we're ignoring it + return true; } \ No newline at end of file diff --git a/src/Map.h b/src/Map.h index 44f8d92..ff974a0 100644 --- a/src/Map.h +++ b/src/Map.h @@ -37,6 +37,8 @@ class Map { void SearchMap(); void SortNodes(); + bool isFrustum(const glm::vec3& position, float radius); + /* Variables */ diff --git a/src/Onnx.cpp b/src/Onnx.cpp index 64d2e5c..c709c6a 100644 --- a/src/Onnx.cpp +++ b/src/Onnx.cpp @@ -245,13 +245,8 @@ void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo) pixels.allocate(fbo.getWidth(), fbo.getHeight(), OF_PIXELS_GRAY); // Copy data from resizedMat to ofPixels - for (size_t y = 0; y < fbo.getHeight(); ++y) { - for (size_t x = 0; x < fbo.getWidth(); ++x) { - unsigned char value = resizedMat.at(y, x); - pixels.setColor(x, y, ofColor(value)); - } - } - + memcpy(pixels.getData(), resizedMat.data, fbo.getWidth() * fbo.getHeight()); + // Update FBO with new pixels fbo.begin(); ofTexture& texture = fbo.getTexture(); diff --git a/src/Onnx.h b/src/Onnx.h index 0ce95d8..8e17c51 100644 --- a/src/Onnx.h +++ b/src/Onnx.h @@ -33,7 +33,7 @@ void Normalize(float* data, size_t size, float min_value, float max_value); void DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo); void Softmax(float* data, size_t size); - bool timeStamp = false; + bool timeStamp = true; bool log = false; protected: diff --git a/src/ofApp.cpp b/src/ofApp.cpp index 09c4339..d2c8a0d 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -3,28 +3,31 @@ //-------------------------------------------------------------- void ofApp::setup(){ ofDisableArbTex(); - ofSetFrameRate(24); - ofSetVerticalSync(true); + ofSetFrameRate(60); + // ofSetVerticalSync(true); + + tf.load("data/fonts/jetbrainsmono-regular.ttf", 20); map.Setup(); - player.Setup(); - player.SetVideo("videos/demo.mp4", fbo); + //player.Setup(); + //player.SetVideo("videos/demo.mp4", fbo); - emoteImage.allocate(260, 260); - tempImage.allocate(emoteImage.getWidth(), emoteImage.getHeight(), OF_IMAGE_COLOR); + //emoteImage.allocate(260, 260); + //tempImage.allocate(emoteImage.getWidth(), emoteImage.getHeight(), OF_IMAGE_COLOR); ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/depth_anything_v2_vitb.onnx"; ORTCHAR_T* modelPath2 = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/yolov5s-face.onnx"; ORTCHAR_T* modelPath3 = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/rgb_emotion.onnx"; /* Setup Models (modelPath, log, useCuda) */ - yolo.Setup(modelPath2, false, true); + //yolo.Setup(modelPath2, false, true); depth.Setup(modelPath, false, true); - emotion.Setup(modelPath3, false, true); + //emotion.Setup(modelPath3, false, true); /* Load shader, allocated rampedFbo */ depthToColourShader.load("data/shader/rampShader.vert", "data/shader/rampShader.frag"); + fbo.allocate(1600, 800, GL_RGB); rampedFbo.allocate(1600, 800); } @@ -34,6 +37,7 @@ void ofApp::setup(){ void ofApp::update(){ /* Check to see if the application has moved to the first frame As the models need to load first, as the first inference is quite slow */ + auto start = std::chrono::high_resolution_clock::now(); if(ofGetFrameNum() > 0) firstRun = false; @@ -48,11 +52,13 @@ void ofApp::update(){ } /* Setup model input using ofImage, allocated fbo */ - player.Update(img); - img.setFromPixels(player.GetVideoPixels()); + //player.Update(img); + //img.setFromPixels(player.GetVideoPixels()); /* Run Models */ try{ + + auto output_tensors = depth.Run(map.fboImage); float* output_ptr = output_tensors.front().GetTensorMutableData(); size_t num_elements = output_tensors.front().GetTensorTypeAndShapeInfo().GetElementCount(); @@ -64,17 +70,17 @@ void ofApp::update(){ depth.DataToFbo(output_ptr, 518, 518, fbo); - auto output_tensors_face = yolo.Run(map.fboImage); + // auto output_tensors_face = yolo.Run(map.fboImage); - auto output_faces = output_tensors_face.front().GetTensorTypeAndShapeInfo().GetShape(); + // auto output_faces = output_tensors_face.front().GetTensorTypeAndShapeInfo().GetShape(); - unsigned int num_anchors = output_faces[1]; // Number of anchors + // unsigned int num_anchors = output_faces[1]; // Number of anchors - float* output_face_ptr = output_tensors_face.front().GetTensorMutableData(); + // float* output_face_ptr = output_tensors_face.front().GetTensorMutableData(); - faceDetector.ParseOutput(output_face_ptr, detected_faces, num_anchors); + // faceDetector.ParseOutput(output_face_ptr, detected_faces, num_anchors); - faceDetector.ConvertBoxCoordsToOriginalSize(detected_faces, fbo.getWidth(), fbo.getHeight()); + // faceDetector.ConvertBoxCoordsToOriginalSize(detected_faces, fbo.getWidth(), fbo.getHeight()); /* As no input is generated for the emotion recognition model, run a dummy vector through the model So it can load */ @@ -86,13 +92,13 @@ void ofApp::update(){ If the batch_size does change it will completely slow down inference, due to how the cudnn_search_algo is set. None of the other search alogithms bar EXHAUSTIVE will work.. no idea why. */ - for(int i = 0; i < emotionImageMaxBatchSize; i++){ - tempImage.setFromPixels(emoteImage.getPixels()); - croppedFaces.push_back(tempImage); - } + // for(int i = 0; i < emotionImageMaxBatchSize; i++){ + // tempImage.setFromPixels(emoteImage.getPixels()); + // croppedFaces.push_back(tempImage); + // } // Run model to warmup - auto emotion_output_tensor = emotion.RunBatch(croppedFaces); + // auto emotion_output_tensor = emotion.RunBatch(croppedFaces); } else { //inferEmotionalState(); @@ -107,23 +113,39 @@ void ofApp::update(){ } + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + std::cout << "Time taken for Update: " << duration.count() << " seconds" << std::endl; + } //-------------------------------------------------------------- void ofApp::draw(){ + auto start = std::chrono::high_resolution_clock::now(); map.Draw(); + renderDepthMap(); - if(!firstRun){ - faceDetector.DrawBox(detected_faces); - faceDetector.DrawCenter(detected_faces); - } + // if(!firstRun){ + // faceDetector.DrawBox(detected_faces); + // faceDetector.DrawCenter(detected_faces); + // } + + ofPushMatrix(); + ofSetColor(255); + ofSetBackgroundColor(0); + tf.drawString(std::to_string(ofGetFrameRate()), 10, 30); + ofPopMatrix(); // emoteImage.draw(640, 0); // for(auto& face : detected_faces){ // ofDrawBitmapString(std::to_string(face.box.emotional_state.emotions[0]), 700, 300); // } + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + std::cout << "Time taken for Draw: " << duration.count() << " seconds" << std::endl; + } //-------------------------------------------------------------- @@ -185,9 +207,9 @@ void ofApp::renderDepthMap(){ //-------------------------------------------------------------- void ofApp::keyPressed(int key){ - if (key=OF_KEY_LEFT){ - player.SetRandomFrame(); - } + // if (key=OF_KEY_LEFT){ + // player.SetRandomFrame(); + // } } //-------------------------------------------------------------- diff --git a/src/ofApp.h b/src/ofApp.h index 66a5be6..df45979 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -8,6 +8,8 @@ #include #include "Player.h" #include "Map.h" +#include +#include class ofApp : public ofBaseApp{ @@ -53,4 +55,6 @@ class ofApp : public ofBaseApp{ ofShader depthToColourShader; ofFbo rampedFbo; + + ofTrueTypeFont tf; };