B&W Shader added; Fixed face & emotion detection

bin/data/shaders/grayscale/grayscale.frag

@@ -0,0 +1,14 @@
+OF_GLSL_SHADER_HEADER
+precision mediump float;
+
+uniform sampler2DRect tex0;
+in vec2 varyingtexcoord;
+out vec4 fragColor;
+
+
+
+void main() {
+    vec4 texColor = texture(tex0, varyingtexcoord);
+    float gray = dot(texColor.rgb, vec3(0.299, 0.587, 0.114)); // Standard luminance formula
+    fragColor = vec4(vec3(gray), 1.0);
+}

bin/data/shaders/grayscale/grayscale.vert

@@ -0,0 +1,17 @@
+OF_GLSL_SHADER_HEADER
+
+uniform mat4 modelViewMatrix;
+uniform mat4 projectionMatrix;
+uniform mat4 textureMatrix;
+uniform mat4 modelViewProjectionMatrix;
+
+in vec4 position;
+in vec2 texcoord;
+
+out vec2 varyingtexcoord;
+
+void main()
+{
+    varyingtexcoord = texcoord;
+    gl_Position = modelViewProjectionMatrix * position;
+}

src/main.cpp

@@ -6,6 +6,7 @@ int main( ){
 
 	//Use ofGLFWWindowSettings for more options like multi-monitor fullscreen
 	ofGLWindowSettings settings;
+	settings.setGLVersion(3, 2);
 	settings.setSize(1920, 1080);
 	settings.windowMode = OF_WINDOW; //can also be OF_FULLSCREEN
 

src/ofApp.cpp

@@ -3,6 +3,9 @@
 //--------------------------------------------------------------
 void ofApp::setup(){
     ofSetFrameRate(60);
+    //ofDisableArbTex();
+    //ofEnableDepthTest();
+    ofEnableAlphaBlending();
 
     videoPlayer.load(videoPath);
     videoPlayer.setLoopState(OF_LOOP_NORMAL);
@@ -22,6 +25,7 @@ void ofApp::update(){
         videoFrame.setFromPixels(p);
     }
     onnx.update(videoFrame);
+    ofLog() << ofGetFrameRate();
 }
 
 //--------------------------------------------------------------

src/ofYolo.cpp

@@ -25,7 +25,7 @@ void ofYolo::ParseOutput(float* &output_tensors, std::vector<BoxfWithLandmarks>
         face.box.y1 = cy - h / 2;
         face.box.x2 = cx + w / 2;
         face.box.y2 = cy + h / 2;
-        face.box.score = cls_conf;
+        face.box.score = cls_conf * obj_conf;
         face.box.label_text = "face";
         
         // Extract landmarks
@@ -44,13 +44,59 @@ void ofYolo::ParseOutput(float* &output_tensors, std::vector<BoxfWithLandmarks>
 }
 
 // Simple helper for drawing boxes given x1, y1, x2, y2 coordinates.
-void ofYolo::DrawBox(std::vector<BoxfWithLandmarks> &detected_faces){
+void ofYolo::DrawBox(std::vector<BoxfWithLandmarks> &detected_faces, int limit){
+    int count = 0;
     for (const auto &face : detected_faces) {
+        if (count >= limit) break;
         ofPushStyle();
         ofNoFill();
-        ofSetColor(ofColor::cyan);
-        ofDrawRectangle(face.box.x1, face.box.y1, ((face.box.x2) - (face.box.x1)), face.box.y2 - face.box.y1);
+        ofSetColor(ofColor::red);
+        float width = face.box.x2 - face.box.x1;
+        float height = face.box.y2 - face.box.y1;
+        float side = std::max(width, height);
+
+        // Center of the box
+        float centerX = (face.box.x1 + face.box.x2) / 2.0f;
+        float centerY = (face.box.y1 + face.box.y2) / 2.0f;
+
+        // Top-left corner of the square
+        float x = centerX - side / 2.0f;
+        float y = centerY - side / 2.0f;
+
+        ofDrawRectangle(x, y, side, side);
+        ofPopStyle();
+        count++;
+    }
+}
+
+void ofYolo::DrawBoxLabels(const std::vector<BoxfWithLandmarks>& detected_faces, int limit) {
+    int count = 0;
+    for (const auto& face : detected_faces) {
+        if (count >= limit) break;
+        float width = face.box.x2 - face.box.x1;
+        float height = face.box.y2 - face.box.y1;
+        float side = std::max(width, height);
+
+        float centerX = (face.box.x1 + face.box.x2) / 2.0f;
+        float centerY = (face.box.y1 + face.box.y2) / 2.0f;
+        float x = centerX - side / 2.0f;
+        float y = centerY - side / 2.0f;
+
+        float margin = 4.0f;
+        float textX = x + side - margin;
+        float textY = y + margin + 12;
+
+        std::string label = "Score: " + ofToString(face.box.score, 2) +
+                            "\n" + face.emotion.getDominantEmotion();
+
+        // Calculate bounding box width for right alignment
+        ofRectangle bbox = ofBitmapFont().getBoundingBox(label, 0, 0);
+
+        ofPushStyle();
+        ofSetColor(ofColor::yellow);
+        ofDrawBitmapStringHighlight(label, textX - bbox.getWidth(), textY);
         ofPopStyle();
+        count++;
     }
 }
 
@@ -170,6 +216,7 @@ void ofYolo::CropFaceToImage(ofImage &inputImage, BoxfWithLandmarks &face, ofIma
 void ofYolo::SortDetectedFaces(std::vector<BoxfWithLandmarks> &detectedFaces){
     std::sort(detectedFaces.begin(), detectedFaces.end(), 
     [](const BoxfWithLandmarks &a, const BoxfWithLandmarks &b) {
-        return a.box.center.x > b.box.center.x;  // Sort in descending order
+        //return a.box.center.x > b.box.center.x;  // Sort in descending order
+        return a.box.score > b.box.score;
     });
 }

src/ofYolo.h

@@ -116,12 +116,13 @@ class ofYolo{
     public:
         ofYolo(){};
         void ParseOutput(float* &out_ptr, std::vector<BoxfWithLandmarks> &sorted_faces, unsigned int num_anchors);
-        void DrawBox(std::vector<BoxfWithLandmarks> &detected_faces);
+        void DrawBox(std::vector<BoxfWithLandmarks> &detected_faces, int limit);
         void DrawCenter(std::vector<BoxfWithLandmarks> &detected_faces);
         void NonMaximumSuppression(std::vector<BoxfWithLandmarks> &input_faces, std::vector<BoxfWithLandmarks> &output_faces, float iou_threshold); 
         void ConvertBoxCoordsToOriginalSize(std::vector<BoxfWithLandmarks> &detected_faces, size_t original_width, size_t original_height);
         void CropFaceToImage(ofImage &inputImage, BoxfWithLandmarks &face, ofImage &colorImage, int imageSize);
         void SortDetectedFaces(std::vector<BoxfWithLandmarks> &detectedFaces);
+        void DrawBoxLabels(const std::vector<BoxfWithLandmarks>& detected_faces, int limit);
     private:
         // Input dimenions of the model -- used for coordinate scaling.
         size_t modelSize = 640;

src/onxProcess.cpp

@@ -25,6 +25,10 @@ void onxProcess::setup(ofImage* img) {
 
     fdThread.setup(&onxFaceDetection);
     fdThread.start();
+
+    // Load Shaders
+    grayscaleShader.load("shaders/grayscale/grayscale");
+    grayscaleRender.allocate(ofGetWidth(), ofGetHeight(), GL_RGBA);
 }
 
 void onxProcess::update(ofImage& frame) {
@@ -99,9 +103,12 @@ void onxProcess::updateEmotions() {
 
 void onxProcess::draw() {
     // Drawing code here
-    //videoFrame.draw(0, 0);
-    //yolo.DrawBox(detectedFaces);
+    convertToGrayscale();
+    //drawGrid();
+    
+}
 
+void onxProcess::drawGrid() {
     int faceSize = modelInputDimensionsFer;
     int facesPerRow = ofGetWindowWidth() / faceSize;
     
@@ -116,30 +123,16 @@ void onxProcess::draw() {
     }
 }
 
-void onxProcess::drawGrid() {
-    ofImage temp;
-    int faceSize = 384;
-    int facesPerRow = ofGetWindowWidth() / faceSize;
-    int totalRows = (detectedFaces.size() + facesPerRow - 1) / facesPerRow;
-
-    for (int row = 0; row < totalRows; ++row) {
-        // Calculate number of faces in this row
-        int startIdx = row * facesPerRow;
-        int endIdx = std::min(startIdx + facesPerRow, (int)detectedFaces.size());
-        int facesInThisRow = endIdx - startIdx;
-
-        // Calculate horizontal offset to center faces in this row
-        int rowWidth = facesInThisRow * faceSize;
-        int xOffset = (ofGetWindowWidth() - rowWidth) / 2;
-
-        for (int i = 0; i < facesInThisRow; ++i) {
-            int idx = startIdx + i;
-            yolo.CropFaceToImage(videoFrame, detectedFaces[idx], temp, faceSize);
-
-            int x = xOffset + i * faceSize;
-            int y = row * faceSize;
-
-            temp.draw(x, y);
-        }
-    }
+void onxProcess::convertToGrayscale() {
+    grayscaleRender.begin();
+    ofClear(0, 0, 0, 255);
+    grayscaleShader.begin();
+    grayscaleShader.setUniformTexture("tex0", videoFrame.getTexture(), 0);
+    videoFrame.draw(0, 0, videoFrame.getWidth(), videoFrame.getHeight());
+    grayscaleShader.end();
+    yolo.DrawBox(detectedFaces, batchSizeFer);
+    yolo.DrawBoxLabels(detectedFaces, batchSizeFer);
+    grayscaleRender.end();
+
+    grayscaleRender.draw(0, 0);
 }

src/onxProcess.h

@@ -13,6 +13,7 @@ public:
     void updateEmotions();
     void draw();
     void drawGrid();
+    void convertToGrayscale();
 
     ofxOnnxRuntime::BaseHandler onxFaceDetection;
     ofxOnnxRuntime::OnnxThread fdThread;
@@ -33,7 +34,7 @@ public:
     int modelInputHeight = 640;
 
     int modelInputDimensionsFer = 260;
-    int batchSizeFer = 2;
+    int batchSizeFer = 4;
 
     std::string modelPath = "yolov5s-face.onnx";
     std::string modelPathFer = "fer.onnx";
@@ -41,4 +42,6 @@ public:
     std::vector<BoxfWithLandmarks> detectedFaces;
 
     ofImage videoFrame;
+    ofShader grayscaleShader;
+    ofFbo grayscaleRender;
 };