Browse Source

B&W Shader added; Fixed face & emotion detection

main
cailean 4 days ago
parent
commit
698ebae59c
  1. 14
      bin/data/shaders/grayscale/grayscale.frag
  2. 17
      bin/data/shaders/grayscale/grayscale.vert
  3. 1
      src/main.cpp
  4. 4
      src/ofApp.cpp
  5. 57
      src/ofYolo.cpp
  6. 3
      src/ofYolo.h
  7. 49
      src/onxProcess.cpp
  8. 5
      src/onxProcess.h

14
bin/data/shaders/grayscale/grayscale.frag

@ -0,0 +1,14 @@
OF_GLSL_SHADER_HEADER
precision mediump float;
uniform sampler2DRect tex0;
in vec2 varyingtexcoord;
out vec4 fragColor;
void main() {
vec4 texColor = texture(tex0, varyingtexcoord);
float gray = dot(texColor.rgb, vec3(0.299, 0.587, 0.114)); // Standard luminance formula
fragColor = vec4(vec3(gray), 1.0);
}

17
bin/data/shaders/grayscale/grayscale.vert

@ -0,0 +1,17 @@
OF_GLSL_SHADER_HEADER
uniform mat4 modelViewMatrix;
uniform mat4 projectionMatrix;
uniform mat4 textureMatrix;
uniform mat4 modelViewProjectionMatrix;
in vec4 position;
in vec2 texcoord;
out vec2 varyingtexcoord;
void main()
{
varyingtexcoord = texcoord;
gl_Position = modelViewProjectionMatrix * position;
}

1
src/main.cpp

@ -6,6 +6,7 @@ int main( ){
//Use ofGLFWWindowSettings for more options like multi-monitor fullscreen //Use ofGLFWWindowSettings for more options like multi-monitor fullscreen
ofGLWindowSettings settings; ofGLWindowSettings settings;
settings.setGLVersion(3, 2);
settings.setSize(1920, 1080); settings.setSize(1920, 1080);
settings.windowMode = OF_WINDOW; //can also be OF_FULLSCREEN settings.windowMode = OF_WINDOW; //can also be OF_FULLSCREEN

4
src/ofApp.cpp

@ -3,6 +3,9 @@
//-------------------------------------------------------------- //--------------------------------------------------------------
void ofApp::setup(){ void ofApp::setup(){
ofSetFrameRate(60); ofSetFrameRate(60);
//ofDisableArbTex();
//ofEnableDepthTest();
ofEnableAlphaBlending();
videoPlayer.load(videoPath); videoPlayer.load(videoPath);
videoPlayer.setLoopState(OF_LOOP_NORMAL); videoPlayer.setLoopState(OF_LOOP_NORMAL);
@ -22,6 +25,7 @@ void ofApp::update(){
videoFrame.setFromPixels(p); videoFrame.setFromPixels(p);
} }
onnx.update(videoFrame); onnx.update(videoFrame);
ofLog() << ofGetFrameRate();
} }
//-------------------------------------------------------------- //--------------------------------------------------------------

57
src/ofYolo.cpp

@ -25,7 +25,7 @@ void ofYolo::ParseOutput(float* &output_tensors, std::vector<BoxfWithLandmarks>
face.box.y1 = cy - h / 2; face.box.y1 = cy - h / 2;
face.box.x2 = cx + w / 2; face.box.x2 = cx + w / 2;
face.box.y2 = cy + h / 2; face.box.y2 = cy + h / 2;
face.box.score = cls_conf; face.box.score = cls_conf * obj_conf;
face.box.label_text = "face"; face.box.label_text = "face";
// Extract landmarks // Extract landmarks
@ -44,13 +44,59 @@ void ofYolo::ParseOutput(float* &output_tensors, std::vector<BoxfWithLandmarks>
} }
// Simple helper for drawing boxes given x1, y1, x2, y2 coordinates. // Simple helper for drawing boxes given x1, y1, x2, y2 coordinates.
void ofYolo::DrawBox(std::vector<BoxfWithLandmarks> &detected_faces){ void ofYolo::DrawBox(std::vector<BoxfWithLandmarks> &detected_faces, int limit){
int count = 0;
for (const auto &face : detected_faces) { for (const auto &face : detected_faces) {
if (count >= limit) break;
ofPushStyle(); ofPushStyle();
ofNoFill(); ofNoFill();
ofSetColor(ofColor::cyan); ofSetColor(ofColor::red);
ofDrawRectangle(face.box.x1, face.box.y1, ((face.box.x2) - (face.box.x1)), face.box.y2 - face.box.y1); float width = face.box.x2 - face.box.x1;
float height = face.box.y2 - face.box.y1;
float side = std::max(width, height);
// Center of the box
float centerX = (face.box.x1 + face.box.x2) / 2.0f;
float centerY = (face.box.y1 + face.box.y2) / 2.0f;
// Top-left corner of the square
float x = centerX - side / 2.0f;
float y = centerY - side / 2.0f;
ofDrawRectangle(x, y, side, side);
ofPopStyle(); ofPopStyle();
count++;
}
}
void ofYolo::DrawBoxLabels(const std::vector<BoxfWithLandmarks>& detected_faces, int limit) {
int count = 0;
for (const auto& face : detected_faces) {
if (count >= limit) break;
float width = face.box.x2 - face.box.x1;
float height = face.box.y2 - face.box.y1;
float side = std::max(width, height);
float centerX = (face.box.x1 + face.box.x2) / 2.0f;
float centerY = (face.box.y1 + face.box.y2) / 2.0f;
float x = centerX - side / 2.0f;
float y = centerY - side / 2.0f;
float margin = 4.0f;
float textX = x + side - margin;
float textY = y + margin + 12;
std::string label = "Score: " + ofToString(face.box.score, 2) +
"\n" + face.emotion.getDominantEmotion();
// Calculate bounding box width for right alignment
ofRectangle bbox = ofBitmapFont().getBoundingBox(label, 0, 0);
ofPushStyle();
ofSetColor(ofColor::yellow);
ofDrawBitmapStringHighlight(label, textX - bbox.getWidth(), textY);
ofPopStyle();
count++;
} }
} }
@ -170,6 +216,7 @@ void ofYolo::CropFaceToImage(ofImage &inputImage, BoxfWithLandmarks &face, ofIma
void ofYolo::SortDetectedFaces(std::vector<BoxfWithLandmarks> &detectedFaces){ void ofYolo::SortDetectedFaces(std::vector<BoxfWithLandmarks> &detectedFaces){
std::sort(detectedFaces.begin(), detectedFaces.end(), std::sort(detectedFaces.begin(), detectedFaces.end(),
[](const BoxfWithLandmarks &a, const BoxfWithLandmarks &b) { [](const BoxfWithLandmarks &a, const BoxfWithLandmarks &b) {
return a.box.center.x > b.box.center.x; // Sort in descending order //return a.box.center.x > b.box.center.x; // Sort in descending order
return a.box.score > b.box.score;
}); });
} }

3
src/ofYolo.h

@ -116,12 +116,13 @@ class ofYolo{
public: public:
ofYolo(){}; ofYolo(){};
void ParseOutput(float* &out_ptr, std::vector<BoxfWithLandmarks> &sorted_faces, unsigned int num_anchors); void ParseOutput(float* &out_ptr, std::vector<BoxfWithLandmarks> &sorted_faces, unsigned int num_anchors);
void DrawBox(std::vector<BoxfWithLandmarks> &detected_faces); void DrawBox(std::vector<BoxfWithLandmarks> &detected_faces, int limit);
void DrawCenter(std::vector<BoxfWithLandmarks> &detected_faces); void DrawCenter(std::vector<BoxfWithLandmarks> &detected_faces);
void NonMaximumSuppression(std::vector<BoxfWithLandmarks> &input_faces, std::vector<BoxfWithLandmarks> &output_faces, float iou_threshold); void NonMaximumSuppression(std::vector<BoxfWithLandmarks> &input_faces, std::vector<BoxfWithLandmarks> &output_faces, float iou_threshold);
void ConvertBoxCoordsToOriginalSize(std::vector<BoxfWithLandmarks> &detected_faces, size_t original_width, size_t original_height); void ConvertBoxCoordsToOriginalSize(std::vector<BoxfWithLandmarks> &detected_faces, size_t original_width, size_t original_height);
void CropFaceToImage(ofImage &inputImage, BoxfWithLandmarks &face, ofImage &colorImage, int imageSize); void CropFaceToImage(ofImage &inputImage, BoxfWithLandmarks &face, ofImage &colorImage, int imageSize);
void SortDetectedFaces(std::vector<BoxfWithLandmarks> &detectedFaces); void SortDetectedFaces(std::vector<BoxfWithLandmarks> &detectedFaces);
void DrawBoxLabels(const std::vector<BoxfWithLandmarks>& detected_faces, int limit);
private: private:
// Input dimenions of the model -- used for coordinate scaling. // Input dimenions of the model -- used for coordinate scaling.
size_t modelSize = 640; size_t modelSize = 640;

49
src/onxProcess.cpp

@ -25,6 +25,10 @@ void onxProcess::setup(ofImage* img) {
fdThread.setup(&onxFaceDetection); fdThread.setup(&onxFaceDetection);
fdThread.start(); fdThread.start();
// Load Shaders
grayscaleShader.load("shaders/grayscale/grayscale");
grayscaleRender.allocate(ofGetWidth(), ofGetHeight(), GL_RGBA);
} }
void onxProcess::update(ofImage& frame) { void onxProcess::update(ofImage& frame) {
@ -99,9 +103,12 @@ void onxProcess::updateEmotions() {
void onxProcess::draw() { void onxProcess::draw() {
// Drawing code here // Drawing code here
//videoFrame.draw(0, 0); convertToGrayscale();
//yolo.DrawBox(detectedFaces); //drawGrid();
}
void onxProcess::drawGrid() {
int faceSize = modelInputDimensionsFer; int faceSize = modelInputDimensionsFer;
int facesPerRow = ofGetWindowWidth() / faceSize; int facesPerRow = ofGetWindowWidth() / faceSize;
@ -116,30 +123,16 @@ void onxProcess::draw() {
} }
} }
void onxProcess::drawGrid() { void onxProcess::convertToGrayscale() {
ofImage temp; grayscaleRender.begin();
int faceSize = 384; ofClear(0, 0, 0, 255);
int facesPerRow = ofGetWindowWidth() / faceSize; grayscaleShader.begin();
int totalRows = (detectedFaces.size() + facesPerRow - 1) / facesPerRow; grayscaleShader.setUniformTexture("tex0", videoFrame.getTexture(), 0);
videoFrame.draw(0, 0, videoFrame.getWidth(), videoFrame.getHeight());
for (int row = 0; row < totalRows; ++row) { grayscaleShader.end();
// Calculate number of faces in this row yolo.DrawBox(detectedFaces, batchSizeFer);
int startIdx = row * facesPerRow; yolo.DrawBoxLabels(detectedFaces, batchSizeFer);
int endIdx = std::min(startIdx + facesPerRow, (int)detectedFaces.size()); grayscaleRender.end();
int facesInThisRow = endIdx - startIdx;
grayscaleRender.draw(0, 0);
// Calculate horizontal offset to center faces in this row
int rowWidth = facesInThisRow * faceSize;
int xOffset = (ofGetWindowWidth() - rowWidth) / 2;
for (int i = 0; i < facesInThisRow; ++i) {
int idx = startIdx + i;
yolo.CropFaceToImage(videoFrame, detectedFaces[idx], temp, faceSize);
int x = xOffset + i * faceSize;
int y = row * faceSize;
temp.draw(x, y);
}
}
} }

5
src/onxProcess.h

@ -13,6 +13,7 @@ public:
void updateEmotions(); void updateEmotions();
void draw(); void draw();
void drawGrid(); void drawGrid();
void convertToGrayscale();
ofxOnnxRuntime::BaseHandler onxFaceDetection; ofxOnnxRuntime::BaseHandler onxFaceDetection;
ofxOnnxRuntime::OnnxThread fdThread; ofxOnnxRuntime::OnnxThread fdThread;
@ -33,7 +34,7 @@ public:
int modelInputHeight = 640; int modelInputHeight = 640;
int modelInputDimensionsFer = 260; int modelInputDimensionsFer = 260;
int batchSizeFer = 2; int batchSizeFer = 4;
std::string modelPath = "yolov5s-face.onnx"; std::string modelPath = "yolov5s-face.onnx";
std::string modelPathFer = "fer.onnx"; std::string modelPathFer = "fer.onnx";
@ -41,4 +42,6 @@ public:
std::vector<BoxfWithLandmarks> detectedFaces; std::vector<BoxfWithLandmarks> detectedFaces;
ofImage videoFrame; ofImage videoFrame;
ofShader grayscaleShader;
ofFbo grayscaleRender;
}; };
Loading…
Cancel
Save