diff --git a/src/ofApp.cpp b/src/ofApp.cpp index 701470d..9e44cf1 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -26,7 +26,8 @@ void ofApp::update(){ } onnx.update(videoFrame); - teleprompter->updateCVData(onnx.detectedFaces.size()); + // Get the intensity, emotion of highest intensity, and number of people detected + teleprompter->updateCVData(onnx.detectedFaces.size(), onnx.dominantEmotion, onnx.highestEmotionIntensity); ofLog() << ofGetFrameRate(); } diff --git a/src/ofTeleprompter.cpp b/src/ofTeleprompter.cpp index da60951..94b6f88 100644 --- a/src/ofTeleprompter.cpp +++ b/src/ofTeleprompter.cpp @@ -21,7 +21,7 @@ void ofTeleprompter::update() { // set labels currentSpeaker = script[currentLine].speaker; currentEmotion = script[currentLine].emotion; - currentLineIndex = ofToString(currentLine); + currentLineIndex = ofToString(currentLine + 1) + " / " + ofToString(script.size() + 1); // Teleprompter logic (letter by letter) if (currentLetterIndex < currentSentence.size()) { @@ -42,6 +42,7 @@ void ofTeleprompter::draw() { void ofTeleprompter::setupGUI() { nextLine.addListener(this, &ofTeleprompter::nextLinePressed); + reset.addListener(this, &ofTeleprompter::resetScript); gui.setDefaultWidth(400); gui.setup(); @@ -55,17 +56,7 @@ void ofTeleprompter::setupGUI() { gui.add(useLLMOnly.setup("Use LLM Only", false)); gui.add(useTextOnly.setup("Use Text Only", false)); gui.add(nextLine.setup("Next Line")); -} - -void ofTeleprompter::nextLinePressed() { - ofLog() << "Next Line!"; - currentLine++; - - // Prepare teleprompter effect for letter-by-letter - currentSentence = script[currentLine].sentence; - displayedSentence.clear(); - currentLetterIndex = 0; - lastWordTime = ofGetElapsedTimeMillis(); + gui.add(reset.setup("Reset Script")); } void ofTeleprompter::loadText() { @@ -165,6 +156,35 @@ std::string ofTeleprompter::wrapStringToWidth(const std::string& text, float max return wrapped; } -void ofTeleprompter::updateCVData(int numOfFacesDetected) { +void ofTeleprompter::updateCVData(int numOfFacesDetected, std::string emotion, float intensity) { facesDetected = ofToString(numOfFacesDetected); + emotionDetected = emotion; + emotionIntensity = ofToString(intensity); +} + +void ofTeleprompter::nextLinePressed() { + ofLog() << "Next Line!"; + + // Check if it exceeds the length of the script + if (currentLine < script.size()){ + currentLine++; + } + + // Prepare teleprompter effect for letter-by-letter + currentSentence = script[currentLine].sentence; + displayedSentence.clear(); + currentLetterIndex = 0; + lastWordTime = ofGetElapsedTimeMillis(); +} + +void ofTeleprompter::resetScript() { + // Need to reset the text, id, etc. + ofLog() << "Reset script."; + currentLine = 0; + + // Prepare teleprompter effect for letter-by-letter + currentSentence = script[currentLine].sentence; + displayedSentence.clear(); + currentLetterIndex = 0; + lastWordTime = ofGetElapsedTimeMillis(); } \ No newline at end of file diff --git a/src/ofTeleprompter.h b/src/ofTeleprompter.h index ab92048..89ca2c9 100644 --- a/src/ofTeleprompter.h +++ b/src/ofTeleprompter.h @@ -16,9 +16,10 @@ class ofTeleprompter: public ofBaseApp{ void draw(); void setupGUI(); void nextLinePressed(); + void resetScript(); void loadText(); void drawText(); - void updateCVData(int numOfFacesDetected); + void updateCVData(int numOfFacesDetected, std::string emotion, float intensity); std::string wrapStringToWidth(const std::string& text, float maxWidth); @@ -29,6 +30,7 @@ class ofTeleprompter: public ofBaseApp{ ofxButton nextLine; ofxToggle useLLMOnly; ofxToggle useTextOnly; + ofxButton reset; ofxLabel currentLineIndex; ofxLabel currentSpeaker; diff --git a/src/onxProcess.cpp b/src/onxProcess.cpp index c871790..f1987c8 100644 --- a/src/onxProcess.cpp +++ b/src/onxProcess.cpp @@ -58,6 +58,10 @@ void onxProcess::update(ofImage& frame) { updateEmotions(); + if (detectedFaces.size() != 0) { + setTeleprompterValues(); + } + try{ fdThread.update(); } catch (_exception e) { @@ -137,4 +141,30 @@ void onxProcess::convertToGrayscale() { grayscaleRender.end(); grayscaleRender.draw(0, 0); +} + +void onxProcess::setTeleprompterValues() { + // Figure out highest intensity & Get that emotion index + float maxIntensity = 0.0f; + int maxEmotionIdx = -1; + size_t maxFaceIdx = 0; + + + for (size_t i = 0; i < detectedFaces.size() & i < inputImagesFer.size(); i++) { + const Emotion& e = detectedFaces[i].emotion; + float emotions[7] = {e.anger, e.disgust, e.fear, e.happiness, e.neutral, e.sadness, e.surprise}; + for (int j = 0; j < 7; j++) { + if (emotions[j] > maxIntensity) { + maxIntensity = emotions[j]; + maxEmotionIdx = j; + maxFaceIdx = i; + } + } + } + + const char* emotions[7] = {"anger", "disgust", "fear", "happiness", "neutral", "sadness", "surprise"}; + std::string maxEmotionName = emotions[maxEmotionIdx]; + + dominantEmotion = maxEmotionName; + highestEmotionIntensity = maxIntensity; } \ No newline at end of file diff --git a/src/onxProcess.h b/src/onxProcess.h index b5680fb..de1e79c 100644 --- a/src/onxProcess.h +++ b/src/onxProcess.h @@ -14,6 +14,7 @@ public: void draw(); void drawGrid(); void convertToGrayscale(); + void setTeleprompterValues(); ofxOnnxRuntime::BaseHandler onxFaceDetection; ofxOnnxRuntime::OnnxThread fdThread; @@ -44,4 +45,7 @@ public: ofImage videoFrame; ofShader grayscaleShader; ofFbo grayscaleRender; + + float highestEmotionIntensity = 0; + std::string dominantEmotion = "n/a"; }; \ No newline at end of file