cailean
/
MacbethFER
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Added camera support, updated fonts, renamed FER classes, option for multiple scripts, keypresses

main
Cailean 4 weeks ago
parent
1a7818e460
commit
4c45524d66
10 changed files with 1174 additions and 42 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. BIN
      bin/data/Roboto-SemiBold.ttf
  2. BIN
      bin/data/cambria.ttc
  3. 1046
      bin/data/text/contemp-macbeth.json
  4. 2
      src/main.cpp
  5. 31
      src/ofApp.cpp
  6. 5
      src/ofApp.h
  7. 111
      src/ofTeleprompter.cpp
  8. 15
      src/ofTeleprompter.h
  9. 2
      src/ofYolo.h
  10. 4
      src/onxProcess.cpp

BIN
bin/data/Roboto-SemiBold.ttf
View File

Binary file not shown.

BIN
bin/data/cambria.ttc
View File

Binary file not shown.

1046
bin/data/text/contemp-macbeth.json
View File

File diff suppressed because it is too large

2
src/main.cpp
View File

@ -8,7 +8,7 @@ int main( ){
//Use ofGLFWWindowSettings for more options like multi-monitor fullscreen //Use ofGLFWWindowSettings for more options like multi-monitor fullscreen
ofGLWindowSettings settings; ofGLWindowSettings settings;
settings.setGLVersion(3, 2); settings.setGLVersion(3, 2);
settings.setSize(1920, 1080); settings.setSize(1280, 720);
settings.windowMode = OF_WINDOW; //can also be OF_FULLSCREEN settings.windowMode = OF_WINDOW; //can also be OF_FULLSCREEN
auto mainWindow = ofCreateWindow(settings); auto mainWindow = ofCreateWindow(settings);

31
src/ofApp.cpp
View File

@ -9,20 +9,35 @@ void ofApp::setup(){
videoPlayer.load(videoPath); videoPlayer.load(videoPath);
videoPlayer.setLoopState(OF_LOOP_NORMAL); videoPlayer.setLoopState(OF_LOOP_NORMAL);
videoPlayer.play(); //videoPlayer.play();
inputImage.allocate(640, 640, OF_IMAGE_COLOR); // Allocate image, so we don't get any issues when processing on the thread inputImage.allocate(640, 640, OF_IMAGE_COLOR); // Allocate image, so we don't get any issues when processing on the thread
videoFrame.allocate(1920, 1080, OF_IMAGE_COLOR); videoFrame.allocate(1280, 720, OF_IMAGE_COLOR);
webcam.setDeviceID(0);
webcam.setDesiredFrameRate(60);
webcam.setup(webcamWidth, webcamHeight);
ofLog() << "Webcam size: " << webcam.getWidth() << "x" << webcam.getHeight();
onnx.setup(&inputImage); // setup onnx -> will need to pass in a pointer to the two fbos? onnx.setup(&inputImage); // setup onnx -> will need to pass in a pointer to the two fbos?
} }
//-------------------------------------------------------------- //--------------------------------------------------------------
void ofApp::update(){ void ofApp::update(){
videoPlayer.update(); //videoPlayer.update();
if(videoPlayer.isFrameNew()) { webcam.update();
ofPixels & p = videoPlayer.getPixels(); // if(videoPlayer.isFrameNew()) {
videoFrame.setFromPixels(p); // ofPixels & p = videoPlayer.getPixels();
// videoFrame.setFromPixels(p);
// }
if (webcam.isFrameNew() && webcam.isInitialized()) {
// ofPixels & webcamPixels = webcam.getPixels();
// // Resize webcamPixels to match videoFrame's size
// webcamPixels.resizeTo(videoFrame.getPixelsRef());
// videoFrame.setFromPixels(videoFrame.getPixels());
videoFrame.setFromPixels(webcam.getPixels());
} }
onnx.update(videoFrame); onnx.update(videoFrame);
@ -45,7 +60,9 @@ void ofApp::exit(){
//-------------------------------------------------------------- //--------------------------------------------------------------
void ofApp::keyPressed(int key){ void ofApp::keyPressed(int key){
if(key == 'f' || key == 'F'){
ofToggleFullscreen();
}
} }
//-------------------------------------------------------------- //--------------------------------------------------------------

5
src/ofApp.h
View File

@ -23,4 +23,9 @@ class ofApp : public ofBaseApp{
ofImage videoFrame; ofImage videoFrame;
shared_ptr<ofTeleprompter> teleprompter; shared_ptr<ofTeleprompter> teleprompter;
/* webcam */
ofVideoGrabber webcam;
int webcamWidth = 1280;
int webcamHeight = 720;
}; };

111
src/ofTeleprompter.cpp
View File

@ -2,16 +2,21 @@
void ofTeleprompter::setup() { void ofTeleprompter::setup() {
ofBackground(0); ofBackground(255);
ofSetVerticalSync(false); ofSetVerticalSync(false);
setupGUI(); setupGUI();
loadText();
textFont.load("Avara.otf", 32); /* load both texts */
detailsFont.load("Avara-Bold.otf", 24); loadText(script, filePath);
loadText(scriptContemporary, filePathContemp);
activeScript = &script;
textFont.load("Roboto-SemiBold.ttf", 24);
detailsFont.load("Roboto-SemiBold.ttf", 22);
// Prepare first line for teleprompter // Prepare first line for teleprompter
currentSentence = script[currentLine].sentence; currentSentence = (*activeScript)[currentLine].sentence;
displayedSentence.clear(); displayedSentence.clear();
currentLetterIndex = 0; currentLetterIndex = 0;
lastWordTime = ofGetElapsedTimeMillis(); lastWordTime = ofGetElapsedTimeMillis();
@ -23,9 +28,9 @@ void ofTeleprompter::setup() {
void ofTeleprompter::update() { void ofTeleprompter::update() {
if(ofGetFrameNum() < 2) { if(ofGetFrameNum() < 2) {
currentSpeaker = script[currentLine].speaker; currentSpeaker = (*activeScript)[currentLine].speaker;
currentEmotion = script[currentLine].emotion; currentEmotion = (*activeScript)[currentLine].emotion;
currentSentence = script[currentLine].sentence; currentSentence = (*activeScript)[currentLine].sentence;
} }
currentLineIndex = ofToString(currentLine + 1) + " / " + ofToString(script.size() + 1); currentLineIndex = ofToString(currentLine + 1) + " / " + ofToString(script.size() + 1);
@ -85,6 +90,7 @@ void ofTeleprompter::setupGUI() {
useLLMOnly.addListener(this, &ofTeleprompter::toggleOffText); useLLMOnly.addListener(this, &ofTeleprompter::toggleOffText);
useTextOnly.addListener(this, &ofTeleprompter::toggleOffLLM); useTextOnly.addListener(this, &ofTeleprompter::toggleOffLLM);
useContempTextOnly.addListener(this, &ofTeleprompter::toggleContempScript);
gui.setDefaultWidth(400); gui.setDefaultWidth(400);
gui.setup(); gui.setup();
@ -97,14 +103,16 @@ void ofTeleprompter::setupGUI() {
gui.add(temperature.setup("Temperature", 0.7, 0, 1.5)); gui.add(temperature.setup("Temperature", 0.7, 0, 1.5));
gui.add(useLLMOnly.setup("Use LLM Only", false)); gui.add(useLLMOnly.setup("Use LLM Only", false));
gui.add(useTextOnly.setup("Use Text Only", false)); gui.add(useTextOnly.setup("Use Text Only", false));
gui.add(useContempTextOnly.setup("Use Contept Text Only", false));
gui.add(useGeneratedFeedback.setup("Use LLM Feedback", false));
gui.add(nextLine.setup("Next Line")); gui.add(nextLine.setup("Next Line"));
gui.add(reset.setup("Reset Script")); gui.add(reset.setup("Reset Script"));
} }
void ofTeleprompter::loadText() { void ofTeleprompter::loadText(std::vector<Line> & _script, std::string & _file) {
script.clear(); _script.clear();
ofFile jsonFile(filePath); ofFile jsonFile(_file);
if(jsonFile.exists()) { if(jsonFile.exists()) {
ofJson json = ofLoadJson(jsonFile); ofJson json = ofLoadJson(jsonFile);
int idx = 0; int idx = 0;
@ -115,18 +123,18 @@ void ofTeleprompter::loadText() {
l.speaker = entry.value("first_speaker", ""); l.speaker = entry.value("first_speaker", "");
l.sentence = entry.value("first_text", ""); l.sentence = entry.value("first_text", "");
l.emotion = entry.value("first_emotion", ""); l.emotion = entry.value("first_emotion", "");
script.push_back(l); _script.push_back(l);
} }
} else { } else {
ofLogError() << "JSON file not found: " << filePath; ofLogError() << "JSON file not found: " << filePath;
} }
// Random Check // Random Check
if (!script.empty()) { if (!_script.empty()) {
int randomIdx = ofRandom(script.size()); // returns float int randomIdx = ofRandom(_script.size()); // returns float
int idx = static_cast<int>(randomIdx); // convert to int int idx = static_cast<int>(randomIdx); // convert to int
ofLog() << "Random line: " << script[idx].speaker << ": " << script[idx].sentence; ofLog() << "Random line: " << _script[idx].speaker << ": " << _script[idx].sentence;
ofLog() << "Number of lines: " << script.size(); ofLog() << "Number of lines: " << _script.size();
} }
@ -135,7 +143,7 @@ void ofTeleprompter::loadText() {
} }
void ofTeleprompter::drawText() { void ofTeleprompter::drawText() {
ofSetColor(ofColor::white); ofSetColor(ofColor::red);
// --- Display speaker and emotion centered at the top --- // --- Display speaker and emotion centered at the top ---
std::string speakerText = "Speaker: " + currentSpeaker.getParameter().toString(); std::string speakerText = "Speaker: " + currentSpeaker.getParameter().toString();
@ -154,7 +162,7 @@ void ofTeleprompter::drawText() {
// ------- // -------
ofSetColor(ofColor::yellow); ofSetColor(ofColor::black);
float margin = 128; // pixels float margin = 128; // pixels
float maxWidth = ofGetWidth() - margin * 2; float maxWidth = ofGetWidth() - margin * 2;
std::string wrapped = wrapStringToWidth(displayedSentence, maxWidth); std::string wrapped = wrapStringToWidth(displayedSentence, maxWidth);
@ -174,7 +182,7 @@ void ofTeleprompter::drawText() {
// Draw each line centered horizontally // Draw each line centered horizontally
for (size_t i = 0; i < lines.size(); ++i) { for (size_t i = 0; i < lines.size(); ++i) {
ofRectangle bbox = textFont.getStringBoundingBox(lines[i], 0, 0); ofRectangle bbox = textFont.getStringBoundingBox(lines[i], 0, 0);
float x = (ofGetWidth() - bbox.width) / 2.0f; float x = 128;//(ofGetWidth() - bbox.width) / 2.0f;
float y = startY + i * textFont.getLineHeight(); float y = startY + i * textFont.getLineHeight();
textFont.drawString(lines[i], x, y); textFont.drawString(lines[i], x, y);
} }
@ -226,18 +234,25 @@ void ofTeleprompter::nextLinePressed() {
if (((currentEmotionIntensity > 0.8 && currentEmotionDetetced != "neutral") || useLLMOnly) && !useTextOnly) { if (((currentEmotionIntensity > 0.8 && currentEmotionDetetced != "neutral") || useLLMOnly) && !useTextOnly) {
ofLog() << "Generate Line!"; ofLog() << "Generate Line!";
std::string speaker = script[currentLine - 1].speaker; std::string speaker = (*activeScript)[currentLine - 1].speaker;
std::string sentence = script[currentLine - 1].sentence; std::string sentence = (*activeScript)[currentLine - 1].sentence;
std::string emotion = script[currentLine].emotion; std::string emotion = (*activeScript)[currentLine].emotion;
if (useGeneratedFeedback) {
speaker = currentSpeaker;
sentence = currentSentence;
emotion = currentEmotion;
ofLog() << "Using Generated Feedback";
}
llmThread.requestPrompt(speaker, sentence, currentEmotionDetetced, temperature); llmThread.requestPrompt(speaker, sentence, currentEmotionDetetced, temperature);
waitingForLLM = true; waitingForLLM = true;
// Don't set currentSentence yet! // Don't set currentSentence yet!
} else { } else {
currentSpeaker = script[currentLine].speaker; currentSpeaker = (*activeScript)[currentLine].speaker;
currentEmotion = script[currentLine].emotion; currentEmotion = (*activeScript)[currentLine].emotion;
currentSentence = script[currentLine].sentence; currentSentence = (*activeScript)[currentLine].sentence;
displayedSentence.clear(); displayedSentence.clear();
currentLetterIndex = 0; currentLetterIndex = 0;
lastWordTime = ofGetElapsedTimeMillis(); lastWordTime = ofGetElapsedTimeMillis();
@ -250,9 +265,9 @@ void ofTeleprompter::resetScript() {
currentLine = 0; currentLine = 0;
// Prepare teleprompter effect for letter-by-letter // Prepare teleprompter effect for letter-by-letter
currentSpeaker = script[currentLine].speaker; currentSpeaker = (*activeScript)[currentLine].speaker;
currentEmotion = script[currentLine].emotion; currentEmotion = (*activeScript)[currentLine].emotion;
currentSentence = script[currentLine].sentence; currentSentence = (*activeScript)[currentLine].sentence;
displayedSentence.clear(); displayedSentence.clear();
currentLetterIndex = 0; currentLetterIndex = 0;
@ -270,3 +285,43 @@ void ofTeleprompter::toggleOffText(bool & val) {
useTextOnly = false; useTextOnly = false;
} }
} }
void ofTeleprompter::toggleContempScript(bool & val) {
if (val) {
activeScript = &scriptContemporary;
} else {
activeScript = &script;
}
ofLog() << "Script Size:" + (*activeScript).size();
}
void ofTeleprompter::keyPressed(int key){
if(key == 'f' || key == 'F'){
ofToggleFullscreen();
}
if(key == OF_KEY_RIGHT) {
nextLinePressed();
}
if(key == OF_KEY_LEFT) {
pastLine();
}
if(key == 'r' || key == 'R'){
resetScript();
}
}
void ofTeleprompter::pastLine() {
if (currentLine < script.size()) {
currentLine--;
currentSpeaker = (*activeScript)[currentLine].speaker;
currentEmotion = (*activeScript)[currentLine].emotion;
currentSentence = (*activeScript)[currentLine].sentence;
displayedSentence.clear();
currentLetterIndex = 0;
lastWordTime = ofGetElapsedTimeMillis();
}
}

15
src/ofTeleprompter.h
View File

@ -17,14 +17,18 @@ class ofTeleprompter: public ofBaseApp{
void draw(); void draw();
void setupGUI(); void setupGUI();
void nextLinePressed(); void nextLinePressed();
void pastLine();
void resetScript(); void resetScript();
void sendLLMRequest(); void sendLLMRequest();
void loadText(); void loadText(std::vector<Line> & script, std::string & file);
void drawText(); void drawText();
void updateCVData(int numOfFacesDetected, std::string emotion, float intensity); void updateCVData(int numOfFacesDetected, std::string emotion, float intensity);
void toggleOffLLM(bool & val); void toggleOffLLM(bool & val);
void toggleOffText(bool & val); void toggleOffText(bool & val);
void toggleContempScript(bool & val);
void toggleGeneratedFeeback(bool & val);
void keyPressed(int key);
std::string wrapStringToWidth(const std::string& text, float maxWidth); std::string wrapStringToWidth(const std::string& text, float maxWidth);
@ -35,6 +39,8 @@ class ofTeleprompter: public ofBaseApp{
ofxButton nextLine; ofxButton nextLine;
ofxToggle useLLMOnly; ofxToggle useLLMOnly;
ofxToggle useTextOnly; ofxToggle useTextOnly;
ofxToggle useContempTextOnly;
ofxToggle useGeneratedFeedback;
ofxButton reset; ofxButton reset;
ofxLabel currentLineIndex; ofxLabel currentLineIndex;
@ -47,16 +53,19 @@ class ofTeleprompter: public ofBaseApp{
/* script */ /* script */
std::vector<Line> script; std::vector<Line> script;
std::vector<Line> scriptContemporary;
std::string filePath = "text/preprocess_original.json"; std::string filePath = "text/preprocess_original.json";
std::string filePathContemp = "text/contemp-macbeth.json";
int currentLine = 0; int currentLine = 0;
std::string currentLineString = "N/A"; std::string currentLineString = "N/A";
const char* emotions[7] = {"anger", "disgust", "fear", "happiness", "neutral", "sadness", "surprise"}; const char* emotions[7] = {"anger", "disgust", "fear", "excited", "bored", "upset", "interested"};
std::vector<Line>* activeScript = nullptr;
/* scrolling text */ /* scrolling text */
int currentLetterIndex = 0; int currentLetterIndex = 0;
std::string currentSentence; std::string currentSentence;
uint64_t lastWordTime = 0; uint64_t lastWordTime = 0;
uint64_t wordDelay = 10; uint64_t wordDelay = 40;
std::string displayedSentence; std::string displayedSentence;
ofTrueTypeFont textFont; ofTrueTypeFont textFont;
ofTrueTypeFont detailsFont; ofTrueTypeFont detailsFont;

2
src/ofYolo.h
View File

@ -48,7 +48,7 @@ struct Emotion {
std::string getDominantEmotion() const { std::string getDominantEmotion() const {
float values[7] = {anger, disgust, fear, happiness, neutral, sadness, surprise}; float values[7] = {anger, disgust, fear, happiness, neutral, sadness, surprise};
const char* names[7] = {"anger", "disgust", "fear", "happiness", "neutral", "sadness", "surprise"}; const char* names[7] = {"anger", "disgust", "fear", "excited", "bored", "upset", "interested"};
int maxIndex = 0; int maxIndex = 0;
float maxValue = values[0]; float maxValue = values[0];

4
src/onxProcess.cpp
View File

@ -48,7 +48,7 @@ void onxProcess::update(ofImage& frame) {
detectedFaces.clear(); detectedFaces.clear();
yolo.ParseOutput(tensor, detectedFaces, numAnchors); yolo.ParseOutput(tensor, detectedFaces, numAnchors);
yolo.ConvertBoxCoordsToOriginalSize(detectedFaces, 1920, 1080); yolo.ConvertBoxCoordsToOriginalSize(detectedFaces, 1280, 720);
fdThread.resetInferenceFlag(); fdThread.resetInferenceFlag();
fdThread.shouldRunInference = true; fdThread.shouldRunInference = true;
@ -162,7 +162,7 @@ void onxProcess::setTeleprompterValues() {
} }
} }
const char* emotions[7] = {"anger", "disgust", "fear", "happiness", "neutral", "sadness", "surprise"}; const char* emotions[7] = {"anger", "disgust", "fear", "excited", "bored", "upset", "interested"};
std::string maxEmotionName = emotions[maxEmotionIdx]; std::string maxEmotionName = emotions[maxEmotionIdx];
dominantEmotion = maxEmotionName; dominantEmotion = maxEmotionName;

Write Preview
Loading…
Cancel
Save