osc added, video seeking added, fixed model threads

src/Player.cpp

@@ -52,6 +52,7 @@ void Player::Update(ofImage &img, bool show_frame){
     y_pos = (ofGetWindowHeight() - new_height) / 2;  // Center vertically
 
     if(show_frame && frame.isAllocated()){
+        set_frame = true;
         temp.begin();
         frame.draw(x_pos, y_pos, new_width, new_height);
         temp.end();
@@ -81,13 +82,43 @@ ofPixels Player::GetVideoPixels(){
 /* Loads the video from path:
     (1) Sets the frame
     (2) Allocates the fbo dims for final render */
-void Player::SetVideo(std::string path, ofFbo &fbo){
+void Player::SetVideoOnAwake(std::string path, ofFbo &fbo){
     videoPlayer.load(path);
     videoPlayer.play();
     videoPlayer.setFrame(800);
+    /* pause video initially */
     fbo.allocate(ofGetWindowWidth() / 2, ofGetWindowHeight(), GL_RGB);
 }
 
+void Player::SetVideo(std::string path, std::string frame, bool is_active){
+    if(path.empty()){
+        path = "videos/demo.mp4";
+    }
+
+    /* convert str frame -> int (remove _x)*/
+    int f_number = 0;
+
+    try {
+        size_t pos = frame.find("_");
+        std::string frame_number = frame.substr(0, pos);
+        f_number = std::stoi(frame_number);
+    } catch (exception e){
+        std::cout << "No underscore found in frame name" << std::endl;
+    }
+
+    if(!is_active){
+        if(videoPlayer.getMoviePath() != path){
+            videoPlayer.loadAsync(path);
+            videoPlayer.play();
+        }
+
+        if(set_frame){
+            videoPlayer.setFrame(800);
+            set_frame = false;
+        }
+    }
+}
+
 /* Loads a frame from path */
 void Player::SetFrame(std::string path){
     frame.load(path);

src/Player.h

@@ -10,7 +10,8 @@ class Player {
     void Setup();
     void Update(ofImage &img, bool show_frame);
     void Draw();
-    void SetVideo(std::string path, ofFbo &fbo);
+    void SetVideoOnAwake(std::string path, ofFbo &fbo);
+    void SetVideo(std::string path, std::string frame, bool show_frame);
     void SetFrame(std::string path);
     ofPixels GetVideoPixels();
     void SetVideoPosition(ofFbo& output_fbo);
@@ -39,6 +40,8 @@ class Player {
     float new_width;
     float new_height;
 
+    bool set_frame;
+
     ofImage frame;
     
     Player();

src/Server.cpp

@@ -3,6 +3,7 @@
 void Server::start(){
     std::cout << "Initialising TCP sever" << std::endl;
     server.setup(port);
+    osc_sender.setup(OSC_HOST, OSC_PORT);
     http.setup(http_ip, http_port, http_page);
     is_active = true;
     previous_embedding = embedding;
@@ -32,6 +33,7 @@ void Server::update(){
 
     updateEmbedding();
     checkActivity();
+    sendOSCMessage();
 
     if(debug){
         printClients();
@@ -110,6 +112,47 @@ void Server::checkActivity(){
     }
 }
 
+/* send osc msg, check if audio file exists and it is different to the past audiofile */
+void Server::sendOSCMessage(){
+    std::vector<ofxOscMessage> messages;
+
+    ofxOscMessage me_0;
+    ofxOscMessage me_1;
+    ofxOscMessage me_2;
+    ofxOscMessage me_3;
+    ofxOscMessage me_file;
+
+    std::string audio_file = vp_resp.folder;
+
+    // Check if file exists in a given dir
+    ofFile file("/home/cailean/Desktop/rave/all_wav_files/" +  audio_file + ".wav");
+
+    if(!is_active && (audio_file != past_audio_file) && file.exists()){
+        me_file.setAddress("/emote/filename");
+        me_file.addStringArg(audio_file + ".wav");
+        messages.push_back(me_file);
+        past_audio_file = audio_file;
+    }
+
+    me_0.setAddress("/emote/0");
+    me_0.addFloatArg(embedding.neutral);
+    messages.push_back(me_0);
+    me_1.setAddress("/emote/1");
+    me_1.addFloatArg(embedding.neutral);
+    messages.push_back(me_1);
+    me_2.setAddress("/emote/2");
+    me_2.addFloatArg(embedding.neutral);
+    messages.push_back(me_2);
+    me_3.setAddress("/emote/3");
+    me_3.addFloatArg(embedding.neutral);
+    messages.push_back(me_3);
+
+    for (auto& msg : messages){
+         osc_sender.sendMessage(msg, false);
+    }
+   
+}
+
 /* sends request to http server if is_active = true */
 void Server::sendHttpRequest(){
     vp_resp = http.query(embedding);

src/Server.h

@@ -6,6 +6,10 @@
 #include "Request.h"
 #include <unordered_map>
 #include <chrono>
+#include "ofxOsc.h"
+
+#define OSC_HOST "127.0.0.1"
+#define OSC_PORT 9002
 
 struct ClientInfo {
     float value;
@@ -24,7 +28,7 @@ class Server{
     void updateEmbedding();
     void checkActivity();
     void sendHttpRequest();
-
+    void sendOSCMessage();
 
     int port;
     ofxTCPServer server;
@@ -42,6 +46,9 @@ class Server{
     private:
     Vector7D previous_embedding;
     std::chrono::time_point<std::chrono::steady_clock> last_change_time;
+
+    ofxOscSender osc_sender;
+    std::string past_audio_file;
 };
 
 #endif

src/main.cpp

@@ -8,7 +8,7 @@ int main( ){
 	ofGLWindowSettings settings;
 	settings.setSize(1512, 1080);
 	settings.setGLVersion(3, 2);
-	settings.windowMode = OF_FULLSCREEN; //can also be OF_FULLSCREEN
+	settings.windowMode = OF_WINDOW; //can also be OF_FULLSCREEN
 
 	auto window = ofCreateWindow(settings);
 

src/ofApp.cpp

@@ -19,7 +19,7 @@ void ofApp::setup(){
 
     /* setup video */
     player.Setup();
-    player.SetVideo("videos/demo.mp4", model_output_fbo_1);
+    player.SetVideoOnAwake("videos/demo.mp4", model_output_fbo_1);
     player.SetFrame("demo.jpg");
 
     /* setup models (modelPath, log, useCuda)   */
@@ -28,9 +28,9 @@ void ofApp::setup(){
     ORTCHAR_T* modelPath3 = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/rgb_emotion.onnx";
     ORTCHAR_T* modelPath4 = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/depth_anything_v2_vits.onnx";
 
-    yolo.Setup(modelPath2, false, true);
-    depth.Setup(modelPath, false, true);
-    depth_small.Setup(modelPath4, false, true);
+    //yolo.Setup(modelPath2, false, true);
+    depth.Setup(modelPath, true, true);
+    depth_small.Setup(modelPath, true, true);
     //emotion.Setup(modelPath3, false, true);
 
     /* Depth output fbo */
@@ -46,7 +46,7 @@ void ofApp::setup(){
 
     threadMap.setup(&map.fboImage, &model_output_fbo, &depth);
     threadVideo.setup(&img, &model_output_fbo_1, &depth_small);
-    threadYolo.setupYolo(&img, &detected_faces, &yolo, &faceDetector);
+    //threadYolo.setupYolo(&img, &detected_faces, &yolo, &faceDetector);
 
      /* 
         Create a dummy initial input of batch_size = 5, as 
@@ -75,11 +75,11 @@ void ofApp::update(){
 
     /*  Check to see if the application has moved to the first frame    
         As the models need to load first, as the first inference is quite slow  */ 
-    if(ofGetFrameNum() > 0 && ofGetFrameNum() < 2){
+    if(ofGetFrameNum() < 2){
         firstRun = false;
         threadMap.start();
         threadVideo.start();
-        threadYolo.start();
+        //threadYolo.start();
     }
     
     /* Clear detetced face list */
@@ -100,11 +100,11 @@ void ofApp::update(){
 
         threadMap.update();
         threadVideo.update();
-        threadYolo.update();
+        //threadYolo.update();
 
         depth.SetPixels(model_output_fbo);
         depth_small.SetPixels(model_output_fbo_1);
-        faceDetector.ConvertBoxCoordsToOriginalSize(detected_faces, model_output_fbo_1.getWidth(), model_output_fbo_1.getHeight());
+        //faceDetector.ConvertBoxCoordsToOriginalSize(detected_faces, model_output_fbo_1.getWidth(), model_output_fbo_1.getHeight());
 
         /* As no input is generated for the emotion recognition model, run a dummy vector through the model
             So it can load.
@@ -133,10 +133,10 @@ void ofApp::draw(){
 
     renderDepthMap();
 
-    if(!firstRun && detected_faces.size() != 0){
-        faceDetector.DrawBox(detected_faces);
-        faceDetector.DrawCenter(detected_faces);
-    }
+    // if(!firstRun && detected_faces.size() != 0){
+    //     faceDetector.DrawBox(detected_faces);
+    //     faceDetector.DrawCenter(detected_faces);
+    // }
 
     ofPushMatrix();
     ofSetColor(255);
@@ -225,6 +225,8 @@ void ofApp::printEmotions(){
 void ofApp::displayFrame(){
     /* set frame from recent vantage-point tree response */
     player.SetFrame(server->vp_resp.image);
+    /* set video from recent vantage-point tree resp */
+    player.SetVideo(server->vp_resp.video, server->vp_resp.frame, server->is_active);
     /* set image for model input, checks if the clients are active -> yes-frame no-video */
     player.Update(img, server->is_active);
 }