frame update & switch

src/Player.cpp

@@ -6,58 +6,66 @@ Player::Player(){
 
 /* Basic ofVideoPlayer setup */
 void Player::Setup(){
-    videoPlayer.setLoopState(OF_LOOP_NONE);
+    videoPlayer.setLoopState(OF_LOOP_NORMAL);
     videoPlayer.setVolume(0);
 }
 
 /* Updated the video player:
     (1) Allocates the required W x H for the model input
     (2) Updates the video texture, and sets the current frame value */
-void Player::Update(ofImage &img){
+void Player::Update(ofImage &img, bool show_frame){
 
     if(!img.isAllocated()){
         img.allocate(ofGetWindowWidth() / 2, ofGetWindowHeight(), OF_IMAGE_COLOR);
         temp.allocate(ofGetWindowWidth() / 2, ofGetWindowHeight(), GL_RGB);
     }
 
-    if(videoPlayer.isLoaded()){
+    // Calculate the target width and height for model_output_fbo_1
+    float fbo_1_target_width = img.getWidth();  // 1/2 of the screen width (990px)
+    float fbo_1_target_height = img.getHeight();  // Full height of the screen
+    float fbo_aspect_ratio = fbo_1_target_width / fbo_1_target_height;
+    float aspect_ratio;
+
+    if(show_frame && frame.isAllocated()){
+        aspect_ratio = frame.getWidth() / frame.getHeight();
+    } 
+
+    if(videoPlayer.isLoaded() && !show_frame){
         hasVideo = true;
         playerCurrentFrame = videoPlayer.getCurrentFrame();
         videoPlayer.update();
-        videoPlayer.play();
+        aspect_ratio = videoPlayer.getWidth() / videoPlayer.getHeight();
-
+    }
-        // Calculate the target width and height for model_output_fbo_1
-        float fbo_1_target_width = img.getWidth();  // 1/2 of the screen width (990px)
-        float fbo_1_target_height = img.getHeight();  // Full height of the screen
-
-        // Calculate the aspect ratio of the video and the FBO
-        float video_aspect_ratio = videoPlayer.getWidth() / videoPlayer.getHeight();
-        float fbo_aspect_ratio = fbo_1_target_width / fbo_1_target_height;
 
-        // Adjust the scaling to cover the FBO area while maintaining aspect ratio
+    if (fbo_aspect_ratio > aspect_ratio) {
+        // FBO is wider; scale by width to fill the FBO
+        new_width = fbo_1_target_width;
+        new_height = new_width / aspect_ratio;  // Scale height to maintain aspect ratio
+    } else {
+        // FBO is taller; scale by height to fill the FBO
+        new_height = fbo_1_target_height;
+        new_width = new_height * aspect_ratio;  // Scale width to maintain aspect ratio
+    }
 
-        if (fbo_aspect_ratio > video_aspect_ratio) {
+    // Center the video to ensure it fills the FBO and is cropped if necessary
-            // FBO is wider; scale by width to fill the FBO
+    x_pos = (ofGetWindowWidth() * 0.25) - (new_width / 2);
-            new_width = fbo_1_target_width;
+    y_pos = (ofGetWindowHeight() - new_height) / 2;  // Center vertically
-            new_height = new_width / video_aspect_ratio;  // Scale height to maintain aspect ratio
-        } else {
-            // FBO is taller; scale by height to fill the FBO
-            new_height = fbo_1_target_height;
-            new_width = new_height * video_aspect_ratio;  // Scale width to maintain aspect ratio
-        }
 
-        // Center the video to ensure it fills the FBO and is cropped if necessary
+    if(show_frame && frame.isAllocated()){
-        x_pos = (ofGetWindowWidth() * 0.25) - (new_width / 2);
+        temp.begin();
-        y_pos = (ofGetWindowHeight() - new_height) / 2;  // Center vertically
+        frame.draw(x_pos, y_pos, new_width, new_height);
+        temp.end();
+    } 
 
+    if(videoPlayer.isLoaded() && !show_frame){
         temp.begin();
         videoPlayer.draw(x_pos, y_pos, new_width, new_height);
         temp.end();
-
-        ofPixels pixels;
-        temp.readToPixels(pixels);
-        img.setFromPixels(pixels);
     }
+
+    ofPixels pixels;
+    temp.readToPixels(pixels);
+    img.setFromPixels(pixels);
 }
 
 void Player::Draw(){
@@ -75,10 +83,16 @@ ofPixels Player::GetVideoPixels(){
     (2) Allocates the fbo dims for final render */
 void Player::SetVideo(std::string path, ofFbo &fbo){
     videoPlayer.load(path);
+    videoPlayer.play();
     videoPlayer.setFrame(800);
     fbo.allocate(ofGetWindowWidth() / 2, ofGetWindowHeight(), GL_RGB);
 }
 
+/* Loads a frame from path */
+void Player::SetFrame(std::string path){
+    frame.load(path);
+}
+
 // Sets a random frame in the active video
 void Player::SetRandomFrame(){
     int randomFrame = ofRandom(0, videoPlayer.getTotalNumFrames());

src/Player.h

@@ -8,9 +8,10 @@ class Player {
     public:
     
     void Setup();
-    void Update(ofImage &img);
+    void Update(ofImage &img, bool show_frame);
     void Draw();
     void SetVideo(std::string path, ofFbo &fbo);
+    void SetFrame(std::string path);
     ofPixels GetVideoPixels();
     void SetVideoPosition(ofFbo& output_fbo);
     void SetRandomFrame();
@@ -37,6 +38,8 @@ class Player {
     float y_pos;
     float new_width;
     float new_height;
+
+    ofImage frame;
     
     
     Player();

src/Request.cpp

@@ -2,30 +2,40 @@
 
 /* setup http server */
 void Request::setup(std::string ip, int port, std::string page){
+    std::cout << "Initialising HTTP Server" << std::endl;
+
     req.method = ofHttpRequest::POST;
     req.url = "http://" + ip + ":" + ofToString(port) + "/" + page;
     req.headers["Content-Type"] = "application/json";
 }
 
 /* send a request to vp_server & return frame/video/folder */
-VPResp Request::query(Vector7D in){
+VPResp Request::query(Vector7D& in){
-    req.body = "{\"vector\": [" +
+    std::cout << "Sending Request to HTTP Server" << std::endl;
-            ofToString(in.angry) + "," +
-            ofToString(in.disgust) + "," +
-            ofToString(in.fear) + "," +
-            ofToString(in.happy) + "," +
-            ofToString(in.sad) + "," +
-            ofToString(in.surprise) + "," +
-            ofToString(in.neutral) + "]}";
-            
-    auto resp = http.handleRequest(req);
-    json_resp = ofJson::parse(resp.data.getText());
-
     VPResp vp_resp;
-    vp_resp.folder = json_resp["folder"];
+    try {
-    vp_resp.image = json_resp["image"];
+        req.body = "{\"vector\": [" +
-    vp_resp.video = json_resp["video"];
+                ofToString(in.angry) + "," +
-    vp_resp.frame = json_resp["frame"];
+                ofToString(in.disgust) + "," +
+                ofToString(in.fear) + "," +
+                ofToString(in.happy) + "," +
+                ofToString(in.sad) + "," +
+                ofToString(in.surprise) + "," +
+                ofToString(in.neutral) + "]}";
+
+        auto resp = http.handleRequest(req);
+        json_resp = ofJson::parse(resp.data.getText());
+        vp_resp.folder = json_resp["folder"];
+        vp_resp.image = json_resp["image"];
+        vp_resp.video = json_resp["video"];
+        vp_resp.frame = json_resp["frame"];
+        vp_resp.lost = json_resp["lost"];
+
+        past_vp_resp = vp_resp;
 
-    return vp_resp;
+        return vp_resp;
+    } catch (exception e) {
+        // Some issue happening here when plugging in controllers, or when they initially connect
+        return past_vp_resp;
+    }
 }

src/Request.h

@@ -29,16 +29,18 @@ struct VPResp{
     std::string folder;
     std::string video;
     std::string image;
-    int frame;
+    std::string frame;
+    int lost;
 };
 
 class Request{
     public:
     void setup(std::string ip, int port, std::string page);
-    VPResp query(Vector7D in);
+    VPResp query(Vector7D& in);
 
     ofHttpRequest req;
     ofURLFileLoader http;
     ofJson json_resp;
+    VPResp past_vp_resp;
 };
 #endif

src/Server.cpp

@@ -1,7 +1,9 @@
 #include "Server.h"
 
 void Server::start(){
+    std::cout << "Initialising TCP sever" << std::endl;
     server.setup(port);
+    http.setup(http_ip, http_port, http_page);
     is_active = true;
     previous_embedding = embedding;
     last_change_time = std::chrono::steady_clock::now();
@@ -93,6 +95,7 @@ void Server::checkActivity(){
         last_change_time = std::chrono::steady_clock::now();  // Reset the timer if there is a change
         previous_embedding = embedding;  // Update the previous embedding to the current one
         is_active = true;
+        sendHttpRequest();
     } else {
         // Calculate the time since the last change
         auto now = std::chrono::steady_clock::now();
@@ -100,8 +103,12 @@ void Server::checkActivity(){
 
         if (duration >= 2) {
             is_active = false;
+            std::cout << is_active << std::endl;
         }
     }
+}
 
-    std::cout << is_active << std::endl;
+/* sends request to http server if is_active = true */
+void Server::sendHttpRequest(){
+    vp_resp = http.query(embedding);
 }

src/Server.h

@@ -14,8 +14,8 @@ struct ClientInfo {
 
 class Server{
     public:
-    Server(int port, Vector7D& _embedding, bool debug) 
+    Server(int port, Vector7D& _embedding, bool debug, std::string _http_ip, int _http_port, std::string _http_page) 
-        : port(port), embedding(_embedding), debug(debug) {}
+        : port(port), embedding(_embedding), debug(debug), http_ip(_http_ip), http_port(_http_port), http_page(_http_page) {}
 
     void start();
     void update();
@@ -23,6 +23,8 @@ class Server{
     void printClients();
     void updateEmbedding();
     void checkActivity();
+    void sendHttpRequest();
+
 
     int port;
     ofxTCPServer server;
@@ -31,10 +33,15 @@ class Server{
     bool is_active;
     Vector7D& embedding;
 
+    Request http;
+    std::string http_ip;
+    int http_port;
+    std::string http_page;
+    VPResp vp_resp;
+
     private:
     Vector7D previous_embedding;
     std::chrono::time_point<std::chrono::steady_clock> last_change_time;
-    bool hasChanged();
 };
 
 #endif

src/ofApp.cpp

@@ -20,6 +20,7 @@ void ofApp::setup(){
     /* setup video */
     player.Setup();
     player.SetVideo("videos/demo.mp4", model_output_fbo_1);
+    player.SetFrame("demo.jpg");
 
     /* setup models (modelPath, log, useCuda)   */
     ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/depth_anything_v2_vitb.onnx";
@@ -60,7 +61,7 @@ void ofApp::setup(){
     */
 
     /* setup server (http & tcp) */
-    server = std::make_unique<Server>(12345, embedding, true);
+    server = std::make_unique<Server>(12345, embedding, false, "192.168.0.253", 2000, "search");
     server->start();
 }
 
@@ -91,8 +92,8 @@ void ofApp::update(){
         map.Draw(); 
     }
     
-    /* Setup model input using ofImage, allocated fbo */
+    /* Setup model input using ofImage, allocated fbo, checks if a frame or video_frme should be drawn to the screen */
-    player.Update(img);
+    player.Update(img, server->is_active);
     
     /* Run Models, and set pixels */
     try{
@@ -144,7 +145,7 @@ void ofApp::draw(){
     ofPopMatrix();
 
     printEmotions();
-    
+
 //   emoteImage.draw(640, 0);
 //   for(auto& face : detected_faces){
 //     ofDrawBitmapString(std::to_string(face.box.emotional_state.emotions[0]), 700, 300);
@@ -220,6 +221,11 @@ void ofApp::printEmotions(){
     ofPopMatrix();
 }
 
+/* check vp_resp, set frame to video_player fbo (centered + scaled) */
+void ofApp::displayFrame(){
+
+}
+
 //--------------------------------------------------------------
 void ofApp::keyPressed(int key){
     // if (key=OF_KEY_LEFT){

src/ofApp.h

@@ -35,6 +35,7 @@ class ofApp : public ofBaseApp{
 		void inferEmotionalState();
 		void renderDepthMap();
 		void printEmotions();
+		void displayFrame();
 
 		float window_height;
 		float window_width;
@@ -70,6 +71,7 @@ class ofApp : public ofBaseApp{
 		ofFbo video_player_fbo;
 		ofFbo model_output_fbo;
 		ofFbo model_output_fbo_1;
+		ofFbo model_output_fbo_frame;
 		ofFbo screen_fbo;
 
 		ModelThread threadMap;