From acf3174228f8dd249ce7e439c117f00d05164df5 Mon Sep 17 00:00:00 2001 From: cailean Date: Mon, 7 Oct 2024 14:36:31 +0100 Subject: [PATCH] frame update & switch --- src/Player.cpp | 72 +++++++++++++++++++++++++++++-------------------- src/Player.h | 5 +++- src/Request.cpp | 46 ++++++++++++++++++------------- src/Request.h | 6 +++-- src/Server.cpp | 9 ++++++- src/Server.h | 13 ++++++--- src/ofApp.cpp | 14 +++++++--- src/ofApp.h | 2 ++ 8 files changed, 109 insertions(+), 58 deletions(-) diff --git a/src/Player.cpp b/src/Player.cpp index 9cb137e..4af7150 100644 --- a/src/Player.cpp +++ b/src/Player.cpp @@ -6,58 +6,66 @@ Player::Player(){ /* Basic ofVideoPlayer setup */ void Player::Setup(){ - videoPlayer.setLoopState(OF_LOOP_NONE); + videoPlayer.setLoopState(OF_LOOP_NORMAL); videoPlayer.setVolume(0); } /* Updated the video player: (1) Allocates the required W x H for the model input (2) Updates the video texture, and sets the current frame value */ -void Player::Update(ofImage &img){ +void Player::Update(ofImage &img, bool show_frame){ if(!img.isAllocated()){ img.allocate(ofGetWindowWidth() / 2, ofGetWindowHeight(), OF_IMAGE_COLOR); temp.allocate(ofGetWindowWidth() / 2, ofGetWindowHeight(), GL_RGB); } - if(videoPlayer.isLoaded()){ + // Calculate the target width and height for model_output_fbo_1 + float fbo_1_target_width = img.getWidth(); // 1/2 of the screen width (990px) + float fbo_1_target_height = img.getHeight(); // Full height of the screen + float fbo_aspect_ratio = fbo_1_target_width / fbo_1_target_height; + float aspect_ratio; + + if(show_frame && frame.isAllocated()){ + aspect_ratio = frame.getWidth() / frame.getHeight(); + } + + if(videoPlayer.isLoaded() && !show_frame){ hasVideo = true; playerCurrentFrame = videoPlayer.getCurrentFrame(); videoPlayer.update(); - videoPlayer.play(); - - // Calculate the target width and height for model_output_fbo_1 - float fbo_1_target_width = img.getWidth(); // 1/2 of the screen width (990px) - float fbo_1_target_height = img.getHeight(); // Full height of the screen - - // Calculate the aspect ratio of the video and the FBO - float video_aspect_ratio = videoPlayer.getWidth() / videoPlayer.getHeight(); - float fbo_aspect_ratio = fbo_1_target_width / fbo_1_target_height; + aspect_ratio = videoPlayer.getWidth() / videoPlayer.getHeight(); + } - // Adjust the scaling to cover the FBO area while maintaining aspect ratio + if (fbo_aspect_ratio > aspect_ratio) { + // FBO is wider; scale by width to fill the FBO + new_width = fbo_1_target_width; + new_height = new_width / aspect_ratio; // Scale height to maintain aspect ratio + } else { + // FBO is taller; scale by height to fill the FBO + new_height = fbo_1_target_height; + new_width = new_height * aspect_ratio; // Scale width to maintain aspect ratio + } - if (fbo_aspect_ratio > video_aspect_ratio) { - // FBO is wider; scale by width to fill the FBO - new_width = fbo_1_target_width; - new_height = new_width / video_aspect_ratio; // Scale height to maintain aspect ratio - } else { - // FBO is taller; scale by height to fill the FBO - new_height = fbo_1_target_height; - new_width = new_height * video_aspect_ratio; // Scale width to maintain aspect ratio - } + // Center the video to ensure it fills the FBO and is cropped if necessary + x_pos = (ofGetWindowWidth() * 0.25) - (new_width / 2); + y_pos = (ofGetWindowHeight() - new_height) / 2; // Center vertically - // Center the video to ensure it fills the FBO and is cropped if necessary - x_pos = (ofGetWindowWidth() * 0.25) - (new_width / 2); - y_pos = (ofGetWindowHeight() - new_height) / 2; // Center vertically + if(show_frame && frame.isAllocated()){ + temp.begin(); + frame.draw(x_pos, y_pos, new_width, new_height); + temp.end(); + } + if(videoPlayer.isLoaded() && !show_frame){ temp.begin(); videoPlayer.draw(x_pos, y_pos, new_width, new_height); temp.end(); - - ofPixels pixels; - temp.readToPixels(pixels); - img.setFromPixels(pixels); } + + ofPixels pixels; + temp.readToPixels(pixels); + img.setFromPixels(pixels); } void Player::Draw(){ @@ -75,10 +83,16 @@ ofPixels Player::GetVideoPixels(){ (2) Allocates the fbo dims for final render */ void Player::SetVideo(std::string path, ofFbo &fbo){ videoPlayer.load(path); + videoPlayer.play(); videoPlayer.setFrame(800); fbo.allocate(ofGetWindowWidth() / 2, ofGetWindowHeight(), GL_RGB); } +/* Loads a frame from path */ +void Player::SetFrame(std::string path){ + frame.load(path); +} + // Sets a random frame in the active video void Player::SetRandomFrame(){ int randomFrame = ofRandom(0, videoPlayer.getTotalNumFrames()); diff --git a/src/Player.h b/src/Player.h index bc522ba..f13eb62 100644 --- a/src/Player.h +++ b/src/Player.h @@ -8,9 +8,10 @@ class Player { public: void Setup(); - void Update(ofImage &img); + void Update(ofImage &img, bool show_frame); void Draw(); void SetVideo(std::string path, ofFbo &fbo); + void SetFrame(std::string path); ofPixels GetVideoPixels(); void SetVideoPosition(ofFbo& output_fbo); void SetRandomFrame(); @@ -37,6 +38,8 @@ class Player { float y_pos; float new_width; float new_height; + + ofImage frame; Player(); diff --git a/src/Request.cpp b/src/Request.cpp index dc3db8b..43b3705 100644 --- a/src/Request.cpp +++ b/src/Request.cpp @@ -2,30 +2,40 @@ /* setup http server */ void Request::setup(std::string ip, int port, std::string page){ + std::cout << "Initialising HTTP Server" << std::endl; + req.method = ofHttpRequest::POST; req.url = "http://" + ip + ":" + ofToString(port) + "/" + page; req.headers["Content-Type"] = "application/json"; } /* send a request to vp_server & return frame/video/folder */ -VPResp Request::query(Vector7D in){ - req.body = "{\"vector\": [" + - ofToString(in.angry) + "," + - ofToString(in.disgust) + "," + - ofToString(in.fear) + "," + - ofToString(in.happy) + "," + - ofToString(in.sad) + "," + - ofToString(in.surprise) + "," + - ofToString(in.neutral) + "]}"; - - auto resp = http.handleRequest(req); - json_resp = ofJson::parse(resp.data.getText()); - +VPResp Request::query(Vector7D& in){ + std::cout << "Sending Request to HTTP Server" << std::endl; VPResp vp_resp; - vp_resp.folder = json_resp["folder"]; - vp_resp.image = json_resp["image"]; - vp_resp.video = json_resp["video"]; - vp_resp.frame = json_resp["frame"]; + try { + req.body = "{\"vector\": [" + + ofToString(in.angry) + "," + + ofToString(in.disgust) + "," + + ofToString(in.fear) + "," + + ofToString(in.happy) + "," + + ofToString(in.sad) + "," + + ofToString(in.surprise) + "," + + ofToString(in.neutral) + "]}"; + + auto resp = http.handleRequest(req); + json_resp = ofJson::parse(resp.data.getText()); + vp_resp.folder = json_resp["folder"]; + vp_resp.image = json_resp["image"]; + vp_resp.video = json_resp["video"]; + vp_resp.frame = json_resp["frame"]; + vp_resp.lost = json_resp["lost"]; + + past_vp_resp = vp_resp; - return vp_resp; + return vp_resp; + } catch (exception e) { + // Some issue happening here when plugging in controllers, or when they initially connect + return past_vp_resp; + } } \ No newline at end of file diff --git a/src/Request.h b/src/Request.h index 7d61ffa..e4bd099 100644 --- a/src/Request.h +++ b/src/Request.h @@ -29,16 +29,18 @@ struct VPResp{ std::string folder; std::string video; std::string image; - int frame; + std::string frame; + int lost; }; class Request{ public: void setup(std::string ip, int port, std::string page); - VPResp query(Vector7D in); + VPResp query(Vector7D& in); ofHttpRequest req; ofURLFileLoader http; ofJson json_resp; + VPResp past_vp_resp; }; #endif \ No newline at end of file diff --git a/src/Server.cpp b/src/Server.cpp index 0177cdb..61fc8b9 100644 --- a/src/Server.cpp +++ b/src/Server.cpp @@ -1,7 +1,9 @@ #include "Server.h" void Server::start(){ + std::cout << "Initialising TCP sever" << std::endl; server.setup(port); + http.setup(http_ip, http_port, http_page); is_active = true; previous_embedding = embedding; last_change_time = std::chrono::steady_clock::now(); @@ -93,6 +95,7 @@ void Server::checkActivity(){ last_change_time = std::chrono::steady_clock::now(); // Reset the timer if there is a change previous_embedding = embedding; // Update the previous embedding to the current one is_active = true; + sendHttpRequest(); } else { // Calculate the time since the last change auto now = std::chrono::steady_clock::now(); @@ -100,8 +103,12 @@ void Server::checkActivity(){ if (duration >= 2) { is_active = false; + std::cout << is_active << std::endl; } } +} - std::cout << is_active << std::endl; +/* sends request to http server if is_active = true */ +void Server::sendHttpRequest(){ + vp_resp = http.query(embedding); } \ No newline at end of file diff --git a/src/Server.h b/src/Server.h index 83d8d52..a972a08 100644 --- a/src/Server.h +++ b/src/Server.h @@ -14,8 +14,8 @@ struct ClientInfo { class Server{ public: - Server(int port, Vector7D& _embedding, bool debug) - : port(port), embedding(_embedding), debug(debug) {} + Server(int port, Vector7D& _embedding, bool debug, std::string _http_ip, int _http_port, std::string _http_page) + : port(port), embedding(_embedding), debug(debug), http_ip(_http_ip), http_port(_http_port), http_page(_http_page) {} void start(); void update(); @@ -23,6 +23,8 @@ class Server{ void printClients(); void updateEmbedding(); void checkActivity(); + void sendHttpRequest(); + int port; ofxTCPServer server; @@ -31,10 +33,15 @@ class Server{ bool is_active; Vector7D& embedding; + Request http; + std::string http_ip; + int http_port; + std::string http_page; + VPResp vp_resp; + private: Vector7D previous_embedding; std::chrono::time_point last_change_time; - bool hasChanged(); }; #endif \ No newline at end of file diff --git a/src/ofApp.cpp b/src/ofApp.cpp index d2375da..665395b 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -20,6 +20,7 @@ void ofApp::setup(){ /* setup video */ player.Setup(); player.SetVideo("videos/demo.mp4", model_output_fbo_1); + player.SetFrame("demo.jpg"); /* setup models (modelPath, log, useCuda) */ ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/onnx-test/bin/data/depth_anything_v2_vitb.onnx"; @@ -60,7 +61,7 @@ void ofApp::setup(){ */ /* setup server (http & tcp) */ - server = std::make_unique(12345, embedding, true); + server = std::make_unique(12345, embedding, false, "192.168.0.253", 2000, "search"); server->start(); } @@ -91,8 +92,8 @@ void ofApp::update(){ map.Draw(); } - /* Setup model input using ofImage, allocated fbo */ - player.Update(img); + /* Setup model input using ofImage, allocated fbo, checks if a frame or video_frme should be drawn to the screen */ + player.Update(img, server->is_active); /* Run Models, and set pixels */ try{ @@ -144,7 +145,7 @@ void ofApp::draw(){ ofPopMatrix(); printEmotions(); - + // emoteImage.draw(640, 0); // for(auto& face : detected_faces){ // ofDrawBitmapString(std::to_string(face.box.emotional_state.emotions[0]), 700, 300); @@ -220,6 +221,11 @@ void ofApp::printEmotions(){ ofPopMatrix(); } +/* check vp_resp, set frame to video_player fbo (centered + scaled) */ +void ofApp::displayFrame(){ + +} + //-------------------------------------------------------------- void ofApp::keyPressed(int key){ // if (key=OF_KEY_LEFT){ diff --git a/src/ofApp.h b/src/ofApp.h index 4093219..a330f2a 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -35,6 +35,7 @@ class ofApp : public ofBaseApp{ void inferEmotionalState(); void renderDepthMap(); void printEmotions(); + void displayFrame(); float window_height; float window_width; @@ -70,6 +71,7 @@ class ofApp : public ofBaseApp{ ofFbo video_player_fbo; ofFbo model_output_fbo; ofFbo model_output_fbo_1; + ofFbo model_output_fbo_frame; ofFbo screen_fbo; ModelThread threadMap;