From 1d53af5b0f46a16ce722d6724c92979606c9ddf9 Mon Sep 17 00:00:00 2001 From: cailean Date: Mon, 7 Oct 2024 12:42:46 +0100 Subject: [PATCH] tcp server working! --- src/Request.cpp | 31 ++++++++++++++ src/Request.h | 44 ++++++++++++++++++++ src/Server.cpp | 107 ++++++++++++++++++++++++++++++++++++++++++++++++ src/Server.h | 40 ++++++++++++++++++ src/ofApp.cpp | 33 +++++++++------ src/ofApp.h | 7 +++- 6 files changed, 248 insertions(+), 14 deletions(-) create mode 100644 src/Request.cpp create mode 100644 src/Request.h create mode 100644 src/Server.cpp create mode 100644 src/Server.h diff --git a/src/Request.cpp b/src/Request.cpp new file mode 100644 index 0000000..dc3db8b --- /dev/null +++ b/src/Request.cpp @@ -0,0 +1,31 @@ +#include "Request.h" + +/* setup http server */ +void Request::setup(std::string ip, int port, std::string page){ + req.method = ofHttpRequest::POST; + req.url = "http://" + ip + ":" + ofToString(port) + "/" + page; + req.headers["Content-Type"] = "application/json"; +} + +/* send a request to vp_server & return frame/video/folder */ +VPResp Request::query(Vector7D in){ + req.body = "{\"vector\": [" + + ofToString(in.angry) + "," + + ofToString(in.disgust) + "," + + ofToString(in.fear) + "," + + ofToString(in.happy) + "," + + ofToString(in.sad) + "," + + ofToString(in.surprise) + "," + + ofToString(in.neutral) + "]}"; + + auto resp = http.handleRequest(req); + json_resp = ofJson::parse(resp.data.getText()); + + VPResp vp_resp; + vp_resp.folder = json_resp["folder"]; + vp_resp.image = json_resp["image"]; + vp_resp.video = json_resp["video"]; + vp_resp.frame = json_resp["frame"]; + + return vp_resp; +} \ No newline at end of file diff --git a/src/Request.h b/src/Request.h new file mode 100644 index 0000000..7d61ffa --- /dev/null +++ b/src/Request.h @@ -0,0 +1,44 @@ +#ifndef _REQUEST +#define _REQUEST + +#include "ofMain.h" + +/* emotional embedding */ +struct Vector7D { + float angry; + float disgust; + float fear; + float happy; + float sad; + float surprise; + float neutral; + + bool operator!=(const Vector7D &other) const { + return angry != other.angry || + disgust != other.disgust || + fear != other.fear || + happy != other.happy || + sad != other.sad || + surprise != other.surprise || + neutral != other.neutral; + } +}; + +/* Vantage point query structure */ +struct VPResp{ + std::string folder; + std::string video; + std::string image; + int frame; +}; + +class Request{ + public: + void setup(std::string ip, int port, std::string page); + VPResp query(Vector7D in); + + ofHttpRequest req; + ofURLFileLoader http; + ofJson json_resp; +}; +#endif \ No newline at end of file diff --git a/src/Server.cpp b/src/Server.cpp new file mode 100644 index 0000000..0177cdb --- /dev/null +++ b/src/Server.cpp @@ -0,0 +1,107 @@ +#include "Server.h" + +void Server::start(){ + server.setup(port); + is_active = true; + previous_embedding = embedding; + last_change_time = std::chrono::steady_clock::now(); +} + +void Server::update(){ + + for ( int i = 0; i < server.getLastID(); i++){ + if (server.isClientConnected(i)) { + const int buffer_size = 8; + char buffer[buffer_size]; + int bytes_recieved = server.receiveRawBytes(i, buffer, buffer_size); + + if (bytes_recieved == buffer_size){ + float value; + int id; + memcpy(&value, buffer, sizeof(float)); + memcpy(&id, buffer + sizeof(float), sizeof(int)); + + std::string ip_address = server.getClientIP(i); + + addOrUpdateClient(id, value, ip_address); + } + } + } + + updateEmbedding(); + checkActivity(); + + if(debug){ + printClients(); + } +} + +void Server::addOrUpdateClient(int client_id, float value, const std::string& ip_address){ + ClientInfo client; + + client.ip_address = ip_address; + client.value = value; + + clients[client_id] = client; +} + +void Server::updateEmbedding(){ + for(const auto& c : clients){ + const ClientInfo& info = c.second; + float val = std::round(info.value * 1000.0f) / 1000.0f; + switch(c.first){ + case 0: + embedding.angry = val; + break; + case 1: + embedding.disgust = val; + break; + case 2: + embedding.fear = val; + break; + case 3: + embedding.happy = val; + break; + case 4: + embedding.sad = val; + break; + case 5: + embedding.surprise = val; + break; + case 6: + embedding.neutral = val; + break; + } + } +} + +void Server::printClients(){ + for( const auto& c : clients){ + int id = c.first; + const ClientInfo& info = c.second; + std::cout << "id: " << id + << ", value: " << info.value + << ", IP: " << info.ip_address << std::endl; + } + + std::cout << is_active << std::endl; +} + +/* check if the controllers are in use */ +void Server::checkActivity(){ + if (previous_embedding != embedding) { // Check if embedding has changed + last_change_time = std::chrono::steady_clock::now(); // Reset the timer if there is a change + previous_embedding = embedding; // Update the previous embedding to the current one + is_active = true; + } else { + // Calculate the time since the last change + auto now = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(now - last_change_time).count(); + + if (duration >= 2) { + is_active = false; + } + } + + std::cout << is_active << std::endl; +} \ No newline at end of file diff --git a/src/Server.h b/src/Server.h new file mode 100644 index 0000000..83d8d52 --- /dev/null +++ b/src/Server.h @@ -0,0 +1,40 @@ +#ifndef _SERVER +#define _SERVER + +#include "ofMain.h" +#include "ofxNetwork.h" +#include "Request.h" +#include +#include + +struct ClientInfo { + float value; + std::string ip_address; +}; + +class Server{ + public: + Server(int port, Vector7D& _embedding, bool debug) + : port(port), embedding(_embedding), debug(debug) {} + + void start(); + void update(); + void addOrUpdateClient(int client_id, float value, const std::string& ip_address); + void printClients(); + void updateEmbedding(); + void checkActivity(); + + int port; + ofxTCPServer server; + std::unordered_map clients; + bool debug; + bool is_active; + Vector7D& embedding; + + private: + Vector7D previous_embedding; + std::chrono::time_point last_change_time; + bool hasChanged(); +}; + +#endif \ No newline at end of file diff --git a/src/ofApp.cpp b/src/ofApp.cpp index aae8e29..d2375da 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -58,12 +58,20 @@ void ofApp::setup(){ croppedFaces.push_back(tempImage); } */ + + /* setup server (http & tcp) */ + server = std::make_unique(12345, embedding, true); + server->start(); } //-------------------------------------------------------------- void ofApp::update(){ + + /* update server - check for clients and values */ + server->update(); + /* Check to see if the application has moved to the first frame As the models need to load first, as the first inference is quite slow */ if(ofGetFrameNum() > 0 && ofGetFrameNum() < 2){ @@ -134,6 +142,8 @@ void ofApp::draw(){ ofSetBackgroundColor(0); tf.drawString(std::to_string(ofGetFrameRate()), 10, 30); ofPopMatrix(); + + printEmotions(); // emoteImage.draw(640, 0); // for(auto& face : detected_faces){ @@ -142,19 +152,6 @@ void ofApp::draw(){ } -void ofApp::inferDepthImage(ofFbo& fbo, ofImage& img, Onnx& model){ - auto output_tensors = model.Run(img); - float* output_ptr = output_tensors.front().GetTensorMutableData(); - size_t num_elements = output_tensors.front().GetTensorTypeAndShapeInfo().GetElementCount(); - - float min_value = model.ReduceMin(output_ptr, num_elements); - float max_value = model.ReduceMax(output_ptr, num_elements); - - model.Normalize(output_ptr, num_elements, min_value, max_value); - - model.DataToFbo(output_ptr, 518, 518, fbo); -} - //-------------------------------------------------------------- void ofApp::inferEmotionalState(){ /* @@ -213,6 +210,16 @@ void ofApp::renderDepthMap(){ rampedFbo.draw(0,0); } +void ofApp::printEmotions(){ + ofPushMatrix(); + ofSetColor(255); + ofSetBackgroundColor(0); + char buffer[50]; // Create a buffer to hold the formatted string + std::sprintf(buffer, "Value: %.3f", embedding.neutral); // + tf.drawString(buffer, 10, 90); + ofPopMatrix(); +} + //-------------------------------------------------------------- void ofApp::keyPressed(int key){ // if (key=OF_KEY_LEFT){ diff --git a/src/ofApp.h b/src/ofApp.h index 63f89fe..4093219 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -11,6 +11,8 @@ #include #include #include "ModelThread.h" +#include "Server.h" +#include "Request.h" class ofApp : public ofBaseApp{ @@ -32,7 +34,7 @@ class ofApp : public ofBaseApp{ void gotMessage(ofMessage msg); void inferEmotionalState(); void renderDepthMap(); - void inferDepthImage(ofFbo& fbo, ofImage& img, Onnx& model); + void printEmotions(); float window_height; float window_width; @@ -74,4 +76,7 @@ class ofApp : public ofBaseApp{ ModelThread threadVideo; ModelThread threadYolo; ModelThread threadEmotion; + + std::unique_ptr server; + Vector7D embedding; };