diff --git a/src/Map.cpp b/src/Map.cpp index 7b23eb1..ffae41e 100644 --- a/src/Map.cpp +++ b/src/Map.cpp @@ -1,4 +1,5 @@ #include "Map.h" +#include "algorithm" Map::Map(){ @@ -6,6 +7,7 @@ Map::Map(){ void Map::Setup(){ ofDisableArbTex(); + scale = 10000; isDebug = true; json_embeddings = ofLoadJson(jsonPath); @@ -17,21 +19,107 @@ void Map::Setup(){ SetupTSNE(); } - mapFbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA); + mapFbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGB); + + fboImage.allocate(ofGetWindowWidth(), ofGetWindowHeight(), OF_IMAGE_COLOR); + + Setup3D(); + + SetupNodes(); } void Map::Update(){ - + time = ofGetElapsedTimef(); + for(auto& n : nodes){ + glm::vec2 offset = n.amplitude * sin(n.speed * time); + n.offset = glm::vec3(offset.x, offset.y, 0); + } } void Map::Draw(){ - + mapFbo.begin(); + + cam.begin(); + + ofClear(0, 0, 0, 1); + + for (auto& n :nodes){ + n.texture.getTexture().bind(); + ofPushMatrix(); + ofFill(); + n.geom.setPosition(n.position + n.offset); + n.geom.draw(); + ofPopMatrix(); + n.texture.getTexture().unbind(); + } + + cam.end(); + + mapFbo.end(); + + mapFbo.readToPixels(fboPixels); + fboImage.setFromPixels(fboPixels); + + //mapFbo.draw(0, 0); } +/* + Setup texture for each node +*/ void Map::SetupNodes(){ + std::cout << "Setting up nodes.." << std::endl; + for (auto& node : nodes){ + ofImage img; + ofPlanePrimitive plane; + + img.load("data/" + node.image_path); + int imageW = img.getWidth(); + int imageH = img.getHeight(); + int maxWidth = 100; + int maxHeight = 100; + float aspectRatio = (float)imageW / (float)imageH; + + // Determine plane dimensions based on aspect ratio and constraints + float planeW, planeH; + if (aspectRatio > (float)maxWidth / maxHeight) { + // Image is wider relative to the max constraints + planeW = maxWidth; + planeH = maxWidth / aspectRatio; + } else { + // Image is taller relative to the max constraints + planeH = maxHeight; + planeW = maxHeight * aspectRatio; + } + + // Ensure that dimensions do not exceed constraints + if (planeH > maxHeight) { + planeH = maxHeight; + planeW = maxHeight * aspectRatio; + } + if (planeW > maxWidth) { + planeW = maxWidth; + planeH = maxWidth / aspectRatio; + } + plane.set(planeW, planeH); + plane.setPosition(node.position); + plane.setResolution(2, 2); + plane.mapTexCoordsFromTexture(img.getTexture()); + img.getTexture().setTextureWrap(GL_REPEAT, GL_REPEAT); + plane.setScale(1); + + node.geom = plane; + node.texture = img; + node.speed = glm::vec2(ofRandom(0.1, 2), ofRandom(0.1, 2)); + node.amplitude = glm::vec2(ofRandom(1, 5), ofRandom(1, 5)); + node.phase = glm::vec2(ofRandom(0, TWO_PI), ofRandom(0, TWO_PI)); + } + std::cout << "Finished setting up nodes!" << std::endl; } +/* + Setup TSNE - reads JSON, converts to points, creates nodes & hashmap +*/ void Map::SetupTSNE(){ for (const auto& entry: json_embeddings) { if (entry.contains("vector") && entry["vector"].is_array()) { @@ -58,13 +146,47 @@ void Map::SetupTSNE(){ } std::cout << nodes.size() << std::endl; - std::cout << embeddings.size() << std::endl; + points = tsne.run(embeddings, dims, perplexity, theta, normalise, runManually); for (size_t i = 0; i < points.size(); i++){ const auto& vec = points[i]; auto& n = nodes[i]; - n.position = glm::vec3(vec[0], vec[1], vec[2]); + n.position = ( (glm::vec3(vec[0], vec[1], vec[2]) * scale) - (scale / 2) ); + node_hash_map[n.image_path] = &n; } + SortNodes(); +} + +/* + Setup 3D environment +*/ +void Map::Setup3D(){ + cam.setFov(20); + cam.removeAllInteractions(); + cam.addInteraction(ofEasyCam::TRANSFORM_TRANSLATE_XY, OF_MOUSE_BUTTON_LEFT); + cam.addInteraction(ofEasyCam::TRANSFORM_TRANSLATE_Z, OF_MOUSE_BUTTON_MIDDLE); +} + +/* + Query hashmap +*/ +void Map::SearchMap(){ + std::string t = "dataset/20230601_webexclusi-irelandsey_cl11536538/segmented_images/000000000_0.png"; + if(node_hash_map.find(t) != node_hash_map.end()){ + Node* n = node_hash_map[t]; + std::cout << n->foldername << std::endl; + } +} + +/* + Sort nodes by z position, for draw calls (lowest -> highest) +*/ +void Map::SortNodes(){ + std::cout << "Sorting Nodes.." << std::endl; + std::sort(nodes.begin(), nodes.end(), [](const Node& a, const Node&b){ + return a.position.z < b.position.z; + }); + std::cout << "Finished Sorting!" << std::endl; } \ No newline at end of file diff --git a/src/Map.h b/src/Map.h index b848c75..44f8d92 100644 --- a/src/Map.h +++ b/src/Map.h @@ -9,7 +9,9 @@ struct Node { glm::vec2 speed; glm::vec2 amplitude; glm::vec2 phase; + glm::vec3 offset; ofImage texture; + ofPlanePrimitive geom; glm::vec3 position; std::string image_path; std::string foldername; @@ -20,31 +22,30 @@ struct Node { class Map { public: - /* - Constructor - */ - - Map(); - - /* Methods */ + Map(); void Setup(); void Update(); void Draw(); void SetupNodes(); void SetupTSNE(); + void Setup3D(); + + void SearchMap(); + void SortNodes(); /* Variables */ bool isDebug; + float time; ofFbo mapFbo; - vector geoms; vector nodes; + float scale; ofxTSNE tsne; vector> points; @@ -57,9 +58,19 @@ class Map { ofJson json_embeddings; std::string jsonPath = "data/json/embeddings.json"; - private: + std::unordered_map node_hash_map; + + /* + 3D Variables + */ + ofLight light; + ofEasyCam cam; + float zoom; + ofImage fboImage; + ofPixels fboPixels; + private: }; #endif \ No newline at end of file diff --git a/src/Onnx.cpp b/src/Onnx.cpp index 4a543e3..62e468d 100644 --- a/src/Onnx.cpp +++ b/src/Onnx.cpp @@ -1,4 +1,5 @@ #include "Onnx.h" +#include // Setups the model. CUDA is enabled by default void Onnx::Setup(ORTCHAR_T* modelPath, bool isLog, bool useCuda){ @@ -75,14 +76,10 @@ std::vector Onnx::Run(ofImage &img){ std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char), [&](const std::string& str) { return str.c_str(); }); - std::cout << "Running model... "; - try { auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(), input_names_char.size(), output_names_char.data(), output_names_char.size()); - std::cout << "Done!" << std::endl; - if (timeStamp) { auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration elapsed = end - start; @@ -146,14 +143,10 @@ std::vector Onnx::RunBatch(std::vector& images){ std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char), [&](const std::string& str) { return str.c_str(); }); - std::cout << "Running model... "; - try { auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(), input_names_char.size(), output_names_char.data(), output_names_char.size()); - std::cout << "Done!" << std::endl; - if (timeStamp) { auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration elapsed = end - start; @@ -235,7 +228,7 @@ void Onnx::Normalize(float* data, size_t size, float min_value, float max_value) } // Coverts the output tensor data to a texture of a given ofFbo. -void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo){ +void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo, ofShader& shader){ // Convert data into opencv mat cv::Mat inputMat(height, width, CV_32FC1, const_cast(data)); @@ -261,8 +254,11 @@ void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo) // Update FBO with new pixels fbo.begin(); + shader.begin(); ofTexture& texture = fbo.getTexture(); texture.loadData(pixels); + shader.setUniformTexture("tex", texture, 0); + shader.end(); fbo.end(); diff --git a/src/Onnx.h b/src/Onnx.h index 8e17c51..65299bc 100644 --- a/src/Onnx.h +++ b/src/Onnx.h @@ -31,9 +31,9 @@ float ReduceMin(const float* data, size_t size); float ReduceMax(const float* data, size_t size); void Normalize(float* data, size_t size, float min_value, float max_value); - void DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo); + void DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo, ofShader& shader); void Softmax(float* data, size_t size); - bool timeStamp = true; + bool timeStamp = false; bool log = false; protected: diff --git a/src/Player.cpp b/src/Player.cpp index 7672811..8d1a491 100644 --- a/src/Player.cpp +++ b/src/Player.cpp @@ -44,7 +44,9 @@ ofPixels Player::GetVideoPixels(){ void Player::SetVideo(std::string path, ofFbo &fbo){ videoPlayer.load(path); videoPlayer.setFrame(800); - fbo.allocate(videoPlayer.getWidth(), videoPlayer.getHeight(), GL_RGB); + //fbo.allocate(videoPlayer.getWidth(), videoPlayer.getHeight(), GL_RGB); + // Just setting the video dims here for the tsne map! + fbo.allocate(1600, 800, GL_RGB); } // Sets a random frame in the active video diff --git a/src/ofApp.cpp b/src/ofApp.cpp index 6720ebe..0d30b95 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -21,6 +21,8 @@ void ofApp::setup(){ yolo.Setup(modelPath2, false, true); depth.Setup(modelPath, false, true); emotion.Setup(modelPath3, false, true); + + depthToColourShader.load("data/shader/rampShader.vert", "data/shader/rampShader.frag"); } @@ -35,13 +37,21 @@ void ofApp::update(){ /* Clear detetced face list */ detected_faces.clear(); + /* tsnemap fbo - first run to allocate fbo */ + map.Update(); + + if(firstRun){ + map.Draw(); + } + /* Setup model input using ofImage */ player.Update(img); img.setFromPixels(player.GetVideoPixels()); + /* Run Models */ try{ - auto output_tensors = depth.Run(img); + auto output_tensors = depth.Run(map.fboImage); float* output_ptr = output_tensors.front().GetTensorMutableData(); size_t num_elements = output_tensors.front().GetTensorTypeAndShapeInfo().GetElementCount(); @@ -50,9 +60,9 @@ void ofApp::update(){ depth.Normalize(output_ptr, num_elements, min_value, max_value); - depth.DataToFbo(output_ptr, 518, 518, fbo); + depth.DataToFbo(output_ptr, 518, 518, fbo, depthToColourShader); - auto output_tensors_face = yolo.Run(img); + auto output_tensors_face = yolo.Run(map.fboImage); auto output_faces = output_tensors_face.front().GetTensorTypeAndShapeInfo().GetShape(); @@ -100,13 +110,15 @@ void ofApp::update(){ //-------------------------------------------------------------- void ofApp::draw(){ + + map.Draw(); + fbo.draw(0, 0); if(!firstRun){ faceDetector.DrawBox(detected_faces); faceDetector.DrawCenter(detected_faces); } - // emoteImage.draw(640, 0); // for(auto& face : detected_faces){ // ofDrawBitmapString(std::to_string(face.box.emotional_state.emotions[0]), 700, 300); diff --git a/src/ofApp.h b/src/ofApp.h index a51db76..6d23674 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -49,4 +49,6 @@ class ofApp : public ofBaseApp{ std::vector detected_faces; Map map; + + ofShader depthToColourShader; };