right up until shader!

src/Map.cpp

@@ -1,4 +1,5 @@
 #include "Map.h"
+#include "algorithm"
 
 Map::Map(){
     
@@ -6,6 +7,7 @@ Map::Map(){
 
 void Map::Setup(){
     ofDisableArbTex();
+    scale = 10000;
     isDebug = true;
 
     json_embeddings = ofLoadJson(jsonPath);
@@ -17,21 +19,107 @@ void Map::Setup(){
         SetupTSNE();
     }
 
-    mapFbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA);
+    mapFbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGB);
+
+    fboImage.allocate(ofGetWindowWidth(), ofGetWindowHeight(), OF_IMAGE_COLOR);
+
+    Setup3D();
+
+    SetupNodes();
 }
 
 void Map::Update(){
-    
+    time = ofGetElapsedTimef();
+    for(auto& n : nodes){
+        glm::vec2 offset = n.amplitude * sin(n.speed * time);
+        n.offset = glm::vec3(offset.x, offset.y, 0);
+    }
 }
 
 void Map::Draw(){
+    mapFbo.begin();
+
+    cam.begin();
+
+    ofClear(0, 0, 0, 1);
 
+    for (auto& n :nodes){
+            n.texture.getTexture().bind();
+            ofPushMatrix();
+                ofFill();
+                n.geom.setPosition(n.position + n.offset);
+                n.geom.draw();
+            ofPopMatrix();
+            n.texture.getTexture().unbind();
+    }
+
+    cam.end();
+
+    mapFbo.end();
+
+    mapFbo.readToPixels(fboPixels);
+    fboImage.setFromPixels(fboPixels);
+
+    //mapFbo.draw(0, 0);
 }
 
+/*
+    Setup texture for each node
+*/
 void Map::SetupNodes(){
+    std::cout << "Setting up nodes.." << std::endl;
+    for (auto& node : nodes){
+        ofImage img;
+        ofPlanePrimitive plane;
+
+        img.load("data/" + node.image_path);
+        int imageW = img.getWidth();
+        int imageH = img.getHeight();
+        int maxWidth = 100;
+        int maxHeight = 100;
+        float aspectRatio = (float)imageW / (float)imageH;
+
+        // Determine plane dimensions based on aspect ratio and constraints
+        float planeW, planeH;
+        if (aspectRatio > (float)maxWidth / maxHeight) {
+            // Image is wider relative to the max constraints
+            planeW = maxWidth;
+            planeH = maxWidth / aspectRatio;
+        } else {
+            // Image is taller relative to the max constraints
+            planeH = maxHeight;
+            planeW = maxHeight * aspectRatio;
+        }
 
+        // Ensure that dimensions do not exceed constraints
+        if (planeH > maxHeight) {
+            planeH = maxHeight;
+            planeW = maxHeight * aspectRatio;
+        }
+        if (planeW > maxWidth) {
+            planeW = maxWidth;
+            planeH = maxWidth / aspectRatio;
+        }
+
+        plane.set(planeW, planeH);
+        plane.setPosition(node.position);
+        plane.setResolution(2, 2);
+        plane.mapTexCoordsFromTexture(img.getTexture());
+        img.getTexture().setTextureWrap(GL_REPEAT, GL_REPEAT);
+        plane.setScale(1);
+
+        node.geom = plane;
+        node.texture = img;
+        node.speed = glm::vec2(ofRandom(0.1, 2), ofRandom(0.1, 2));
+        node.amplitude = glm::vec2(ofRandom(1, 5), ofRandom(1, 5));
+        node.phase = glm::vec2(ofRandom(0, TWO_PI), ofRandom(0, TWO_PI));
+    }
+    std::cout << "Finished setting up nodes!" << std::endl;
 }
 
+/*
+    Setup TSNE - reads JSON, converts to points, creates nodes & hashmap
+*/
 void Map::SetupTSNE(){
     for (const auto& entry: json_embeddings) {
         if (entry.contains("vector") && entry["vector"].is_array()) {
@@ -58,13 +146,47 @@ void Map::SetupTSNE(){
     }
 
     std::cout << nodes.size() << std::endl;
-    std::cout << embeddings.size() << std::endl;
+
     points = tsne.run(embeddings, dims, perplexity, theta, normalise, runManually);
 
     for (size_t i = 0; i < points.size(); i++){
         const auto& vec = points[i];
         auto& n = nodes[i];
-        n.position = glm::vec3(vec[0], vec[1], vec[2]);
+        n.position = ( (glm::vec3(vec[0], vec[1], vec[2]) * scale) - (scale / 2) );
+        node_hash_map[n.image_path] = &n;
+    }
+
+    SortNodes();
+}
+
+/*
+    Setup 3D environment
+*/
+void Map::Setup3D(){
+    cam.setFov(20);
+    cam.removeAllInteractions();
+    cam.addInteraction(ofEasyCam::TRANSFORM_TRANSLATE_XY, OF_MOUSE_BUTTON_LEFT);
+    cam.addInteraction(ofEasyCam::TRANSFORM_TRANSLATE_Z, OF_MOUSE_BUTTON_MIDDLE);
+}
+
+/*
+    Query hashmap
+*/
+void Map::SearchMap(){
+    std::string t = "dataset/20230601_webexclusi-irelandsey_cl11536538/segmented_images/000000000_0.png";
+    if(node_hash_map.find(t) != node_hash_map.end()){
+        Node* n = node_hash_map[t];
+        std::cout << n->foldername << std::endl;
     }
+}
 
+/*
+    Sort nodes by z position, for draw calls (lowest -> highest)
+*/
+void Map::SortNodes(){
+    std::cout << "Sorting Nodes.." << std::endl;
+    std::sort(nodes.begin(), nodes.end(), [](const Node& a, const Node&b){
+        return a.position.z < b.position.z;
+    });
+    std::cout << "Finished Sorting!" << std::endl;
 }

src/Map.h

@@ -9,7 +9,9 @@ struct Node {
     glm::vec2 speed;
     glm::vec2 amplitude;
     glm::vec2 phase;
+    glm::vec3 offset;
     ofImage texture;
+    ofPlanePrimitive geom;
     glm::vec3 position;
     std::string image_path;
     std::string foldername;
@@ -20,31 +22,30 @@ struct Node {
 class Map {
     public:
 
-    /*
-        Constructor 
-    */
-
-    Map();
-    
-
     /*
         Methods
     */
 
+    Map();
     void Setup();
     void Update();
     void Draw();
     void SetupNodes();
     void SetupTSNE();
+    void Setup3D();
+
+    void SearchMap();
+    void SortNodes();
 
     /*
         Variables
     */
     bool isDebug;
+    float time;
 
     ofFbo mapFbo;
-    vector<ofPlanePrimitive> geoms;
     vector<Node> nodes;
+    float scale;
 
     ofxTSNE tsne;
     vector<vector<double>> points;
@@ -57,9 +58,19 @@ class Map {
     ofJson json_embeddings;
     std::string jsonPath = "data/json/embeddings.json";
 
-    private:
+    std::unordered_map<std::string, Node*> node_hash_map;
+
+    /*
+        3D Variables
+    */
+    ofLight light;
+    ofEasyCam cam;
+    float zoom;
 
+    ofImage fboImage;
+    ofPixels fboPixels;
 
+    private:
 };
 
 #endif

src/Onnx.cpp

@@ -1,4 +1,5 @@
 #include "Onnx.h"
+#include <cmath> 
 
 // Setups the model. CUDA is enabled by default
 void Onnx::Setup(ORTCHAR_T* modelPath, bool isLog, bool useCuda){
@@ -75,14 +76,10 @@ std::vector<Ort::Value> Onnx::Run(ofImage &img){
     std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char),
                     [&](const std::string& str) { return str.c_str(); });
 
-    std::cout << "Running model... ";
-
     try {
         auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(),
                                         input_names_char.size(), output_names_char.data(), output_names_char.size());
 
-        std::cout << "Done!" << std::endl;
-
         if (timeStamp) {
             auto end = std::chrono::high_resolution_clock::now();
             std::chrono::duration<double, std::milli> elapsed = end - start;
@@ -146,14 +143,10 @@ std::vector<Ort::Value> Onnx::RunBatch(std::vector<ofImage>& images){
     std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char),
                     [&](const std::string& str) { return str.c_str(); });
 
-    std::cout << "Running model... ";
-
     try {
         auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(),
                                         input_names_char.size(), output_names_char.data(), output_names_char.size());
 
-        std::cout << "Done!" << std::endl;
-
         if (timeStamp) {
             auto end = std::chrono::high_resolution_clock::now();
             std::chrono::duration<double, std::milli> elapsed = end - start;
@@ -235,7 +228,7 @@ void Onnx::Normalize(float* data, size_t size, float min_value, float max_value)
 }
 
 // Coverts the output tensor data to a texture of a given ofFbo.
-void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo){
+void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo, ofShader& shader){
     // Convert data into opencv mat
     cv::Mat inputMat(height, width, CV_32FC1, const_cast<float*>(data));
 
@@ -261,8 +254,11 @@ void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo)
 
     // Update FBO with new pixels
     fbo.begin();
+    shader.begin();
     ofTexture& texture = fbo.getTexture();
     texture.loadData(pixels);
+    shader.setUniformTexture("tex", texture, 0);
+    shader.end();
     fbo.end();
 
 

src/Onnx.h

@@ -31,9 +31,9 @@
         float ReduceMin(const float* data, size_t size);
         float ReduceMax(const float* data, size_t size);
         void Normalize(float* data, size_t size, float min_value, float max_value);
-        void DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo);
+        void DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo, ofShader& shader);
         void Softmax(float* data, size_t size);
-        bool timeStamp = true;
+        bool timeStamp = false;
         bool log = false;
     
     protected:

src/Player.cpp

@@ -44,7 +44,9 @@ ofPixels Player::GetVideoPixels(){
 void Player::SetVideo(std::string path, ofFbo &fbo){
     videoPlayer.load(path);
     videoPlayer.setFrame(800);
-    fbo.allocate(videoPlayer.getWidth(), videoPlayer.getHeight(), GL_RGB);
+    //fbo.allocate(videoPlayer.getWidth(), videoPlayer.getHeight(), GL_RGB);
+    // Just setting the video dims here for the tsne map!
+    fbo.allocate(1600, 800, GL_RGB);
 }
 
 // Sets a random frame in the active video

src/ofApp.cpp

@@ -21,6 +21,8 @@ void ofApp::setup(){
     yolo.Setup(modelPath2, false, true);
     depth.Setup(modelPath, false, true);
     emotion.Setup(modelPath3, false, true);
+
+    depthToColourShader.load("data/shader/rampShader.vert", "data/shader/rampShader.frag");
 }
 
 
@@ -35,13 +37,21 @@ void ofApp::update(){
     /* Clear detetced face list */
     detected_faces.clear();
 
+    /* tsnemap fbo - first run to allocate fbo */
+    map.Update();
+
+    if(firstRun){
+        map.Draw(); 
+    }
+    
     /* Setup model input using ofImage */
     player.Update(img);
     img.setFromPixels(player.GetVideoPixels());
     
+    
     /* Run Models */
     try{
-        auto output_tensors = depth.Run(img);
+        auto output_tensors = depth.Run(map.fboImage);
         float* output_ptr = output_tensors.front().GetTensorMutableData<float>();
         size_t num_elements = output_tensors.front().GetTensorTypeAndShapeInfo().GetElementCount();
 
@@ -50,9 +60,9 @@ void ofApp::update(){
 
         depth.Normalize(output_ptr, num_elements, min_value, max_value);
 
-        depth.DataToFbo(output_ptr, 518, 518, fbo);
+        depth.DataToFbo(output_ptr, 518, 518, fbo, depthToColourShader);
 
-        auto output_tensors_face = yolo.Run(img);
+        auto output_tensors_face = yolo.Run(map.fboImage);
 
         auto output_faces = output_tensors_face.front().GetTensorTypeAndShapeInfo().GetShape();
 
@@ -100,13 +110,15 @@ void ofApp::update(){
 
 //--------------------------------------------------------------
 void ofApp::draw(){
+
+    map.Draw();
+
     fbo.draw(0, 0);
 
     if(!firstRun){
         faceDetector.DrawBox(detected_faces);
         faceDetector.DrawCenter(detected_faces);
     }
-    
 //   emoteImage.draw(640, 0);
 //   for(auto& face : detected_faces){
 //     ofDrawBitmapString(std::to_string(face.box.emotional_state.emotions[0]), 700, 300);

src/ofApp.h

@@ -49,4 +49,6 @@ class ofApp : public ofBaseApp{
 		std::vector<types::BoxfWithLandmarks> detected_faces;
 
 		Map map;
+
+		ofShader depthToColourShader;
 };