From 12cd2778aceeb7258174e49c5fb19241d425260d Mon Sep 17 00:00:00 2001 From: cailean Date: Wed, 21 Aug 2024 15:37:13 +0100 Subject: [PATCH] tsne added --- .vscode/c_cpp_properties.json | 8 ++-- Demo.code-workspace | 89 +---------------------------------- Demo.qbs | 8 ++++ addons.make | 2 + src/Player.cpp | 9 ++++ src/Player.h | 2 + src/TSNEMap.cpp | 78 +++++++++++++++++++++++++----- src/TSNEMap.h | 14 ++++++ src/ofApp.cpp | 1 + 9 files changed, 107 insertions(+), 104 deletions(-) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index cfa02d2..7e2796a 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -7,7 +7,6 @@ "/usr/local/include", "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/", "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/4.2.1", - "${workspaceRoot}/../../../addons/ofxGui", "${workspaceRoot}/../../../libs/openFrameworks", "${workspaceRoot}/../../../libs/openFrameworks/3d", "${workspaceRoot}/../../../libs/openFrameworks/app", @@ -71,6 +70,7 @@ "/usr/local/include", "/usr/lib/gcc/x86_64-linux-gnu/5/include-fixed", "/usr/include/x86_64-linux-gnu", + "${workspaceRoot}/../../../addons/ofxTSNE/src}", "${workspaceRoot}/../../../libs/openFrameworks", "${workspaceRoot}/../../../libs/openFrameworks/3d", "${workspaceRoot}/../../../libs/openFrameworks/app", @@ -106,7 +106,7 @@ "/usr/include/gstreamer-1.0", "/usr/include/glib-2.0", "${workspaceRoot}", - "${workspaceRoot}/../../../addons/ofxGui/src" + "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxTSNE/src" ], "browse": { "limitSymbolsToIncludedHeaders": true, @@ -165,7 +165,7 @@ "path": [ "${workspaceRoot}/../../../libs", "${workspaceRoot}/../../../addons", - "${workspaceRoot}/../../../libs/openFrameworks" + "${workspaceRoot}/../../../libs/openFrameworks" ] }, "intelliSenseMode": "clang-x64", @@ -175,4 +175,4 @@ } ], "version": 4 -} +} \ No newline at end of file diff --git a/Demo.code-workspace b/Demo.code-workspace index aca06f2..7c78bf5 100644 --- a/Demo.code-workspace +++ b/Demo.code-workspace @@ -10,92 +10,5 @@ "path": "${workspaceRoot}/../../../../addons" } ], - "settings": { - "files.associations": { - "ostream": "cpp", - "cctype": "cpp", - "clocale": "cpp", - "cmath": "cpp", - "csignal": "cpp", - "cstdarg": "cpp", - "cstddef": "cpp", - "cstdio": "cpp", - "cstdlib": "cpp", - "cstring": "cpp", - "ctime": "cpp", - "cwchar": "cpp", - "cwctype": "cpp", - "any": "cpp", - "array": "cpp", - "atomic": "cpp", - "strstream": "cpp", - "bit": "cpp", - "*.tcc": "cpp", - "bitset": "cpp", - "cfenv": "cpp", - "charconv": "cpp", - "chrono": "cpp", - "codecvt": "cpp", - "compare": "cpp", - "complex": "cpp", - "concepts": "cpp", - "condition_variable": "cpp", - "cstdint": "cpp", - "deque": "cpp", - "forward_list": "cpp", - "list": "cpp", - "map": "cpp", - "set": "cpp", - "string": "cpp", - "unordered_map": "cpp", - "unordered_set": "cpp", - "vector": "cpp", - "exception": "cpp", - "expected": "cpp", - "algorithm": "cpp", - "functional": "cpp", - "iterator": "cpp", - "memory": "cpp", - "memory_resource": "cpp", - "numeric": "cpp", - "optional": "cpp", - "random": "cpp", - "ratio": "cpp", - "regex": "cpp", - "source_location": "cpp", - "string_view": "cpp", - "system_error": "cpp", - "tuple": "cpp", - "type_traits": "cpp", - "utility": "cpp", - "fstream": "cpp", - "future": "cpp", - "initializer_list": "cpp", - "iomanip": "cpp", - "iosfwd": "cpp", - "iostream": "cpp", - "istream": "cpp", - "limits": "cpp", - "mutex": "cpp", - "new": "cpp", - "numbers": "cpp", - "ranges": "cpp", - "semaphore": "cpp", - "shared_mutex": "cpp", - "span": "cpp", - "sstream": "cpp", - "stdexcept": "cpp", - "stop_token": "cpp", - "streambuf": "cpp", - "thread": "cpp", - "cinttypes": "cpp", - "typeindex": "cpp", - "typeinfo": "cpp", - "valarray": "cpp", - "variant": "cpp", - "format": "cpp", - "stdfloat": "cpp", - "__nullptr": "cpp" - } - } + "settings": {} } \ No newline at end of file diff --git a/Demo.qbs b/Demo.qbs index e5cc53b..7243dee 100644 --- a/Demo.qbs +++ b/Demo.qbs @@ -12,8 +12,14 @@ Project{ name: { return FileInfo.baseName(sourceDirectory) } files: [ + 'src/ImageViewer.cpp', + 'src/ImageViewer.h', 'src/Player.cpp', 'src/Player.h', + 'src/Request.cpp', + 'src/Request.h', + 'src/TSNEMap.cpp', + 'src/TSNEMap.h', 'src/main.cpp', 'src/ofApp.cpp', 'src/ofApp.h', @@ -22,6 +28,8 @@ Project{ of.addons: [ 'ofxGui', 'ofxGui', + 'ofxTSNE', + 'ofxTSNE', ] // additional flags for the project. the of module sets some diff --git a/addons.make b/addons.make index 4d88fa0..78cb562 100644 --- a/addons.make +++ b/addons.make @@ -1,2 +1,4 @@ ofxGui ofxGui +ofxTSNE +ofxTSNE diff --git a/src/Player.cpp b/src/Player.cpp index 8fa5a88..51e5ad6 100644 --- a/src/Player.cpp +++ b/src/Player.cpp @@ -110,6 +110,8 @@ void Player::setupGUI(){ gui.add(currentFrameLabel.setup("Current Frame", ofToString(playerCurrentFrame))); gui.add(currentVideoLabel.setup("Video Name", videoPath)); + gui.add(audioToggle.setup("Audio", false)); + currentVector.angry = angrySlider; currentVector.disgust = disgustSlider; currentVector.fear = fearSlider; @@ -147,5 +149,12 @@ void Player::updateGUI(){ lastVector = currentVector; } + + if(audioToggle){ + videoPlayer.setVolume(1); + } else { + videoPlayer.setVolume(0); + } + currentFrameLabel = ofToString(playerCurrentFrame); } diff --git a/src/Player.h b/src/Player.h index bc1bef1..dd158a4 100644 --- a/src/Player.h +++ b/src/Player.h @@ -41,6 +41,8 @@ class Player { ofxFloatSlider surpriseSlider; ofxFloatSlider neutralSlider; + ofxToggle audioToggle; + ofxLabel currentFrameLabel; ofxLabel currentVideoLabel; ofxLabel currentIndexLabel; diff --git a/src/TSNEMap.cpp b/src/TSNEMap.cpp index 71722cb..bb33c62 100644 --- a/src/TSNEMap.cpp +++ b/src/TSNEMap.cpp @@ -6,13 +6,73 @@ TSNEMap::TSNEMap(){ } void TSNEMap::setup(){ - for (int i = 0; i < 5; ++i) { - + setupNodes(); + // Setup FBO + fbo.allocate(ofGetWindowWidth() / 3, ofGetWindowHeight() / 3 * 2, GL_RGBA); + cam.removeAllInteractions(); + cam.addInteraction(ofEasyCam::TRANSFORM_TRANSLATE_XY,OF_MOUSE_BUTTON_LEFT); + cam.enableOrtho(); + // Setup top-down orthographic camera + cam.setNearClip(-1000000); + cam.setFarClip(1000000); +} + +void TSNEMap::setupTSNE(std::string jsonPath){ + // Load in JSON + jsonEmbeddings = ofLoadJson(jsonPath); + + // Check if JSON is an array + if (!jsonEmbeddings.is_array()) { + ofLogError() << "JSON is not an array"; + return; + } + + // Loop through each entry in the JSON array + for (const auto& entry : jsonEmbeddings) { + // Check if the entry has a "vector" field + if (entry.contains("vector") && entry["vector"].is_array()) { + std::vector vec; + + // Loop through each value in the "vector" field + for (const auto& value : entry["vector"]) { + // Check if the value is a number and add it to the vector + if (value.is_number()) { + vec.push_back(value.get()); + } else { + ofLogError() << "Vector value is not a number"; + } + } + + // Add the extracted vector to the list + embeddings.push_back(vec); + } else { + ofLogError() << "Entry does not contain a valid 'vector' field"; + } + } + + // Run tsne + tsnePoints = tsne.run(embeddings, dims, perplexity, theta, normalise, runManually); + + // Output the vectors to the console (for debugging) + for (const auto& vec : tsnePoints) { + for (float value : vec) { + std::cout << value << " "; + } + std::cout << std::endl; + } +} + +void TSNEMap::setupNodes(){ + + int idx = 0; + + for (const auto& vec : tsnePoints) { ofImage img; ImageObject obj; ofPlanePrimitive plane; - img.load("images/happy.png"); + // what image? + img.load(jsonEmbeddings[idx]["image"]); int imageW = image.getWidth(); int imageH = image.getHeight(); @@ -47,7 +107,7 @@ void TSNEMap::setup(){ } plane.set(planeW, planeH); // Random width and height - plane.setPosition(ofRandom(-200, 200), ofRandom(-200, 200), 0); // Random position on XZ plane + plane.setPosition((vec[0] * 5000) - 2500, (vec[1] * 5000) - 2500, 0); // Random position on XZ plane plane.setResolution(2, 2); plane.mapTexCoordsFromTexture(img.getTexture()); @@ -61,16 +121,10 @@ void TSNEMap::setup(){ obj.phaseY = ofRandom(0, TWO_PI); // Random phase objects.push_back(obj); + + idx++; } - // Setup FBO - fbo.allocate(ofGetWindowWidth() / 3, ofGetWindowHeight() / 3 * 2, GL_RGBA); - cam.removeAllInteractions(); - cam.addInteraction(ofEasyCam::TRANSFORM_TRANSLATE_XY,OF_MOUSE_BUTTON_LEFT); - cam.enableOrtho(); - // Setup top-down orthographic camera - cam.setNearClip(-1000000); - cam.setFarClip(1000000); } void TSNEMap::update(){ diff --git a/src/TSNEMap.h b/src/TSNEMap.h index 3f0087b..dfbe63c 100644 --- a/src/TSNEMap.h +++ b/src/TSNEMap.h @@ -2,6 +2,7 @@ #define _TSNEMAP #include "ofMain.h" +#include "ofxTSNE.h" struct ImageObject { float speedX; @@ -25,6 +26,8 @@ class TSNEMap{ void setTexture(std::string imagePath); void updateCameraProjection(); void keyPressed(int key); + void setupTSNE(std::string jsonPath); + void setupNodes(); ofFbo fbo; vector planes; @@ -34,6 +37,17 @@ class TSNEMap{ ofImage image; float zoomLevel; + ofxTSNE tsne; + ofJson jsonEmbeddings; + vector > embeddings; + vector > tsnePoints; + + int dims = 2; + float perplexity = 20; + float theta = 0.2; + bool normalise = true; + bool runManually = false; + TSNEMap(); }; diff --git a/src/ofApp.cpp b/src/ofApp.cpp index cf43a01..093e943 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -9,6 +9,7 @@ void ofApp::setup(){ ofSetVerticalSync(true); player.setup(); imageViewer.setup(); + map.setupTSNE("embeddings.json"); map.setup(); }