cailean
/
sv-map
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

added querying

master
cailean 4 months ago
parent
26247cc2ed
commit
c756613651
15 changed files with 48635 additions and 101 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      .vscode/c_cpp_properties.json
  2. 48434
      bin/data/json/embeddings.json
  3. BIN
      bin/data/new_tree.bin
  4. BIN
      bin/data/vp-tree.bin
  5. BIN
      bin/image-to-mesh
  6. 4
      image-to-mesh.code-workspace
  7. 5
      src/Bullet.cpp
  8. 4
      src/Bullet.h
  9. 24
      src/network/Request.h
  10. 16
      src/network/Server.cpp
  11. 9
      src/network/Server.h
  12. 153
      src/ofApp.cpp
  13. 29
      src/ofApp.h
  14. 36
      src/vp/VP.cpp
  15. 8
      src/vp/VP.h

2
.vscode/c_cpp_properties.json
View File

@ -118,7 +118,7 @@
"/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxVideoRecorder/src", "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxVideoRecorder/src",
"/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxOsc/*", "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxOsc/*",
"/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxNetwork/src/*", "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxNetwork/src/*",
"/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxTSNE/src/*", "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxTSNE/src/**",
"/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxPiMapper/src/*", "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxPiMapper/src/*",
"/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxXmlSettings/src", "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxXmlSettings/src",
"/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxXmlSettings/libs", "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/addons/ofxXmlSettings/libs",

48434
bin/data/json/embeddings.json
View File

File diff suppressed because it is too large

BIN
bin/data/new_tree.bin
View File

Binary file not shown.

BIN
bin/data/vp-tree.bin
View File

Binary file not shown.

BIN
bin/image-to-mesh
View File

Binary file not shown.

4
image-to-mesh.code-workspace
View File

@ -107,7 +107,9 @@
"__bit_reference": "cpp", "__bit_reference": "cpp",
"filesystem": "cpp", "filesystem": "cpp",
"executor": "cpp", "executor": "cpp",
"timer": "cpp" "timer": "cpp",
"buffer": "cpp",
"socket": "cpp"
} }
} }
} }

5
src/Bullet.cpp
View File

@ -1,6 +1,6 @@
#include "Bullet.h" #include "Bullet.h"
void Bullet::setup(){ void Bullet::setup(vector<Node>& _nodes){
/* setup camera & world */ /* setup camera & world */
camera_start_pos = ofVec3f(0, -3.f, -40.f); camera_start_pos = ofVec3f(0, -3.f, -40.f);
camera_end_position = glm::vec3(ofRandom(-100, 100), ofRandom(-100, 100), ofRandom(0, 0)); camera_end_position = glm::vec3(ofRandom(-100, 100), ofRandom(-100, 100), ofRandom(0, 0));
@ -25,6 +25,9 @@ void Bullet::setup(){
shader.load("shaders/vertex_snap"); shader.load("shaders/vertex_snap");
/* assign nodes */
nodes = _nodes;
std::cout << workerThreads.size() << std::endl; std::cout << workerThreads.size() << std::endl;
} }

4
src/Bullet.h
View File

@ -13,11 +13,13 @@ struct Node {
ofImage img; ofImage img;
ofTexture tex; ofTexture tex;
glm::vec3 scale; glm::vec3 scale;
glm::vec3 tsne_position;
vector<float> raw_embedding;
}; };
class Bullet{ class Bullet{
public: public:
void setup(); void setup(vector<Node>& _nodes);
void update(); void update();
void draw(); void draw();
void addMesh(ofMesh _mesh, ofMesh _simple_mesh, ofTexture _tex); void addMesh(ofMesh _mesh, ofMesh _simple_mesh, ofTexture _tex);

24
src/network/Request.h
View File

@ -21,6 +21,30 @@ struct Embedding {
bool operator!=(const Embedding &other) const { bool operator!=(const Embedding &other) const {
return emotions != other.emotions; return emotions != other.emotions;
} }
// New method to convert emotions to vector<float>
std::vector<float> to_vector() const {
std::vector<float> result;
result.reserve(emotions.size()); // Reserve space for efficiency
// Define the order of emotions
const std::vector<std::string> emotion_order = {
"angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"
};
// Fill the vector in the defined order
for (const auto& emotion : emotion_order) {
auto it = emotions.find(emotion);
if (it != emotions.end()) {
result.push_back(it->second);
} else {
// If an emotion is missing, push 0.0f
result.push_back(0.0f);
}
}
return result;
}
}; };
/* Vantage point query structure */ /* Vantage point query structure */

16
src/network/Server.cpp
View File

@ -4,7 +4,6 @@ void Server::start(){
std::cout << "Initialising TCP sever" << std::endl; std::cout << "Initialising TCP sever" << std::endl;
server.setup(port); server.setup(port);
osc_sender.setup(OSC_HOST, OSC_PORT); osc_sender.setup(OSC_HOST, OSC_PORT);
//http.setup(http_ip, http_port, http_page);
is_active = true; is_active = true;
previous_embedding = embedding; previous_embedding = embedding;
last_change_time = std::chrono::steady_clock::now(); last_change_time = std::chrono::steady_clock::now();
@ -52,7 +51,7 @@ void Server::update(ofFbo& esp_comp){
addOrUpdateClient(id, value, ip_address, true); addOrUpdateClient(id, value, ip_address, true);
std::cout << ip_address << " : " << id << std::endl; //std::cout << ip_address << " : " << id << std::endl;
} }
@ -140,11 +139,18 @@ void Server::print(){
/* check if the controllers are in use */ /* check if the controllers are in use */
void Server::checkActivity(){ void Server::checkActivity(){
if (previous_embedding.emotions["neutral"] != embedding.emotions["neutral"]) { // Check if embedding has changed if (previous_embedding.emotions != embedding.emotions) { // Check if embedding has changed
last_change_time = std::chrono::steady_clock::now(); // Reset the timer if there is a change last_change_time = std::chrono::steady_clock::now(); // Reset the timer if there is a change
previous_embedding = embedding; // Update the previous embedding to the current one previous_embedding = embedding; // Update the previous embedding to the current one
is_active = true; is_active = true;
//sendHttpRequest();
/* send a query to search the closest embedding in the vp-tree */
vector<vector<float>> vp_queries;
vp_queries.push_back(embedding.to_vector());
vector<int> ids = tree.query(vp_queries, 1);
chosen_node = &nodes[ids[0]];
ofLogNotice() << chosen_node->tsne_position;
} else { } else {
// Calculate the time since the last change // Calculate the time since the last change
auto now = std::chrono::steady_clock::now(); auto now = std::chrono::steady_clock::now();
@ -227,7 +233,7 @@ void Server::sendImagesToClients(vector<ofImage>& imgs){
// Send RGB pixels as raw bytes & check if the ip addresses align -> get id // Send RGB pixels as raw bytes & check if the ip addresses align -> get id
for(const auto& c_pair : clients){ for(const auto& c_pair : clients){
std::cout << c_pair.first << std::endl; //std::cout << c_pair.first << std::endl;
if(c_pair.second.ip_address == server.getClientIP(i)){ if(c_pair.second.ip_address == server.getClientIP(i)){
unsigned char* image_data = esp_pixels[c_pair.first].getData(); unsigned char* image_data = esp_pixels[c_pair.first].getData();

9
src/network/Server.h
View File

@ -7,6 +7,7 @@
#include <chrono> #include <chrono>
#include "ofxOsc.h" #include "ofxOsc.h"
#include "../vp/VP.h" #include "../vp/VP.h"
#include "../Bullet.h"
#define OSC_HOST "127.0.0.1" #define OSC_HOST "127.0.0.1"
#define OSC_PORT 9002 #define OSC_PORT 9002
@ -19,8 +20,8 @@ struct ClientInfo {
class Server{ class Server{
public: public:
Server(int _port, Embedding& _embedding, VP& _tree, bool debug, std::string _http_ip, int _http_port, std::string _http_page) Server(int _port, Embedding& _embedding, VP& _tree, vector<Node>& _nodes, bool debug, std::string _http_ip, int _http_port, std::string _http_page)
: port(_port), embedding(_embedding), tree(_tree), debug(debug), http_ip(_http_ip), http_port(_http_port), http_page(_http_page) {} : port(_port), embedding(_embedding), tree(_tree), debug(debug), nodes(_nodes), http_ip(_http_ip), http_port(_http_port), http_page(_http_page) {}
void start(); void start();
void update(ofFbo& esp_comp); void update(ofFbo& esp_comp);
@ -42,7 +43,9 @@ class Server{
bool debug; bool debug;
bool is_active; bool is_active;
Embedding embedding; Embedding embedding;
VP tree; VP& tree;
vector<Node>& nodes;
Node* chosen_node;
Request http; Request http;
std::string http_ip; std::string http_ip;

153
src/ofApp.cpp
View File

@ -3,17 +3,6 @@
//-------------------------------------------------------------- //--------------------------------------------------------------
void ofApp::setup(){ void ofApp::setup(){
const int vectorCount = 10000;
const int dimension = 7;
const int queryCount = 10;
const int k = 5; // Number of nearest neighbors to find
auto vectors = vp_tree.generateRandomVectors(vectorCount, dimension);
vpt::VpTree tree = vp_tree.buildTree(vectors);
vp_tree.saveTree("./data/vp-tree.bin", tree);
/* allocated fbo's */ /* allocated fbo's */
map_fbo.allocate((ofGetWindowWidth() / 3) * 2, ofGetWindowHeight(), GL_RGB); map_fbo.allocate((ofGetWindowWidth() / 3) * 2, ofGetWindowHeight(), GL_RGB);
portrait_fbo.allocate((ofGetWindowWidth() / 3) * 1, ofGetWindowHeight(), GL_RGBA); portrait_fbo.allocate((ofGetWindowWidth() / 3) * 1, ofGetWindowHeight(), GL_RGBA);
@ -49,7 +38,7 @@ void ofApp::setup(){
ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx"; ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx";
/* setup */ /* setup */
bullet.setup(); //bullet.setup(nodes);
depth_onnx.Setup(modelPath, true, true); depth_onnx.Setup(modelPath, true, true);
depth_onnx_esp.Setup(modelPath, true, true); depth_onnx_esp.Setup(modelPath, true, true);
@ -57,21 +46,6 @@ void ofApp::setup(){
depth_thread.setup(&model_image, &model_outptut_fbo, &depth_onnx); depth_thread.setup(&model_image, &model_outptut_fbo, &depth_onnx);
depth_esp.setup(&model_image_esp, &model_esp_out_fbo, &depth_onnx_esp); depth_esp.setup(&model_image_esp, &model_esp_out_fbo, &depth_onnx_esp);
/* vp-test */
auto queries = server->generateRandomVectors(queryCount, dimension);
for (const auto& query : queries) {
auto [distances, indices] = tree.getNearestNeighbors(query, k);
// Uncomment the following lines to print results for each query
std::cout << "Query: ";
for (double d : query) std::cout << d << " ";
std::cout << "\nNearest neighbors:\n";
for (int i = 0; i < k; ++i) {
std::cout << " Index: " << indices[i] << ", Distance: " << distances[i] << "\n";
}
}
/* mesh generation test */ /* mesh generation test */
std::string path = "images"; std::string path = "images";
@ -95,17 +69,19 @@ void ofApp::setup(){
vector<ofMesh> mesh_list; // vector<ofMesh> mesh_list;
for(auto& img : images){ // for(auto& img : images){
mesh_list = mesh_generator.generateSimplifiedMesh(img); // mesh_list = mesh_generator.generateSimplifiedMesh(img);
bullet.addMesh(mesh_list[0], mesh_list[1], img.getTexture()); // bullet.addMesh(mesh_list[0], mesh_list[1], img.getTexture());
mesh_list.clear(); // mesh_list.clear();
} // }
last_updated_time = ofGetElapsedTimef(); last_updated_time = ofGetElapsedTimef();
server = std::make_unique<Server>(6762, embed, vp_tree, false, "192.168.0.253", 2000, "search"); createNodes("data/json/embeddings.json");
server = std::make_unique<Server>(6762, embed, vp_tree, nodes, false, "192.168.0.253", 2000, "search");
server->start(); server->start();
} }
@ -144,7 +120,7 @@ void ofApp::update(){
std::cout << "Model did not run" << std::endl; std::cout << "Model did not run" << std::endl;
} }
bullet.update(); //bullet.update();
} }
//-------------------------------------------------------------- //--------------------------------------------------------------
@ -153,7 +129,7 @@ void ofApp::draw(){
ofPushStyle(); ofPushStyle();
map_fbo.begin(); map_fbo.begin();
ofClear(ofColor::grey); ofClear(ofColor::grey);
bullet.draw(); //bullet.draw();
map_fbo.end(); map_fbo.end();
ofPopStyle(); ofPopStyle();
@ -251,59 +227,102 @@ void ofApp::getNearestImages(){
model_image_esp.setFromPixels(esp_comp_pixels); model_image_esp.setFromPixels(esp_comp_pixels);
} }
//-------------------------------------------------------------- void ofApp::createNodes(std::string json_path){
void ofApp::keyPressed(int key){
} /* read in json */
json = ofLoadJson(json_path);
//-------------------------------------------------------------- if(!json.is_array()){
void ofApp::keyReleased(int key){ ofLogError() << "json is not array";
return;
} else {
ofLogNotice() << "json loaded";
}
} /* setup nodes */
for(const auto& j : json){
if(j.contains("vector") && j["vector"].is_array()){
Node n;
n.img.load(j["image"]);
std::vector<float> t_embedding;
//-------------------------------------------------------------- for (const auto& value: j["vector"]){
void ofApp::mouseMoved(int x, int y ){ if(value.is_number()){
t_embedding.push_back(value.get<float>());
} else {
ofLogError() << "Vector value is not a number";
}
}
} n.raw_embedding = t_embedding;
nodes.push_back(n);
}
}
ofLogNotice() << "node count: " << nodes.size();
//-------------------------------------------------------------- /* build vp_tree */
void ofApp::mouseDragged(int x, int y, int button){ auto vectors = createDoubleVectorFromNodes(nodes);
vp_tree.buildTree(vectors);
ofLogNotice() << "vpt built";
} /* run tsne */
vector<vector<float>> tsne_input;
std::vector<std::vector<double>> tsne_points;
//-------------------------------------------------------------- for(const auto& n : nodes){
void ofApp::mousePressed(int x, int y, int button){ tsne_input.push_back(n.raw_embedding);
}
} tsne_points = tsne.run(tsne_input, dims, perplexity, theta, normalise, runManually);
//-------------------------------------------------------------- for(size_t i = 0; i < tsne_points.size(); i++){
void ofApp::mouseReleased(int x, int y, int button){ const auto& vec = tsne_points[i];
auto& n = nodes[i];
n.tsne_position = (glm::vec3(vec[0] * tsne_scale, vec[1] * tsne_scale, -5.0f));
}
} /* vp-test */
// auto queries = server->generateRandomVectors(10, 7);
//-------------------------------------------------------------- // int k = 5;
void ofApp::mouseEntered(int x, int y){
} // vector<vector<float>> test_vectors;
//-------------------------------------------------------------- // // Set up random number generator
void ofApp::mouseExited(int x, int y){ // std::random_device rd;
// std::mt19937 gen(rd());
// std::uniform_real_distribution<> dis(0.0, 1.0); // Random floats between -1 and 1
} // for (int i = 0; i < 1; ++i) {
// std::vector<float> vec;
// for (int j = 0; j < 7; ++j) {
// vec.push_back(static_cast<float>(dis(gen)));
// }
// test_vectors.push_back(vec);
// }
//-------------------------------------------------------------- // std::cout << test_vectors.size() << std::endl;
void ofApp::windowResized(int w, int h){
// vector<int> q_index = vp_tree.query(test_vectors, 5);
} }
//-------------------------------------------------------------- std::vector<std::vector<double>> ofApp::createDoubleVectorFromNodes(const std::vector<Node>& nodes) {
void ofApp::gotMessage(ofMessage msg){ std::vector<std::vector<double>> result;
result.reserve(nodes.size());
} for (const auto& node : nodes) {
std::vector<double> doubleVector;
doubleVector.reserve(node.raw_embedding.size());
//-------------------------------------------------------------- for (const float& value : node.raw_embedding) {
void ofApp::dragEvent(ofDragInfo dragInfo){ doubleVector.push_back(static_cast<double>(value));
}
result.push_back(std::move(doubleVector));
}
return result;
} }
void ofApp::exit(){ void ofApp::exit(){

29
src/ofApp.h
View File

@ -10,6 +10,7 @@
#include "onnx/ModelThread.h" #include "onnx/ModelThread.h"
#include "network/Request.h" #include "network/Request.h"
#include "network/Server.h" #include "network/Server.h"
#include "ofxTSNE.h"
class ofApp : public ofBaseApp{ class ofApp : public ofBaseApp{
@ -18,18 +19,8 @@ class ofApp : public ofBaseApp{
void update(); void update();
void draw(); void draw();
void getNearestImages(); void getNearestImages();
void createNodes(std::string json_path);
void keyPressed(int key); std::vector<std::vector<double>> createDoubleVectorFromNodes(const std::vector<Node>& nodes);
void keyReleased(int key);
void mouseMoved(int x, int y );
void mouseDragged(int x, int y, int button);
void mousePressed(int x, int y, int button);
void mouseReleased(int x, int y, int button);
void mouseEntered(int x, int y);
void mouseExited(int x, int y);
void windowResized(int w, int h);
void dragEvent(ofDragInfo dragInfo);
void gotMessage(ofMessage msg);
void exit(); void exit();
ofEasyCam cam; ofEasyCam cam;
@ -52,6 +43,7 @@ class ofApp : public ofBaseApp{
ofPlanePrimitive plane; ofPlanePrimitive plane;
ofTexture bayer; ofTexture bayer;
VP vp_tree; VP vp_tree;
/* onnx */ /* onnx */
@ -73,4 +65,17 @@ class ofApp : public ofBaseApp{
ofPixels esp_comp_pixels; ofPixels esp_comp_pixels;
float last_updated_time; float last_updated_time;
/* nodes */
ofJson json;
vector<Node> nodes;
ofxTSNE tsne;
int dims = 2;
float perplexity = 20;
float theta = 0.2;
bool normalise = true;
bool runManually = false;
float tsne_scale = 1000;
}; };

36
src/vp/VP.cpp
View File

@ -23,9 +23,39 @@ vpt::VpTree VP::loadTree(std::string path){
void VP::saveTree(std::string filename, vpt::VpTree& tree){ void VP::saveTree(std::string filename, vpt::VpTree& tree){
tree.save(filename); tree.save(filename);
ofLogNotice() << "vpt saved";
} }
vpt::VpTree VP::buildTree(std::vector<vpt::Vector> vectors){ void VP::buildTree(std::vector<vpt::Vector> vectors){
vpt::VpTree tree(vectors); t = std::make_unique<vpt::VpTree>(vectors);
return tree; saveTree("./data/new_tree.bin", *t);
}
vector<int> VP::query(vector<vector<float>> _embedding, int k){
vector<int> indexes;
vector<vector<double>> query_double;
for (const auto& vec : _embedding) {
vector<double> temp;
temp.reserve(vec.size());
for (float f : vec) {
temp.push_back(static_cast<double>(f));
}
query_double.push_back(std::move(temp));
}
for (const auto& q : query_double) {
auto [distances, indices] = t->getNearestNeighbors(q, k);
std::cout << "Query: ";
for (double d : q) std::cout << d << " ";
std::cout << "\nNearest neighbors:\n";
for (int i = 0; i < k; ++i) {
indexes.push_back(indices[i]);
}
}
return indexes;
} }

8
src/vp/VP.h
View File

@ -4,11 +4,17 @@
#include <random> #include <random>
#include <chrono> #include <chrono>
#include "VpTree.hpp" #include "VpTree.hpp"
#include "ofMain.h"
class VP{ class VP{
public: public:
std::vector<std::vector<double>> generateRandomVectors(int count, int dimension); std::vector<std::vector<double>> generateRandomVectors(int count, int dimension);
vpt::VpTree loadTree(std::string path); vpt::VpTree loadTree(std::string path);
void saveTree(std::string filename, vpt::VpTree& tree); void saveTree(std::string filename, vpt::VpTree& tree);
vpt::VpTree buildTree(std::vector<vpt::Vector> vectors); void buildTree(std::vector<vpt::Vector> vectors);
void setVPTree(vpt::VpTree _tree);
vector<int> query(vector<vector<float>> _embedding, int k);
std::unique_ptr<vpt::VpTree> t;
}; };

Write Preview
Loading…
Cancel
Save