diff --git a/.gitignore b/.gitignore index a71ad33..c74a7df 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,11 @@ /bin/data/main_dataset /bin/libs/ +######### +# ignore dlls from copy +######### +/bin/*.dll + ######### # general ######### diff --git a/addons.make b/addons.make index f3ae057..345a513 100644 --- a/addons.make +++ b/addons.make @@ -9,4 +9,5 @@ ofxNetwork ofxTSNE ofxTSNE ofxPiMapper -ofxPiMapper \ No newline at end of file +ofxPiMapper +ofxOnnxRuntime \ No newline at end of file diff --git a/bin/data/new_tree.bin b/bin/data/new_tree.bin index 78184c9..521bc26 100644 Binary files a/bin/data/new_tree.bin and b/bin/data/new_tree.bin differ diff --git a/bin/sv-map.exe b/bin/sv-map.exe new file mode 100644 index 0000000..7472110 Binary files /dev/null and b/bin/sv-map.exe differ diff --git a/image-to-mesh.code-workspace b/image-to-mesh.code-workspace index 15b21d5..98045ff 100644 --- a/image-to-mesh.code-workspace +++ b/image-to-mesh.code-workspace @@ -110,7 +110,26 @@ "timer": "cpp", "buffer": "cpp", "socket": "cpp", - "*.glsl": "cpp" + "*.glsl": "cpp", + "coroutine": "cpp", + "ios": "cpp", + "locale": "cpp", + "xfacet": "cpp", + "xhash": "cpp", + "xiosbase": "cpp", + "xlocale": "cpp", + "xlocbuf": "cpp", + "xlocinfo": "cpp", + "xlocmes": "cpp", + "xlocmon": "cpp", + "xlocnum": "cpp", + "xloctime": "cpp", + "xmemory": "cpp", + "xstddef": "cpp", + "xstring": "cpp", + "xtr1common": "cpp", + "xtree": "cpp", + "xutility": "cpp" } } } \ No newline at end of file diff --git a/src/Bullet.cpp b/src/Bullet.cpp index fdaf32e..458b6f1 100644 --- a/src/Bullet.cpp +++ b/src/Bullet.cpp @@ -185,7 +185,7 @@ void Bullet::draw(){ camera.begin(); glEnable( GL_DEPTH_TEST ); - ofSetLineWidth(20.f); + ofSetLineWidth(1.f); ofNoFill(); ofSetColor(ofColor::yellow); diff --git a/src/network/Request.cpp b/src/network/Request.cpp index 0985a62..c42c429 100644 --- a/src/network/Request.cpp +++ b/src/network/Request.cpp @@ -33,7 +33,7 @@ VPResp Request::query(Embedding& in){ past_vp_resp = vp_resp; return vp_resp; - } catch (exception e) { + } catch (_exception e) { // Some issue happening here when plugging in controllers, or when they initially connect return past_vp_resp; } diff --git a/src/network/Server.cpp b/src/network/Server.cpp index 4b39e98..4bc030e 100644 --- a/src/network/Server.cpp +++ b/src/network/Server.cpp @@ -207,7 +207,9 @@ void Server::sendOSCMessage(){ std::string audio_file = folder; // Check if file exists in a given dir - ofFile file("/home/cailean/Desktop/rave/all_wav_files/" + audio_file + ".wav"); + ofFile file("C:/Users/caile/Desktop/all_wav_files/all_wav_files/" + audio_file + ".wav"); + + ofLog() << file.path; if((audio_file != past_audio_file) && file.exists()){ me_file.setAddress("/emote/filename"); diff --git a/src/network/Server.h b/src/network/Server.h index 15fda28..3819256 100644 --- a/src/network/Server.h +++ b/src/network/Server.h @@ -75,7 +75,7 @@ class Server{ std::chrono::steady_clock::time_point lastUpdateTime; const std::chrono::milliseconds updateInterval{1000}; // 1 second interval - bool m_RanomdEmbed = true; + bool m_RanomdEmbed = false; int regions[4][4] = { {0, 0, 128, 168}, // Top-left diff --git a/src/ofApp.cpp b/src/ofApp.cpp index 791f6c9..324ed4e 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -68,18 +68,18 @@ void ofApp::setup(){ font.load("MaziusDisplay-Bold.otf", 180); ofLog() << font.getLineHeight(); - ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx"; - ORTCHAR_T* modelPath_Small = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vitb.onnx"; - ORTCHAR_T* modelPath_Yolo = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/yolov5s-face.onnx"; - ORTCHAR_T* modelPath_Indoor_Dynamic = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vitl.onnx"; + ORTCHAR_T* modelPath = L"C:/Users/caile/of_v20250417_msys2_mingw64_release/apps/myApps/sv-map/bin/data/models/depth_anything_v2_vits.onnx"; + ORTCHAR_T* modelPath_Small = L"C:/Users/caile/of_v20250417_msys2_mingw64_release/apps/myApps/sv-map/bin/data/models/depth_anything_v2_vitb.onnx"; + ORTCHAR_T* modelPath_Yolo = L"C:/Users/caile/of_v20250417_msys2_mingw64_release/apps/myApps/sv-map/bin/data/models/yolov5s-face.onnx"; + ORTCHAR_T* modelPath_Indoor_Dynamic = L"C:/Users/caile/of_v20250417_msys2_mingw64_release/apps/myApps/sv-map/bin/data/models/depth_anything_v2_vitl.onnx"; bullet.setup(); /* pi mapper setup */ - mapper.registerFboSource(projection_src); - mapper.setup(); - projection_src.setFbo(&comp_fbo); - projection_src.setFbo(&mapper_black); + // mapper.registerFboSource(projection_src); + // mapper.setup(); + // projection_src.setFbo(&comp_fbo); + // projection_src.setFbo(&mapper_black); /* onnx setup */ depth_onnx.Setup(modelPath, false, true); @@ -125,7 +125,7 @@ void ofApp::setup(){ portrait_camera.setScale(current_cp.scale); updateCurrentCameraMode(); - createNodes("data/json/sv_embeddings.json"); + createNodes("json/sv_embeddings.json"); buildMeshes(); @@ -203,6 +203,16 @@ void ofApp::update(){ mapper.update(); bullet.update(server->is_active, n, nn_nodes); + if(tsne_update_complete) { + tsne_update_complete = false; + onTSNEUpdateComplete(); + } + + if(ofGetElapsedTimef() > tsne_start_time + TNSE_DURATION) { + tsne_iter_idx = (tsne_iter_idx + 1) % 3; + updateTSNEPositions(nodes); + } + updateCurrentCameraMode(); if(bullet.getRandomWalkTime() > 60){ @@ -232,11 +242,12 @@ void ofApp::draw(){ esp_comp_fbo.begin(); - esp_shader.begin(); - esp_shader.setUniformTexture("tex0", model_esp_out_fbo.getTexture(), 0); - model_esp_out_fbo.draw(0, 0); + //esp_shader.begin(); + //esp_shader.setUniformTexture("tex0", model_esp_out_fbo.getTexture(), 0); + //model_esp_out_fbo.draw(0, 0); + model_image_esp.draw(0, 0); esp_comp_fbo.end(); - esp_shader.end(); + //esp_shader.end(); portrait_fbo.begin(); @@ -275,7 +286,7 @@ void ofApp::draw(){ ofTranslate(0, 0); comp_fbo.draw(0, 0); - mapper.draw(); + //mapper.draw(); if(bullet.print_debug) @@ -872,4 +883,41 @@ void ofApp::drawText(){ ofPopStyle(); text_display.end(); -} \ No newline at end of file +} + +void ofApp::updateTSNEPositions(vector& _nodes){ + if(tsne_updating) { + ofLog() << "TSNE update already in progress"; + return; + } + tsne_updating = true; + tsne_update_complete = false; // Reset flag + tsne_thread = std::thread(&ofApp::updateTSNEPositionsThreaded, this, std::ref(_nodes)); + tsne_thread.detach(); +} + +void ofApp::updateTSNEPositionsThreaded(vector& _nodes){ + ofLog() << "Starting TSNE update in thread"; + { + std::lock_guard lock(nodes_mutex); + for(size_t i = 0; i < point_iterations[tsne_iter_idx].size(); i++) { + const auto& vec = point_iterations[tsne_iter_idx][i]; + nodes[i].tsne_position = glm::vec3( + ((vec[0] * 2) - 1) * tsne_scale, + ((vec[1] * 2) - 1) * tsne_scale, + -5.0f + ); + } + } + tsne_updating = false; + tsne_update_complete = true; // Set completion flag + ofLog() << "TSNE update complete"; +} + +void ofApp::onTSNEUpdateComplete() { + // Do whatever you need to do after TSNE update + // This will run on the main thread + bullet.updateTSNEPosition(nodes); + tsne_start_time = ofGetElapsedTimef(); + ofLog() << "Handling TSNE update completion on main thread"; +} diff --git a/src/ofApp.h b/src/ofApp.h index 0f70ed0..4c616c0 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -4,7 +4,7 @@ #include "utils/MeshGenerator.h" #include "Bullet.h" #include "vp/VP.h" -#include +#include "ofxOnnxRuntime.h" #include "onnx/Onnx.h" #include "onnx/Yolo.h" #include "onnx/ModelThread.h" @@ -165,7 +165,7 @@ class ofApp : public ofBaseApp{ Bullet bullet; /* simulation update timer */ - const float TNSE_DURATION = 180.0f; + const float TNSE_DURATION = 30.0f; float tsne_start_time; int tsne_iter_idx = 0; diff --git a/src/onnx/Onnx.h b/src/onnx/Onnx.h index 306d41a..7b8cb8d 100644 --- a/src/onnx/Onnx.h +++ b/src/onnx/Onnx.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "ofxOnnxRuntime.h" #include // std::generate #include #include diff --git a/src/onnx/Yolo.h b/src/onnx/Yolo.h index 808cc50..dc93e14 100644 --- a/src/onnx/Yolo.h +++ b/src/onnx/Yolo.h @@ -3,7 +3,7 @@ #include "ofMain.h" #include "ofxOpenCv.h" -#include +#include "ofxOnnxRuntime.h" #include struct Emotef{ diff --git a/src/utils/QuadSource.cpp b/src/utils/QuadSource.cpp index 38d1eee..3131ab5 100644 --- a/src/utils/QuadSource.cpp +++ b/src/utils/QuadSource.cpp @@ -17,5 +17,5 @@ void QuadSource::draw(){ // Fill FBO with our quads ofClear(0); //ofBackground(255, 120, 10); - fbo_in->draw(0, 0); + // fbo_in->draw(0, 0); } \ No newline at end of file