diff --git a/bin/data/new_tree.bin b/bin/data/new_tree.bin index 09f56bd..80f2c40 100644 Binary files a/bin/data/new_tree.bin and b/bin/data/new_tree.bin differ diff --git a/bin/data/shaders/espDepth.frag b/bin/data/shaders/espDepth.frag index af56f16..af61ce6 100644 --- a/bin/data/shaders/espDepth.frag +++ b/bin/data/shaders/espDepth.frag @@ -22,11 +22,11 @@ void main() { // Choose base color based on quadrant if (quadrant == 0) { - baseColor = vec3(0.0, 1.0, 0.0); // Red for bottom-left + baseColor = vec3(1.0, 1.0, 1.0); // Red for bottom-left } else if (quadrant == 1) { - baseColor = vec3(0.0, 0.0, 1.0); // Green for bottom-right + baseColor = vec3(1.0, 1.0, 1.0); // Green for bottom-right } else if (quadrant == 2) { - baseColor = vec3(1.0, 0.0, 0.0); // Blue for top-left + baseColor = vec3(1.0, 1.0, 1.0); // Blue for top-left } else { baseColor = vec3(1.0, 1.0, 1.0); // White for top-right } diff --git a/bin/data/shaders/map_depth.frag b/bin/data/shaders/map_depth.frag new file mode 100644 index 0000000..c4c5eb9 --- /dev/null +++ b/bin/data/shaders/map_depth.frag @@ -0,0 +1,16 @@ +#version 150 + +uniform sampler2DRect tex0; + +in vec2 varyingtexcoord; + +out vec4 outputColor; + +void main() +{ + float depth = texture(tex0, varyingtexcoord).r; + float remappedDepth = mix(0.0, 1.0, depth); // Maps 0-1 to 0.5-1 + //float adjustedDepth = pow(depth, 0.5); + + outputColor = vec4(1.0 - depth, 1.0 - depth, 1.0 - depth, remappedDepth); +} diff --git a/bin/data/shaders/map_depth.vert b/bin/data/shaders/map_depth.vert new file mode 100644 index 0000000..6cd1b21 --- /dev/null +++ b/bin/data/shaders/map_depth.vert @@ -0,0 +1,16 @@ +#version 150 + +uniform mat4 modelViewMatrix; +uniform mat4 projectionMatrix; +uniform mat4 textureMatrix; +uniform mat4 modelViewProjectionMatrix; + +in vec4 position; +in vec2 texcoord; + +out vec2 varyingtexcoord; +void main() +{ + varyingtexcoord = texcoord; + gl_Position = modelViewProjectionMatrix * position; +} \ No newline at end of file diff --git a/bin/data/shaders/p_depth.frag b/bin/data/shaders/p_depth.frag index e85dcc4..a5a824f 100644 --- a/bin/data/shaders/p_depth.frag +++ b/bin/data/shaders/p_depth.frag @@ -14,6 +14,9 @@ void main() if (texColor.a < 0.1) { discard; } + + vec3 color = texColor.rgb; + color = vec3(dot(color, vec3(0.299, 0.587, 0.114))); - outputColor = texColor; + outputColor = vec4(color, texColor.a); } diff --git a/bin/image-to-mesh b/bin/image-to-mesh index 5655b12..5497374 100755 Binary files a/bin/image-to-mesh and b/bin/image-to-mesh differ diff --git a/src/Bullet.cpp b/src/Bullet.cpp index f1dee82..933fa73 100644 --- a/src/Bullet.cpp +++ b/src/Bullet.cpp @@ -11,11 +11,11 @@ void Bullet::setup(vector& _nodes){ camera.setPosition(camera_start_pos); camera.lookAt(ofVec3f(0, 0, 0), ofVec3f(0, -1, 0)); camera.enableOrtho(); - camera.setScale(0.01); + camera.setScale(0.05); camera.setNearClip(-10000); camera.setFarClip(10000); - camera.disableMouseInput(); - camera.removeAllInteractions(); + // camera.disableMouseInput(); + // camera.removeAllInteractions(); world.setup(); world.setCamera(&camera); @@ -35,97 +35,97 @@ void Bullet::update(bool& is_controller_active, Node& chosen_node) { cameraBounds = getCameraBounds(camera); world.update(); - static bool was_controller_active = false; - static float transition_timer = 0.0f; - static glm::vec3 start_pos; + // static bool was_controller_active = false; + // static float transition_timer = 0.0f; + // static glm::vec3 start_pos; - glm::vec3 current_pos = camera.getPosition(); - float deltaTime = ofGetLastFrameTime(); - - // Detect transition from controller to inactive - if (is_controller_active != was_controller_active) { - if (!is_controller_active) { - // Starting transition to new node - transition_timer = 0.0f; - start_pos = current_pos; - } - } - - if (!is_controller_active && &chosen_node != current_target_node) { - current_target_node = &chosen_node; - } - - // Update camera physics - if (is_controller_active) { - // Decelerate over 2 seconds - float deceleration_rate = 0.5f; // 1/2 = 2 seconds - camera_velocity = camera_velocity * (1.0f - deceleration_rate * deltaTime); + // glm::vec3 current_pos = camera.getPosition(); + // float deltaTime = ofGetLastFrameTime(); + + // // Detect transition from controller to inactive + // if (is_controller_active != was_controller_active) { + // if (!is_controller_active) { + // // Starting transition to new node + // transition_timer = 0.0f; + // start_pos = current_pos; + // } + // } + + // if (!is_controller_active && &chosen_node != current_target_node) { + // current_target_node = &chosen_node; + // } + + // // Update camera physics + // if (is_controller_active) { + // // Decelerate over 2 seconds + // float deceleration_rate = 0.5f; // 1/2 = 2 seconds + // camera_velocity = camera_velocity * (1.0f - deceleration_rate * deltaTime); - if (glm::length(camera_velocity) < 0.01f) { - camera_velocity = glm::vec3(0.0f); - } + // if (glm::length(camera_velocity) < 0.01f) { + // camera_velocity = glm::vec3(0.0f); + // } - // Keep base zoom when controller is active - current_zoom = current_zoom + (0.03f - current_zoom) * deltaTime * 5.0f; - } else { - glm::vec3 target_position = current_target_node->collider->getPosition(); + // // Keep base zoom when controller is active + // current_zoom = current_zoom + (0.03f - current_zoom) * deltaTime * 5.0f; + // } else { + // glm::vec3 target_position = current_target_node->collider->getPosition(); - // Transition period (zooming out and in) - if (transition_timer < 10.0f) { // 2 second transition - transition_timer += deltaTime; - float progress = transition_timer / 2.0f; + // // Transition period (zooming out and in) + // if (transition_timer < 10.0f) { // 2 second transition + // transition_timer += deltaTime; + // float progress = transition_timer / 2.0f; - // Zoom curve (out and in) - float zoomProgress; - if (progress < 0.5f) { - // Zoom out during first half - zoomProgress = progress * 2.0f; - float targetZoom = glm::mix(0.03f, 0.6f, zoomProgress); - current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f; - } else { - // Zoom in during second half - zoomProgress = (1.0f - progress) * 2.0f; - float targetZoom = glm::mix(0.03f, 0.6f, zoomProgress); - current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f; - } + // // Zoom curve (out and in) + // float zoomProgress; + // if (progress < 0.5f) { + // // Zoom out during first half + // zoomProgress = progress * 2.0f; + // float targetZoom = glm::mix(0.03f, 0.6f, zoomProgress); + // current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f; + // } else { + // // Zoom in during second half + // zoomProgress = (1.0f - progress) * 2.0f; + // float targetZoom = glm::mix(0.03f, 0.6f, zoomProgress); + // current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f; + // } - // Move towards target during transition - glm::vec3 ideal_pos = glm::mix(start_pos, target_position, progress); - camera_velocity = (ideal_pos - current_pos) / deltaTime; - } else { - // After transition, follow node with spring physics - float k = 2.0f; - float damping = 3.0f; + // // Move towards target during transition + // glm::vec3 ideal_pos = glm::mix(start_pos, target_position, progress); + // camera_velocity = (ideal_pos - current_pos) / deltaTime; + // } else { + // // After transition, follow node with spring physics + // float k = 2.0f; + // float damping = 3.0f; - glm::vec3 displacement = target_position - current_pos; - glm::vec3 spring_force = k * displacement; + // glm::vec3 displacement = target_position - current_pos; + // glm::vec3 spring_force = k * displacement; - camera_velocity = camera_velocity + spring_force * deltaTime; - camera_velocity = camera_velocity * (1.0f - damping * deltaTime); + // camera_velocity = camera_velocity + spring_force * deltaTime; + // camera_velocity = camera_velocity * (1.0f - damping * deltaTime); - // Optional: Velocity-based zoom after transition - float speed = glm::length(camera_velocity); - float maxSpeed = 25.0f; - float speedFactor = glm::min(speed / maxSpeed, 1.0f); - float targetZoom = glm::mix(0.03f, 0.15f, speedFactor); // smaller zoom range for following - current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f; - } + // // Optional: Velocity-based zoom after transition + // float speed = glm::length(camera_velocity); + // float maxSpeed = 25.0f; + // float speedFactor = glm::min(speed / maxSpeed, 1.0f); + // float targetZoom = glm::mix(0.03f, 0.15f, speedFactor); // smaller zoom range for following + // current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f; + // } - // Clamp velocity to maximum speed - float maxSpeed = 25.0f; - float currentSpeed = glm::length(camera_velocity); - if (currentSpeed > maxSpeed) { - camera_velocity = (camera_velocity / currentSpeed) * maxSpeed; - } - } - - camera.setScale(current_zoom); + // // Clamp velocity to maximum speed + // float maxSpeed = 25.0f; + // float currentSpeed = glm::length(camera_velocity); + // if (currentSpeed > maxSpeed) { + // camera_velocity = (camera_velocity / currentSpeed) * maxSpeed; + // } + // } + + // camera.setScale(current_zoom); - // Update position using velocity - glm::vec3 new_position = current_pos + camera_velocity * deltaTime; - camera.setPosition(new_position); + // // Update position using velocity + // glm::vec3 new_position = current_pos + camera_velocity * deltaTime; + // camera.setPosition(new_position); - was_controller_active = is_controller_active; + // was_controller_active = is_controller_active; } void Bullet::draw(){ @@ -138,14 +138,14 @@ void Bullet::draw(){ glEnable( GL_DEPTH_TEST ); ofSetLineWidth(10.f); - ofDrawGrid(20, 100, true, false, false, true); + //ofDrawGrid(100, 100, false, false, false, true); ofNoFill(); ofSetColor(ofColor::yellow); ofNoFill(); - ofSetColor(ofColor::orange); + ofSetColor(ofColor::yellow); for(const auto& n : nodes){ if(checkNodeVisibility(n)){ ofPushMatrix(); @@ -168,7 +168,7 @@ void Bullet::draw(){ ofPushMatrix(); shader.begin(); shader.setUniform1f("gridSize", gridSize); - shader.setUniform2f("resolution", glm::vec2((1920.0 / 3) * 2, 960.0)); + shader.setUniform2f("resolution", glm::vec2(1920.0, 960.0)); shader.setUniformTexture("tex0", n.tex, 0); n.collider->transformGL(); ofScale(n.scale); @@ -205,7 +205,9 @@ void Bullet::addMesh(ofMesh _mesh, ofMesh _simple_mesh, Node& _node){ glm::vec3 random_szie(rand, rand, rand); _node.scale = random_szie; ofQuaternion startRot = ofQuaternion(0., 0., 0., PI); - ofVec3f start_location = ofVec3f( ofRandom(-3000, 3000), ofRandom(-3000, 3000) -5 ); + + + ofVec3f start_location = ofVec3f( ofRandom(_node.tsne_position.x - 50, _node.tsne_position.x + 50), ofRandom(_node.tsne_position.y - 50, _node.tsne_position.y + 50) -5 ); ofxBulletCustomShape* s = new ofxBulletCustomShape(); s->addMesh(_simple_mesh, _node.scale * 1.4, true); @@ -224,7 +226,6 @@ void Bullet::addMesh(ofMesh _mesh, ofMesh _simple_mesh, Node& _node){ _node.collider = s; _node.mesh = _mesh; _node.col_mesh = _simple_mesh; - // nodes.push_back(n); } void Bullet::workerThreadFunction(int threadId) { diff --git a/src/Bullet.h b/src/Bullet.h index 0e408a5..84e6033 100644 --- a/src/Bullet.h +++ b/src/Bullet.h @@ -15,6 +15,7 @@ struct Node { glm::vec3 scale; glm::vec3 tsne_position; vector raw_embedding; + int error; }; class Bullet{ diff --git a/src/network/Request.h b/src/network/Request.h index aab0033..f94c5e3 100644 --- a/src/network/Request.h +++ b/src/network/Request.h @@ -9,12 +9,9 @@ struct Embedding { Embedding() { // Initialize with default values - emotions["angry"] = 0.0f; - emotions["disgust"] = 0.0f; - emotions["fear"] = 0.0f; emotions["happy"] = 0.0f; emotions["sad"] = 0.0f; - emotions["surprise"] = 0.0f; + emotions["angry"] = 0.0f; emotions["neutral"] = 0.0f; } @@ -29,7 +26,7 @@ struct Embedding { // Define the order of emotions const std::vector emotion_order = { - "angry", "disgust", "fear", "happy", "sad", "surprise", "neutral" + "happy", "sad", "angry", "neutral" }; // Fill the vector in the defined order diff --git a/src/network/Server.cpp b/src/network/Server.cpp index a692700..d6b11cf 100644 --- a/src/network/Server.cpp +++ b/src/network/Server.cpp @@ -54,7 +54,7 @@ void Server::update(ofFbo& esp_comp){ addOrUpdateClient(id, value, ip_address, true); - std::cout << ip_address << " : " << id << std::endl; + //std::cout << ip_address << " : " << id << std::endl; } @@ -87,7 +87,7 @@ void Server::addOrUpdateClient(int client_id, float value, const std::string& ip void Server::updateEmbedding() { std::vector emotionNames = { - "happy", "sad", "angry", "neutral", "fear", "surprise", "disgust" + "happy", "sad", "angry", "neutral" }; for (const auto& c : clients) { @@ -115,7 +115,7 @@ void Server::printClients(){ void Server::print(){ const std::vector emotionNames = { - "happy", "sad", "angry", "neutral", "disgust", "fear", "surprise" + "happy", "sad", "angry", "neutral" }; std::ostringstream ss; diff --git a/src/ofApp.cpp b/src/ofApp.cpp index e50d1b0..d829ba3 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -4,14 +4,15 @@ void ofApp::setup(){ /* allocated fbo's */ - map_fbo.allocate((ofGetWindowWidth() / 3) * 2, map_h, GL_RGB); - map_fbo_alpha.allocate((ofGetWindowWidth() / 3) * 2, map_h, GL_RGBA); - portrait_fbo.allocate((ofGetWindowWidth() / 3), map_h, GL_RGBA); - portrait_pre_fbo.allocate((ofGetWindowWidth() / 3) * 1, map_h, GL_RGB); - portrait_pre_fbo_alpha.allocate((ofGetWindowWidth() / 3) * 1, map_h, GL_RGBA); + map_fbo.allocate(ofGetWindowWidth(), map_h, GL_RGB); + map_fbo_alpha.allocate(map_fbo.getWidth(), map_h, GL_RGBA); + map_fbo_post.allocate(map_fbo.getWidth(), map_h, GL_RGBA); + portrait_fbo.allocate(ofGetWindowWidth(), map_h, GL_RGBA); + portrait_pre_fbo.allocate(portrait_fbo.getWidth(), map_h, GL_RGB); + portrait_pre_fbo_alpha.allocate(portrait_fbo.getWidth(), map_h, GL_RGBA); comp_fbo.allocate(ofGetWindowWidth(), map_h, GL_RGBA); - model_outptut_fbo.allocate((ofGetWindowWidth() / 3) * 2, map_h, GL_RGB); + model_outptut_fbo.allocate(map_fbo.getWidth(), map_h, GL_RGB); model_esp_out_fbo.allocate(128 * 2, 168 * 2, GL_RGB); model_portrait_out_fbo.allocate(portrait_pre_fbo_alpha.getWidth(), portrait_pre_fbo_alpha.getHeight(), GL_RGB); @@ -47,6 +48,7 @@ void ofApp::setup(){ esp_shader.load("shaders/espDepth"); vert_snap.load("shaders/vertex_snap"); p_depth.load("shaders/p_depth"); + map_depth.load("shaders/map_depth"); ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx"; ORTCHAR_T* modelPath_Small = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx"; @@ -70,15 +72,17 @@ void ofApp::setup(){ depth_portrait.setup(&model_image_portrait, &model_portrait_out_fbo, &depth_onnx_portrait); /* camera settings for portrait */ - //portrait_camera.setFov(90); + glm::quat rot(0.93, 0.33, -0.14, 0.0565); + portrait_camera.enableOrtho(); portrait_camera.setNearClip(-10000); portrait_camera.setFarClip(10000); - portrait_camera.setPosition(0, 150, 480); - //portrait_camera.removeAllInteractions(); - //portrait_camera.disableMouseInput(); + portrait_camera.setPosition(-183.191, -341.348, 776.852); + portrait_camera.setOrientation(rot); + portrait_camera.setScale(0.5); + portrait_camera.removeAllInteractions(); + portrait_camera.disableMouseInput(); - createNodes("data/json/embeddings.json"); buildMeshes(); @@ -139,6 +143,7 @@ void ofApp::update(){ //mapper.update(); bullet.update(server->is_active, server->getChosenNode()); + //std::cout << portrait_camera.getPosition() << " : " <print(); + //server->print(); } void ofApp::drawPortrait(){ Node n = server->getChosenNode(); - float p_scale = 1 + ( ((1 + ofNoise(ofGetElapsedTimef() / 2)) / 2) * 2); + float p_scale = 1 + ( ((1 + ofNoise(ofGetElapsedTimef() / 5)) / 2) * 0.5); portrait_pre_fbo.begin(); ofClear(0, 0, 0, 0); @@ -305,6 +315,7 @@ void ofApp::drawPortraitZ(){ p_depth.setUniformTexture("tex0", tex_color, 0); p_depth.setUniformTexture("tex1", tex_depth, 1); ofFill(); + //ofRotateDeg(90, 0, 1, 0); ofTranslate(p_noise_x, p_noise_y, 0); plane.draw(); p_depth.end(); @@ -384,11 +395,13 @@ void ofApp::createNodes(std::string json_path){ Node n; n.img.load(j["image"]); n.tex = n.img.getTexture(); + n.error = j["lost"]; std::vector t_embedding; - + int count = 0; for (const auto& value: j["vector"]){ - if(value.is_number()){ + if(value.is_number() && count < 4){ t_embedding.push_back(value.get()); + count++; } else { ofLogError() << "Vector value is not a number"; } @@ -420,30 +433,6 @@ void ofApp::createNodes(std::string json_path){ auto& n = nodes[i]; n.tsne_position = (glm::vec3(((vec[0] * 2) - 1) * tsne_scale, ((vec[1] * 2) - 1) * tsne_scale, -5.0f)); } - /* vp-test */ - // auto queries = server->generateRandomVectors(10, 7); - - // int k = 5; - - // vector> test_vectors; - - // // Set up random number generator - // std::random_device rd; - // std::mt19937 gen(rd()); - // std::uniform_real_distribution<> dis(0.0, 1.0); // Random floats between -1 and 1 - - // for (int i = 0; i < 1; ++i) { - // std::vector vec; - // for (int j = 0; j < 7; ++j) { - // vec.push_back(static_cast(dis(gen))); - // } - // test_vectors.push_back(vec); - // } - - // std::cout << test_vectors.size() << std::endl; - - // vector q_index = vp_tree.query(test_vectors, 5); - } std::vector> ofApp::createDoubleVectorFromNodes(const std::vector& nodes) { @@ -524,19 +513,19 @@ void ofApp::queryKD(glm::vec3& _position, int k){ void ofApp::keyPressed(int key) { switch(key) { case OF_KEY_UP: - g_size += 10; + g_size += 5; ofLog() << "Grid Size: " << g_size; break; case OF_KEY_DOWN: - g_size -= 10; + g_size -= 5; ofLog() << "Grid Size: " << g_size; break; case 'q': - plane_size += 10; + plane_size += 5; ofLog() << "Plane Size: " << plane_size; break; case 'w': - plane_size -=10; + plane_size -=5; ofLog() << "Plane Size: " << plane_size; break; } diff --git a/src/ofApp.h b/src/ofApp.h index eeddfb8..b1dc201 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -45,6 +45,7 @@ class ofApp : public ofBaseApp{ ofFbo map_fbo; ofFbo map_fbo_alpha; + ofFbo map_fbo_post; ofFbo portrait_fbo; ofFbo portrait_pre_fbo; ofFbo portrait_pre_fbo_alpha; @@ -62,6 +63,7 @@ class ofApp : public ofBaseApp{ ofShader esp_shader; ofShader vert_snap; ofShader p_depth; + ofShader map_depth; ofPlanePrimitive plane; ofTexture bayer; @@ -99,14 +101,14 @@ class ofApp : public ofBaseApp{ float theta = 0.2; bool normalise = true; bool runManually = false; - float tsne_scale = 1000; + float tsne_scale = 500; /* kd tree */ std::unique_ptr kd_tree; std::vector kd_result; - float g_size = 10; - float plane_size = 100; + float g_size = 5; + float plane_size = 110; /* pi mapper */ ofxPiMapper mapper;