diff --git a/bin/data/new_tree.bin b/bin/data/new_tree.bin index 5be6d48..46e3e78 100644 Binary files a/bin/data/new_tree.bin and b/bin/data/new_tree.bin differ diff --git a/bin/data/shaders/p_depth.frag b/bin/data/shaders/p_depth.frag index a5a824f..e84bf8f 100644 --- a/bin/data/shaders/p_depth.frag +++ b/bin/data/shaders/p_depth.frag @@ -8,15 +8,26 @@ out vec4 outputColor; void main() { - vec4 texColor = texture(tex0, tex_c); - - // Discard fragments with alpha below a threshold + // Convert normalized coordinates to pixel coordinates + vec2 texCoord = tex_c * vec2(1920.0/2.0, 960.0); + vec4 texColor = texture(tex0, texCoord); + if (texColor.a < 0.1) { discard; } - vec3 color = texColor.rgb; color = vec3(dot(color, vec3(0.299, 0.587, 0.114))); - outputColor = vec4(color, texColor.a); + // vec4 texColor = texture(tex0, tex_c); + + + //Discard fragments with alpha below a threshold + // if (texColor.a < 0.1) { + // discard; + // } + + // vec3 color = texColor.rgb; + // color = vec3(dot(color, vec3(0.299, 0.587, 0.114))); + + // outputColor = vec4(color, texColor.a); } diff --git a/bin/data/shaders/vertex_snap.frag b/bin/data/shaders/vertex_snap.frag index 364f509..ede717d 100644 --- a/bin/data/shaders/vertex_snap.frag +++ b/bin/data/shaders/vertex_snap.frag @@ -6,11 +6,6 @@ in vec2 tex_c; out vec4 outputColor; -float hash13(vec3 p3) { - p3 = fract(p3 * .1031); - p3 += dot(p3, p3.yzx + 19.19); - return fract((p3.x + p3.y) * p3.z); -} void main() { diff --git a/bin/image-to-mesh b/bin/image-to-mesh index 47e734e..8652f34 100755 Binary files a/bin/image-to-mesh and b/bin/image-to-mesh differ diff --git a/src/Bullet.cpp b/src/Bullet.cpp index 57b50f5..3f4eaa5 100644 --- a/src/Bullet.cpp +++ b/src/Bullet.cpp @@ -29,61 +29,79 @@ void Bullet::setup(vector& _nodes){ } void Bullet::update(bool& is_controller_active, Node& chosen_node) { - updateRepulsionNode(); + //updateRepulsionNode(); cameraBounds = getCameraBounds(camera); world.update(); - updateRepulsionNode(); - cameraBounds = getCameraBounds(camera); - world.update(); float d_time = ofGetLastFrameTime(); glm::vec3 target_pos = chosen_node.collider->getPosition(); glm::vec3 current_pos = camera.getPosition(); glm::vec3 dir = current_pos - target_pos; float dist = glm::length(dir); - glm::vec3 norm_dir = glm::normalize(dir); - - if (is_controller_active) { - if (glm::length(camera_velocity) > 0.0f) { - glm::vec3 decel_dir = -glm::normalize(camera_velocity); - camera_velocity += decel_dir * DECELERATION * d_time; - if (glm::length(camera_velocity) < 0.01f) { - camera_velocity = glm::vec3(0.0f); - } - } - } else { - float distance_factor = glm::min(dist / 50.0f, 1.0f); - float current_acceleration = ACCELERATION * distance_factor; - - camera_velocity += -norm_dir * current_acceleration * d_time; - - if (dist < 5.0f) { - float decel_factor = dist / 5.0f; - camera_velocity *= (1.0f - (1.0f - decel_factor) * d_time * 10.0f); - } - - float mag = glm::length(camera_velocity); - if (mag > MAX_VELOCITY) { - camera_velocity = glm::normalize(camera_velocity) * MAX_VELOCITY; - } - // Smoothed zoom based on velocity - float vel_magnitude = glm::length(camera_velocity); - static float current_zoom = 0.02f; // Store current zoom - float target_zoom = ofMap(vel_magnitude, 0, MAX_VELOCITY, 0.02, 0.3, true); - - // Smooth lerp to target zoom - current_zoom = current_zoom + (target_zoom - current_zoom) * 0.1f; - camera.setScale(current_zoom); - } - camera.setPosition(current_pos + camera_velocity * d_time); + // Update camera only if conditions are met + if(!std::isnan(dist) && dist >= 0.0001f) { + glm::vec3 norm_dir = glm::normalize(dir); + + if(!std::isnan(norm_dir.x) && !std::isnan(norm_dir.y) && !std::isnan(norm_dir.z)) { + if (is_controller_active) { + if (glm::length(camera_velocity) > 0.0f) { + glm::vec3 decel_dir = -glm::normalize(camera_velocity); + camera_velocity += decel_dir * DECELERATION * d_time; + if (glm::length(camera_velocity) < 0.01f) { + camera_velocity = glm::vec3(0.0f); + } + } + } else { + float distance_factor = glm::min(dist / 50.0f, 1.0f); + float current_acceleration = ACCELERATION * distance_factor; + + + glm::vec3 acceleration = -norm_dir * current_acceleration * d_time; + // Check for NaN before applying + if(!std::isnan(acceleration.x) && !std::isnan(acceleration.y) && !std::isnan(acceleration.z)) { + camera_velocity += acceleration; + } + + + if (dist < 5.0f) { + float decel_factor = dist / 5.0f; + camera_velocity *= (1.0f - (1.0f - decel_factor) * d_time * 10.0f); + } + + float mag = glm::length(camera_velocity); + if (mag > MAX_VELOCITY) { + camera_velocity = glm::normalize(camera_velocity) * MAX_VELOCITY; + } + + // Smoothed zoom based on velocity + float vel_magnitude = glm::length(camera_velocity); + static float current_zoom = 0.02f; // Store current zoom + float target_zoom = ofMap(vel_magnitude, 0, MAX_VELOCITY, 0.02, 0.3, true); + + // Smooth lerp to target zoom + current_zoom = current_zoom + (target_zoom - current_zoom) * 0.1f; + camera.setScale(current_zoom); + } - if (glm::length(current_pos - target_pos) < 0.01f) { - camera.setPosition(target_pos); - camera_velocity = glm::vec3(0.0f); + // Final position update with safety check + glm::vec3 new_pos = current_pos + camera_velocity * d_time; + if(!std::isnan(new_pos.x) && !std::isnan(new_pos.y) && !std::isnan(new_pos.z)) { + camera.setPosition(new_pos); + } + } + } + + // Always update these regardless of camera movement + for(auto& node : nodes) { + updateNodeActivation(node); } + // if (glm::length(current_pos - target_pos) < 0.01f) { + // camera.setPosition(target_pos); + // camera_velocity = glm::vec3(0.0f); + // } } void Bullet::draw(){ @@ -96,7 +114,6 @@ void Bullet::draw(){ glEnable( GL_DEPTH_TEST ); ofSetLineWidth(10.f); - //ofDrawGrid(100, 100, false, false, false, true); ofNoFill(); ofSetColor(ofColor::yellow); @@ -126,7 +143,7 @@ void Bullet::draw(){ ofPushMatrix(); shader.begin(); shader.setUniform1f("gridSize", gridSize); - shader.setUniform2f("resolution", glm::vec2(1920.0, 960.0)); + shader.setUniform2f("resolution", glm::vec2(1920.0/2, 960.0)); shader.setUniformTexture("tex0", n.tex, 0); n.collider->transformGL(); ofScale(n.scale); @@ -165,16 +182,15 @@ void Bullet::addMesh(ofMesh _mesh, ofMesh _simple_mesh, Node& _node){ ofQuaternion startRot = ofQuaternion(0., 0., 0., PI); - ofVec3f start_location = ofVec3f( ofRandom(_node.tsne_position.x - 50, _node.tsne_position.x + 50), ofRandom(_node.tsne_position.y - 50, _node.tsne_position.y + 50) -5 ); + ofVec3f start_location = ofVec3f( ofRandom(_node.tsne_position.x - 50, _node.tsne_position.x + 50) + 20, ofRandom(_node.tsne_position.y - 50, _node.tsne_position.y + 50) + 20, -5 ); ofxBulletCustomShape* s = new ofxBulletCustomShape(); s->addMesh(_simple_mesh, _node.scale * 1.4, true); s->create( world.world, start_location, startRot, 3.); s->add(); s->getRigidBody()->setAngularFactor(btVector3(0, 0, 1)); - s->getRigidBody()->setDamping(btScalar(0.001), btScalar(0.05)); + s->getRigidBody()->setDamping(btScalar(0.0), btScalar(0.0)); s->getRigidBody()->setLinearFactor(btVector3(1, 1, 0)); - s->getRigidBody()->setActivationState(DISABLE_DEACTIVATION); s->getRigidBody()->setRestitution(btScalar(0.5)); s->getRigidBody()->setFriction(btScalar(1.0)); @@ -210,19 +226,7 @@ void Bullet::updateShapeBatch(size_t start, size_t end) { glm::vec3 norm_dir = glm::normalize(direction); nodes[i].collider->applyCentralForce(1.0f * norm_dir); - // Apply repulsion force if needed - // if (shouldApplyRepulsion && i != repulsionNodeIndex) { - // const Node& repulsionNode = nodes[repulsionNodeIndex]; - // glm::vec3 repulsionDir = pos - repulsionNode.collider->getPosition(); - // float repulsionDist = glm::length(repulsionDir); - - // if (repulsionDist < repulsionRadius && repulsionDist > 0) { - // repulsionDir = glm::normalize(repulsionDir); - // float forceMagnitude = repulsionStrength * (1.0f - repulsionDist / repulsionRadius); - // glm::vec3 repulsionForce = repulsionDir * forceMagnitude; - // nodes[i].collider->applyCentralForce(repulsionForce * 100.0f); - // } - // } + //updateNodeActivation(nodes[i]); } } @@ -251,7 +255,7 @@ glm::vec4 Bullet::getCameraBounds(const ofCamera& cam){ glm::vec3 cam_pos = cam.getPosition(); float cam_scale = cam.getScale().x; - float bounds = ofMap(cam_scale, 0.01, 0.2, 10, 240) + 20; + float bounds = ofMap(cam_scale, 0.01, 0.3, 10, 250); return glm::vec4(cam_pos.x + bounds, cam_pos.x - bounds, cam_pos.y + bounds, cam_pos.y - bounds); } @@ -271,4 +275,34 @@ glm::vec3 Bullet::getCameraPosition(){ void Bullet::setNodes(vector& _nodes){ nodes = _nodes; +} + +/* check if a node is inside a given radius, deactivate & stall if not. if it drifts too far reset position*/ +void Bullet::updateNodeActivation(Node& node) { + glm::vec3 node_pos = node.collider->getPosition(); + bool should_be_active = (node_pos.x < cameraBounds.x + 20 && + node_pos.x > cameraBounds.y - 20 && + node_pos.y < cameraBounds.z + 20 && + node_pos.y > cameraBounds.w - 20); + + btRigidBody* body = node.collider->getRigidBody(); + if (should_be_active && !body->isActive()) { + body->activate(true); + body->setActivationState(ACTIVE_TAG); + } else if (!should_be_active && body->isActive()) { + body->setActivationState(ISLAND_SLEEPING); + body->setLinearVelocity(btVector3(0,0,0)); + } + + // Force position check periodically + if(!should_be_active) { + // If node has drifted too far from target, reset it + glm::vec3 target_pos(node.tsne_position.x, node.tsne_position.y, -5); + float drift_distance = glm::length(node_pos - target_pos); + if(drift_distance > 400.0f) { // Adjust threshold as needed + btTransform transform = body->getWorldTransform(); + transform.setOrigin(btVector3(target_pos.x, target_pos.y, target_pos.z)); + body->setWorldTransform(transform); + } + } } \ No newline at end of file diff --git a/src/Bullet.h b/src/Bullet.h index fd62b7d..2fb2ac3 100644 --- a/src/Bullet.h +++ b/src/Bullet.h @@ -37,6 +37,7 @@ class Bullet{ bool checkNodeVisibility(const Node& n); glm::vec3 getCameraPosition(); void setNodes(vector& _nodes); + void updateNodeActivation(Node& node); ofxBulletWorldRigid world; vector bounds; ofxBulletCustomShape* boundsShape; diff --git a/src/ofApp.cpp b/src/ofApp.cpp index 19b06a1..b3e1633 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -51,7 +51,7 @@ void ofApp::setup(){ map_depth.load("shaders/map_depth"); ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx"; - ORTCHAR_T* modelPath_Small = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx"; + ORTCHAR_T* modelPath_Small = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vitb.onnx"; /* bullet setup */ bullet.setup(nodes); @@ -63,7 +63,7 @@ void ofApp::setup(){ /* onnx setup */ depth_onnx.Setup(modelPath, false, true); - depth_onnx_esp.Setup(modelPath_Small, false, true); + depth_onnx_esp.Setup(modelPath, false, true); depth_onnx_portrait.Setup(modelPath_Small, false, true); /* multi-thread setup */ @@ -115,9 +115,23 @@ void ofApp::update(){ server->update(esp_comp_fbo); float current_time = ofGetElapsedTimef(); + + Node n = server->getChosenNode(); + + // Check if node has changed + if(n.tsne_position != last_chosen_node.tsne_position) { + portrait_needs_update = true; + last_chosen_node = n; + } else { + portrait_needs_update = false; + } + + if(current_time - last_updated_time >= 3){ buildKDTree(); getNearestImages(); + depth_esp.update(); + depth_onnx_esp.SetPixels(model_esp_out_fbo); last_updated_time = current_time; } @@ -141,13 +155,11 @@ void ofApp::update(){ try{ depth_thread.update(); - depth_esp.update(); - depth_portrait.update(); + /* set output to fbo's */ depth_onnx.SetPixels(model_outptut_fbo); - depth_onnx_esp.SetPixels(model_esp_out_fbo); - depth_onnx_portrait.SetPixels(model_portrait_out_fbo); + } catch (exception e){ std::cout << "Model did not run" << std::endl; @@ -156,7 +168,7 @@ void ofApp::update(){ //mapper.update(); bullet.update(server->is_active, server->getChosenNode()); - std::cout << portrait_camera.getPosition() << " : " <getChosenNode(); - float p_scale = 1 + ( ((1 + ofNoise(ofGetElapsedTimef() / 5)) / 2) * 0.5); portrait_pre_fbo.begin(); ofClear(0, 0, 0, 0); float scale = min( - (float)portrait_pre_fbo.getWidth() / n.img.getWidth(), - (float)portrait_pre_fbo.getHeight() / n.img.getHeight() + (float)portrait_pre_fbo.getWidth() / last_chosen_node.img.getWidth(), + (float)portrait_pre_fbo.getHeight() / last_chosen_node.img.getHeight() ); - float scaledWidth = n.img.getWidth() * scale; - float scaledHeight = n.img.getHeight() * scale; + float scaledWidth = last_chosen_node.img.getWidth() * scale; + float scaledHeight = last_chosen_node.img.getHeight() * scale; float xPos = (portrait_pre_fbo.getWidth() - scaledWidth) / 2; float yPos = (portrait_pre_fbo.getHeight() - scaledHeight) / 2; ofPushMatrix(); // Move to center of FBO - ofTranslate(portrait_pre_fbo.getWidth()/2, portrait_pre_fbo.getHeight() * 0.6); // 0.8 + ofTranslate(portrait_pre_fbo.getWidth()/2, portrait_pre_fbo.getHeight()); // 0.8 // Apply scale ofScale(1); // Move back by half the scaled image dimensions - ofTranslate(-scaledWidth/2, -scaledHeight/2); + ofTranslate(-scaledWidth/2, -scaledHeight); // Draw at 0,0 since we've already translated - n.img.draw(0, 0, scaledWidth, scaledHeight); + last_chosen_node.img.draw(0, 0, scaledWidth, scaledHeight); ofPopMatrix(); portrait_pre_fbo.end(); @@ -267,19 +274,37 @@ void ofApp::drawPortrait(){ portrait_pre_fbo_alpha.begin(); ofPushMatrix(); ofClear(0, 0, 0, 0); - ofTranslate(portrait_pre_fbo.getWidth()/2, portrait_pre_fbo.getHeight() * 0.6); + ofTranslate(portrait_pre_fbo.getWidth()/2, portrait_pre_fbo.getHeight()); // Apply scale - ofScale(p_scale); + ofScale(1); // Move back by half the scaled image dimensions - ofTranslate(-scaledWidth/2, -scaledHeight/2); - n.img.draw(0, 0, scaledWidth, scaledHeight); + ofTranslate(-scaledWidth/2, -scaledHeight); + last_chosen_node.img.draw(0, 0, scaledWidth, scaledHeight); ofPopMatrix(); portrait_pre_fbo_alpha.end(); + if(portrait_needs_update) { + ofPixels pix; + portrait_pre_fbo.readToPixels(pix); + model_image_portrait.setFromPixels(pix); - ofPixels pix; - portrait_pre_fbo.readToPixels(pix); - model_image_portrait.setFromPixels(pix); + // Only run portrait model when needed + depth_portrait.update(); + depth_onnx_portrait.SetPixels(model_portrait_out_fbo); + + /* create mesh */ + float planeScale = 1.0; + int planeWidth = portrait_pre_fbo_alpha.getWidth() * planeScale; + int planeHeight = portrait_pre_fbo_alpha.getHeight() * planeScale; + int planeGridSize = plane_size; + int planeColumns = planeWidth / planeGridSize; + int planeRows = planeHeight / planeGridSize; + + plane.set(planeWidth, planeHeight, planeColumns, planeRows, OF_PRIMITIVE_TRIANGLES); + plane.mapTexCoords(0, 0, planeWidth, planeHeight); + custom_mesh = createCustomPlane(planeWidth, planeHeight, planeColumns, planeRows); + // ofLog() << "new mesh"; + } } void ofApp::drawPortraitZ(){ @@ -304,11 +329,6 @@ void ofApp::drawPortraitZ(){ } } - /* map bullet cam scale to portrait z-dist */ - float map_camera_scale = bullet.camera.getScale().x; - float pz = ofMap(map_camera_scale, 0.01, 0.03, 300, 550); - //portrait_camera.setPosition(glm::vec3(portrait_camera.getPosition().x, portrait_camera.getPosition().y, pz)); - portrait_camera.begin(); ofEnableDepthTest(); @@ -317,16 +337,6 @@ void ofApp::drawPortraitZ(){ float p_noise_y = ofSignedNoise(time + 100, time + 100) * 20; ofPushMatrix(); - - float planeScale = 1.0; - int planeWidth = portrait_pre_fbo_alpha.getWidth() * planeScale; - int planeHeight = portrait_pre_fbo_alpha.getHeight() * planeScale; - int planeGridSize = plane_size; - int planeColumns = planeWidth / planeGridSize; - int planeRows = planeHeight / planeGridSize; - - plane.set(planeWidth, planeHeight, planeColumns, planeRows, OF_PRIMITIVE_TRIANGLES); - plane.mapTexCoords(0, 0, planeWidth, planeHeight); p_depth.begin(); p_depth.setUniform1f("gridSize", g_size); @@ -335,9 +345,8 @@ void ofApp::drawPortraitZ(){ p_depth.setUniformTexture("tex0", tex_color, 0); p_depth.setUniformTexture("tex1", tex_depth, 1); ofFill(); - //ofRotateDeg(90, 0, 1, 0); ofTranslate(p_noise_x, p_noise_y, 0); - plane.draw(); + custom_mesh.draw(); p_depth.end(); ofPopMatrix(); ofDisableDepthTest(); @@ -576,4 +585,59 @@ void ofApp::mousePressed(int x, int y, int button){ //-------------------------------------------------------------- void ofApp::mouseReleased(int x, int y, int button){ //mapper.mouseReleased(x, y, button); +} + +ofMesh ofApp::createCustomPlane(float width, float height, int numX, int numY) { + ofMesh mesh; + mesh.setMode(OF_PRIMITIVE_TRIANGLES); + + // Create vertices with non-uniform distribution + for(int y = 0; y <= numY; y++) { + for(int x = 0; x <= numX; x++) { + // Get base uniform position + float xPos = (float)x/numX * width; + float yPos = (float)y/numY * height; + + float texX = (float)x/numX; + float texY = 1.0 - (float)y/numY; + + // Add random offset but keep edges fixed + if(x != 0 && x != numX && y != 0 && y != numY) { + // Random offset proportional to cell size + float cellWidth = width/numX; + float cellHeight = height/numY; + + xPos += ofRandom(-cellWidth * 0.4, cellWidth * 0.4); + yPos += ofRandom(-cellHeight * 0.4, cellHeight * 0.4); + } + + mesh.addVertex(glm::vec3(xPos, yPos, 0)); + + // Add texture coordinates (normalized) + mesh.addTexCoord(glm::vec2(texX, texY)); + mesh.addColor(ofColor(255, 255, 255, 255)); + } + } + + // Create triangles + for(int y = 0; y < numY; y++) { + for(int x = 0; x < numX; x++) { + // Get indices for corners of quad + int nw = y * (numX + 1) + x; + int ne = nw + 1; + int sw = nw + (numX + 1); + int se = sw + 1; + + // Add triangles + mesh.addIndex(nw); + mesh.addIndex(ne); + mesh.addIndex(se); + + mesh.addIndex(nw); + mesh.addIndex(se); + mesh.addIndex(sw); + } + } + + return mesh; } \ No newline at end of file diff --git a/src/ofApp.h b/src/ofApp.h index 3a0be26..ae6b18d 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -29,6 +29,7 @@ class ofApp : public ofBaseApp{ void buildMeshes(); void drawPortrait(); void drawPortraitZ(); + ofMesh createCustomPlane(float width, float height, int numX, int numY); void keyPressed(int key); void keyReleased(int key); void mousePressed(int x, int y, int button); @@ -120,4 +121,8 @@ class ofApp : public ofBaseApp{ /* camera positions */ vector cam_positions; float cam_pos_idx = 0; + + Node last_chosen_node; + bool portrait_needs_update; + ofMesh custom_mesh; };