cailean
/
sv-map
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

finished map, just need camera movement!

master
cailean 4 months ago
parent
d339634347
commit
230aff79a4
12 changed files with 180 additions and 155 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. BIN
      bin/data/new_tree.bin
  2. 6
      bin/data/shaders/espDepth.frag
  3. 16
      bin/data/shaders/map_depth.frag
  4. 16
      bin/data/shaders/map_depth.vert
  5. 5
      bin/data/shaders/p_depth.frag
  6. BIN
      bin/image-to-mesh
  7. 175
      src/Bullet.cpp
  8. 1
      src/Bullet.h
  9. 7
      src/network/Request.h
  10. 6
      src/network/Server.cpp
  11. 95
      src/ofApp.cpp
  12. 8
      src/ofApp.h

BIN
bin/data/new_tree.bin
View File

Binary file not shown.

6
bin/data/shaders/espDepth.frag
View File

@ -22,11 +22,11 @@ void main() {
// Choose base color based on quadrant
if (quadrant == 0) {
baseColor = vec3(0.0, 1.0, 0.0); // Red for bottom-left
baseColor = vec3(1.0, 1.0, 1.0); // Red for bottom-left
} else if (quadrant == 1) {
baseColor = vec3(0.0, 0.0, 1.0); // Green for bottom-right
baseColor = vec3(1.0, 1.0, 1.0); // Green for bottom-right
} else if (quadrant == 2) {
baseColor = vec3(1.0, 0.0, 0.0); // Blue for top-left
baseColor = vec3(1.0, 1.0, 1.0); // Blue for top-left
} else {
baseColor = vec3(1.0, 1.0, 1.0); // White for top-right
}

16
bin/data/shaders/map_depth.frag
View File

@ -0,0 +1,16 @@
#version 150
uniform sampler2DRect tex0;
in vec2 varyingtexcoord;
out vec4 outputColor;
void main()
{
float depth = texture(tex0, varyingtexcoord).r;
float remappedDepth = mix(0.0, 1.0, depth); // Maps 0-1 to 0.5-1
//float adjustedDepth = pow(depth, 0.5);
outputColor = vec4(1.0 - depth, 1.0 - depth, 1.0 - depth, remappedDepth);
}

16
bin/data/shaders/map_depth.vert
View File

@ -0,0 +1,16 @@
#version 150
uniform mat4 modelViewMatrix;
uniform mat4 projectionMatrix;
uniform mat4 textureMatrix;
uniform mat4 modelViewProjectionMatrix;
in vec4 position;
in vec2 texcoord;
out vec2 varyingtexcoord;
void main()
{
varyingtexcoord = texcoord;
gl_Position = modelViewProjectionMatrix * position;
}

5
bin/data/shaders/p_depth.frag
View File

@ -14,6 +14,9 @@ void main()
if (texColor.a < 0.1) {
discard;
}
vec3 color = texColor.rgb;
color = vec3(dot(color, vec3(0.299, 0.587, 0.114)));
outputColor = texColor;
outputColor = vec4(color, texColor.a);
}

BIN
bin/image-to-mesh
View File

Binary file not shown.

175
src/Bullet.cpp
View File

@ -11,11 +11,11 @@ void Bullet::setup(vector<Node>& _nodes){
camera.setPosition(camera_start_pos);
camera.lookAt(ofVec3f(0, 0, 0), ofVec3f(0, -1, 0));
camera.enableOrtho();
camera.setScale(0.01);
camera.setScale(0.05);
camera.setNearClip(-10000);
camera.setFarClip(10000);
camera.disableMouseInput();
camera.removeAllInteractions();
// camera.disableMouseInput();
// camera.removeAllInteractions();
world.setup();
world.setCamera(&camera);
@ -35,97 +35,97 @@ void Bullet::update(bool& is_controller_active, Node& chosen_node) {
cameraBounds = getCameraBounds(camera);
world.update();
static bool was_controller_active = false;
static float transition_timer = 0.0f;
static glm::vec3 start_pos;
// static bool was_controller_active = false;
// static float transition_timer = 0.0f;
// static glm::vec3 start_pos;
glm::vec3 current_pos = camera.getPosition();
float deltaTime = ofGetLastFrameTime();
// Detect transition from controller to inactive
if (is_controller_active != was_controller_active) {
if (!is_controller_active) {
// Starting transition to new node
transition_timer = 0.0f;
start_pos = current_pos;
}
}
if (!is_controller_active && &chosen_node != current_target_node) {
current_target_node = &chosen_node;
}
// Update camera physics
if (is_controller_active) {
// Decelerate over 2 seconds
float deceleration_rate = 0.5f; // 1/2 = 2 seconds
camera_velocity = camera_velocity * (1.0f - deceleration_rate * deltaTime);
// glm::vec3 current_pos = camera.getPosition();
// float deltaTime = ofGetLastFrameTime();
// // Detect transition from controller to inactive
// if (is_controller_active != was_controller_active) {
// if (!is_controller_active) {
// // Starting transition to new node
// transition_timer = 0.0f;
// start_pos = current_pos;
// }
// }
// if (!is_controller_active && &chosen_node != current_target_node) {
// current_target_node = &chosen_node;
// }
// // Update camera physics
// if (is_controller_active) {
// // Decelerate over 2 seconds
// float deceleration_rate = 0.5f; // 1/2 = 2 seconds
// camera_velocity = camera_velocity * (1.0f - deceleration_rate * deltaTime);
if (glm::length(camera_velocity) < 0.01f) {
camera_velocity = glm::vec3(0.0f);
}
// if (glm::length(camera_velocity) < 0.01f) {
// camera_velocity = glm::vec3(0.0f);
// }
// Keep base zoom when controller is active
current_zoom = current_zoom + (0.03f - current_zoom) * deltaTime * 5.0f;
} else {
glm::vec3 target_position = current_target_node->collider->getPosition();
// // Keep base zoom when controller is active
// current_zoom = current_zoom + (0.03f - current_zoom) * deltaTime * 5.0f;
// } else {
// glm::vec3 target_position = current_target_node->collider->getPosition();
// Transition period (zooming out and in)
if (transition_timer < 10.0f) { // 2 second transition
transition_timer += deltaTime;
float progress = transition_timer / 2.0f;
// // Transition period (zooming out and in)
// if (transition_timer < 10.0f) { // 2 second transition
// transition_timer += deltaTime;
// float progress = transition_timer / 2.0f;
// Zoom curve (out and in)
float zoomProgress;
if (progress < 0.5f) {
// Zoom out during first half
zoomProgress = progress * 2.0f;
float targetZoom = glm::mix(0.03f, 0.6f, zoomProgress);
current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f;
} else {
// Zoom in during second half
zoomProgress = (1.0f - progress) * 2.0f;
float targetZoom = glm::mix(0.03f, 0.6f, zoomProgress);
current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f;
}
// // Zoom curve (out and in)
// float zoomProgress;
// if (progress < 0.5f) {
// // Zoom out during first half
// zoomProgress = progress * 2.0f;
// float targetZoom = glm::mix(0.03f, 0.6f, zoomProgress);
// current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f;
// } else {
// // Zoom in during second half
// zoomProgress = (1.0f - progress) * 2.0f;
// float targetZoom = glm::mix(0.03f, 0.6f, zoomProgress);
// current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f;
// }
// Move towards target during transition
glm::vec3 ideal_pos = glm::mix(start_pos, target_position, progress);
camera_velocity = (ideal_pos - current_pos) / deltaTime;
} else {
// After transition, follow node with spring physics
float k = 2.0f;
float damping = 3.0f;
// // Move towards target during transition
// glm::vec3 ideal_pos = glm::mix(start_pos, target_position, progress);
// camera_velocity = (ideal_pos - current_pos) / deltaTime;
// } else {
// // After transition, follow node with spring physics
// float k = 2.0f;
// float damping = 3.0f;
glm::vec3 displacement = target_position - current_pos;
glm::vec3 spring_force = k * displacement;
// glm::vec3 displacement = target_position - current_pos;
// glm::vec3 spring_force = k * displacement;
camera_velocity = camera_velocity + spring_force * deltaTime;
camera_velocity = camera_velocity * (1.0f - damping * deltaTime);
// camera_velocity = camera_velocity + spring_force * deltaTime;
// camera_velocity = camera_velocity * (1.0f - damping * deltaTime);
// Optional: Velocity-based zoom after transition
float speed = glm::length(camera_velocity);
float maxSpeed = 25.0f;
float speedFactor = glm::min(speed / maxSpeed, 1.0f);
float targetZoom = glm::mix(0.03f, 0.15f, speedFactor); // smaller zoom range for following
current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f;
}
// // Optional: Velocity-based zoom after transition
// float speed = glm::length(camera_velocity);
// float maxSpeed = 25.0f;
// float speedFactor = glm::min(speed / maxSpeed, 1.0f);
// float targetZoom = glm::mix(0.03f, 0.15f, speedFactor); // smaller zoom range for following
// current_zoom = current_zoom + (targetZoom - current_zoom) * deltaTime * 5.0f;
// }
// Clamp velocity to maximum speed
float maxSpeed = 25.0f;
float currentSpeed = glm::length(camera_velocity);
if (currentSpeed > maxSpeed) {
camera_velocity = (camera_velocity / currentSpeed) * maxSpeed;
}
}
camera.setScale(current_zoom);
// // Clamp velocity to maximum speed
// float maxSpeed = 25.0f;
// float currentSpeed = glm::length(camera_velocity);
// if (currentSpeed > maxSpeed) {
// camera_velocity = (camera_velocity / currentSpeed) * maxSpeed;
// }
// }
// camera.setScale(current_zoom);
// Update position using velocity
glm::vec3 new_position = current_pos + camera_velocity * deltaTime;
camera.setPosition(new_position);
// // Update position using velocity
// glm::vec3 new_position = current_pos + camera_velocity * deltaTime;
// camera.setPosition(new_position);
was_controller_active = is_controller_active;
// was_controller_active = is_controller_active;
}
void Bullet::draw(){
@ -138,14 +138,14 @@ void Bullet::draw(){
glEnable( GL_DEPTH_TEST );
ofSetLineWidth(10.f);
ofDrawGrid(20, 100, true, false, false, true);
//ofDrawGrid(100, 100, false, false, false, true);
ofNoFill();
ofSetColor(ofColor::yellow);
ofNoFill();
ofSetColor(ofColor::orange);
ofSetColor(ofColor::yellow);
for(const auto& n : nodes){
if(checkNodeVisibility(n)){
ofPushMatrix();
@ -168,7 +168,7 @@ void Bullet::draw(){
ofPushMatrix();
shader.begin();
shader.setUniform1f("gridSize", gridSize);
shader.setUniform2f("resolution", glm::vec2((1920.0 / 3) * 2, 960.0));
shader.setUniform2f("resolution", glm::vec2(1920.0, 960.0));
shader.setUniformTexture("tex0", n.tex, 0);
n.collider->transformGL();
ofScale(n.scale);
@ -205,7 +205,9 @@ void Bullet::addMesh(ofMesh _mesh, ofMesh _simple_mesh, Node& _node){
glm::vec3 random_szie(rand, rand, rand);
_node.scale = random_szie;
ofQuaternion startRot = ofQuaternion(0., 0., 0., PI);
ofVec3f start_location = ofVec3f( ofRandom(-3000, 3000), ofRandom(-3000, 3000) -5 );
ofVec3f start_location = ofVec3f( ofRandom(_node.tsne_position.x - 50, _node.tsne_position.x + 50), ofRandom(_node.tsne_position.y - 50, _node.tsne_position.y + 50) -5 );
ofxBulletCustomShape* s = new ofxBulletCustomShape();
s->addMesh(_simple_mesh, _node.scale * 1.4, true);
@ -224,7 +226,6 @@ void Bullet::addMesh(ofMesh _mesh, ofMesh _simple_mesh, Node& _node){
_node.collider = s;
_node.mesh = _mesh;
_node.col_mesh = _simple_mesh;
// nodes.push_back(n);
}
void Bullet::workerThreadFunction(int threadId) {

1
src/Bullet.h
View File

@ -15,6 +15,7 @@ struct Node {
glm::vec3 scale;
glm::vec3 tsne_position;
vector<float> raw_embedding;
int error;
};
class Bullet{

7
src/network/Request.h
View File

@ -9,12 +9,9 @@ struct Embedding {
Embedding() {
// Initialize with default values
emotions["angry"] = 0.0f;
emotions["disgust"] = 0.0f;
emotions["fear"] = 0.0f;
emotions["happy"] = 0.0f;
emotions["sad"] = 0.0f;
emotions["surprise"] = 0.0f;
emotions["angry"] = 0.0f;
emotions["neutral"] = 0.0f;
}
@ -29,7 +26,7 @@ struct Embedding {
// Define the order of emotions
const std::vector<std::string> emotion_order = {
"angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"
"happy", "sad", "angry", "neutral"
};
// Fill the vector in the defined order

6
src/network/Server.cpp
View File

@ -54,7 +54,7 @@ void Server::update(ofFbo& esp_comp){
addOrUpdateClient(id, value, ip_address, true);
std::cout << ip_address << " : " << id << std::endl;
//std::cout << ip_address << " : " << id << std::endl;
}
@ -87,7 +87,7 @@ void Server::addOrUpdateClient(int client_id, float value, const std::string& ip
void Server::updateEmbedding() {
std::vector<std::string> emotionNames = {
"happy", "sad", "angry", "neutral", "fear", "surprise", "disgust"
"happy", "sad", "angry", "neutral"
};
for (const auto& c : clients) {
@ -115,7 +115,7 @@ void Server::printClients(){
void Server::print(){
const std::vector<std::string> emotionNames = {
"happy", "sad", "angry", "neutral", "disgust", "fear", "surprise"
"happy", "sad", "angry", "neutral"
};
std::ostringstream ss;

95
src/ofApp.cpp
View File

@ -4,14 +4,15 @@
void ofApp::setup(){
/* allocated fbo's */
map_fbo.allocate((ofGetWindowWidth() / 3) * 2, map_h, GL_RGB);
map_fbo_alpha.allocate((ofGetWindowWidth() / 3) * 2, map_h, GL_RGBA);
portrait_fbo.allocate((ofGetWindowWidth() / 3), map_h, GL_RGBA);
portrait_pre_fbo.allocate((ofGetWindowWidth() / 3) * 1, map_h, GL_RGB);
portrait_pre_fbo_alpha.allocate((ofGetWindowWidth() / 3) * 1, map_h, GL_RGBA);
map_fbo.allocate(ofGetWindowWidth(), map_h, GL_RGB);
map_fbo_alpha.allocate(map_fbo.getWidth(), map_h, GL_RGBA);
map_fbo_post.allocate(map_fbo.getWidth(), map_h, GL_RGBA);
portrait_fbo.allocate(ofGetWindowWidth(), map_h, GL_RGBA);
portrait_pre_fbo.allocate(portrait_fbo.getWidth(), map_h, GL_RGB);
portrait_pre_fbo_alpha.allocate(portrait_fbo.getWidth(), map_h, GL_RGBA);
comp_fbo.allocate(ofGetWindowWidth(), map_h, GL_RGBA);
model_outptut_fbo.allocate((ofGetWindowWidth() / 3) * 2, map_h, GL_RGB);
model_outptut_fbo.allocate(map_fbo.getWidth(), map_h, GL_RGB);
model_esp_out_fbo.allocate(128 * 2, 168 * 2, GL_RGB);
model_portrait_out_fbo.allocate(portrait_pre_fbo_alpha.getWidth(), portrait_pre_fbo_alpha.getHeight(), GL_RGB);
@ -47,6 +48,7 @@ void ofApp::setup(){
esp_shader.load("shaders/espDepth");
vert_snap.load("shaders/vertex_snap");
p_depth.load("shaders/p_depth");
map_depth.load("shaders/map_depth");
ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx";
ORTCHAR_T* modelPath_Small = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx";
@ -70,15 +72,17 @@ void ofApp::setup(){
depth_portrait.setup(&model_image_portrait, &model_portrait_out_fbo, &depth_onnx_portrait);
/* camera settings for portrait */
//portrait_camera.setFov(90);
glm::quat rot(0.93, 0.33, -0.14, 0.0565);
portrait_camera.enableOrtho();
portrait_camera.setNearClip(-10000);
portrait_camera.setFarClip(10000);
portrait_camera.setPosition(0, 150, 480);
//portrait_camera.removeAllInteractions();
//portrait_camera.disableMouseInput();
portrait_camera.setPosition(-183.191, -341.348, 776.852);
portrait_camera.setOrientation(rot);
portrait_camera.setScale(0.5);
portrait_camera.removeAllInteractions();
portrait_camera.disableMouseInput();
createNodes("data/json/embeddings.json");
buildMeshes();
@ -139,6 +143,7 @@ void ofApp::update(){
//mapper.update();
bullet.update(server->is_active, server->getChosenNode());
//std::cout << portrait_camera.getPosition() << " : " <<portrait_camera.getOrientationQuat()<< " : " <<portrait_camera.getScale() << std::endl;
}
//--------------------------------------------------------------
@ -146,7 +151,7 @@ void ofApp::draw(){
ofPushStyle();
map_fbo_alpha.begin();
ofClear(0, 0, 0, 0);
ofClear(0);
bullet.draw();
map_fbo_alpha.end();
ofPopStyle();
@ -159,9 +164,6 @@ void ofApp::draw(){
ofTexture t = map_fbo.getTexture();
ofTexture d = model_outptut_fbo.getTexture();
//model_outptut_fbo.draw(ofGetWindowWidth() / 2, 0);
esp_comp_fbo.begin();
esp_shader.begin();
@ -178,22 +180,30 @@ void ofApp::draw(){
portrait_fbo.end();
/* white depth map pass */
map_fbo_post.begin();
ofClear(0);
map_depth.begin();
map_depth.setUniformTexture("tex0", d, 0);
model_outptut_fbo.draw(0, 0);
map_depth.end();
map_fbo_post.end();
drawPortrait();
comp_fbo.begin();
ofClear(0, 0, 0, 0);
ofClear(ofColor::white);
shaders.begin();
portrait_fbo.draw(0, 0);
shaders.setUniformTexture("tex0", t, 0);
shaders.setUniformTexture("tex1", bayer, 1);
shaders.setUniform2f("resolution", map_w, map_h);
shaders.setUniform1f("time", ofGetElapsedTimef());
shaders.setUniform1i("frame", ofGetFrameNum());
model_outptut_fbo.draw(0, 0);
map_fbo_post.draw(0, 0);
/* actual map */
map_fbo_alpha.draw(0,0);
model_portrait_out_fbo.draw((map_w/3) * 2,0);
portrait_fbo.draw((map_w/3) * 2, 0);
//model_portrait_out_fbo.draw(0,0);
shaders.end();
comp_fbo.end();
@ -202,14 +212,14 @@ void ofApp::draw(){
//mapper.draw();
comp_fbo.draw(0,60);
server->print();
//server->print();
}
void ofApp::drawPortrait(){
Node n = server->getChosenNode();
float p_scale = 1 + ( ((1 + ofNoise(ofGetElapsedTimef() / 2)) / 2) * 2);
float p_scale = 1 + ( ((1 + ofNoise(ofGetElapsedTimef() / 5)) / 2) * 0.5);
portrait_pre_fbo.begin();
ofClear(0, 0, 0, 0);
@ -305,6 +315,7 @@ void ofApp::drawPortraitZ(){
p_depth.setUniformTexture("tex0", tex_color, 0);
p_depth.setUniformTexture("tex1", tex_depth, 1);
ofFill();
//ofRotateDeg(90, 0, 1, 0);
ofTranslate(p_noise_x, p_noise_y, 0);
plane.draw();
p_depth.end();
@ -384,11 +395,13 @@ void ofApp::createNodes(std::string json_path){
Node n;
n.img.load(j["image"]);
n.tex = n.img.getTexture();
n.error = j["lost"];
std::vector<float> t_embedding;
int count = 0;
for (const auto& value: j["vector"]){
if(value.is_number()){
if(value.is_number() && count < 4){
t_embedding.push_back(value.get<float>());
count++;
} else {
ofLogError() << "Vector value is not a number";
}
@ -420,30 +433,6 @@ void ofApp::createNodes(std::string json_path){
auto& n = nodes[i];
n.tsne_position = (glm::vec3(((vec[0] * 2) - 1) * tsne_scale, ((vec[1] * 2) - 1) * tsne_scale, -5.0f));
}
/* vp-test */
// auto queries = server->generateRandomVectors(10, 7);
// int k = 5;
// vector<vector<float>> test_vectors;
// // Set up random number generator
// std::random_device rd;
// std::mt19937 gen(rd());
// std::uniform_real_distribution<> dis(0.0, 1.0); // Random floats between -1 and 1
// for (int i = 0; i < 1; ++i) {
// std::vector<float> vec;
// for (int j = 0; j < 7; ++j) {
// vec.push_back(static_cast<float>(dis(gen)));
// }
// test_vectors.push_back(vec);
// }
// std::cout << test_vectors.size() << std::endl;
// vector<int> q_index = vp_tree.query(test_vectors, 5);
}
std::vector<std::vector<double>> ofApp::createDoubleVectorFromNodes(const std::vector<Node>& nodes) {
@ -524,19 +513,19 @@ void ofApp::queryKD(glm::vec3& _position, int k){
void ofApp::keyPressed(int key) {
switch(key) {
case OF_KEY_UP:
g_size += 10;
g_size += 5;
ofLog() << "Grid Size: " << g_size;
break;
case OF_KEY_DOWN:
g_size -= 10;
g_size -= 5;
ofLog() << "Grid Size: " << g_size;
break;
case 'q':
plane_size += 10;
plane_size += 5;
ofLog() << "Plane Size: " << plane_size;
break;
case 'w':
plane_size -=10;
plane_size -=5;
ofLog() << "Plane Size: " << plane_size;
break;
}

8
src/ofApp.h
View File

@ -45,6 +45,7 @@ class ofApp : public ofBaseApp{
ofFbo map_fbo;
ofFbo map_fbo_alpha;
ofFbo map_fbo_post;
ofFbo portrait_fbo;
ofFbo portrait_pre_fbo;
ofFbo portrait_pre_fbo_alpha;
@ -62,6 +63,7 @@ class ofApp : public ofBaseApp{
ofShader esp_shader;
ofShader vert_snap;
ofShader p_depth;
ofShader map_depth;
ofPlanePrimitive plane;
ofTexture bayer;
@ -99,14 +101,14 @@ class ofApp : public ofBaseApp{
float theta = 0.2;
bool normalise = true;
bool runManually = false;
float tsne_scale = 1000;
float tsne_scale = 500;
/* kd tree */
std::unique_ptr<KDTree> kd_tree;
std::vector<pointIndex> kd_result;
float g_size = 10;
float plane_size = 100;
float g_size = 5;
float plane_size = 110;
/* pi mapper */
ofxPiMapper mapper;

Write Preview
Loading…
Cancel
Save