diff --git a/bin/data/camera_angles.txt b/bin/data/camera_angles.txt index 8a31782..95d6a97 100644 --- a/bin/data/camera_angles.txt +++ b/bin/data/camera_angles.txt @@ -1 +1,8 @@ --131.228, 502.624, 706.1 : 0.898726, -0.436474, -0.0351978, -0.0170607 : 0.313542, 0.313542, 0.313542 \ No newline at end of file +-131.228, 502.624, 706.1 : 0.898726, -0.436474, -0.0351978, -0.0170607 : 0.313542, 0.313542, 0.313542 + +ortho: + +-13.9031, -573.818, 730.531 : 0.902208, 0.429743, 0.029696, -0.0140357 : 0.313542, 0.313542, 0.313542 + +-209.623, -505.192, 609.98 : 0.881811, 0.44688, -0.133594, 0.0678069 : 0.313542, 0.313542, 0.313542 + diff --git a/bin/data/new_tree.bin b/bin/data/new_tree.bin index a9377cb..a9ddf82 100644 Binary files a/bin/data/new_tree.bin and b/bin/data/new_tree.bin differ diff --git a/bin/data/ofxpimapper.xml b/bin/data/ofxpimapper.xml index b1a73ac..41e7e12 100644 --- a/bin/data/ofxpimapper.xml +++ b/bin/data/ofxpimapper.xml @@ -2,20 +2,110 @@ - 280.000000000 - -460.000000000 + 0.000000000 + 0.000000000 + + + 0.000000000 + 0.000000000 + + + 0.000000000 + 0.000000000 + + + 0.000000000 + 0.000000000 + + + + + 0.000000000 + 0.000000000 + + + 1.000000000 + 0.000000000 + + + 1.000000000 + 1.000000000 + + + 0.000000000 + 1.000000000 + + + + fbo + stubborn_vectors + + + 0 + + + + + + -21.000000000 + -321.000000000 + + + 1915.000000000 + -122.000000000 + + + 1923.000000000 + 7.000000000 + + + -8.000000000 + 15.000000000 + + + + + 0.120312497 + -0.004629630 + + + 1.120312452 + -0.004629630 + + + 1.120312452 + 0.995370388 + + + 0.120312497 + 0.995370388 + + + + fbo + stubborn_vectors + + + 0 + + + + + + -20.171579361 + 7.979064941 - 2200.000000000 - -460.000000000 + 31.171579361 + 5.964111328 - 2200.000000000 - 500.000000000 + 14.059679985 + 1079.916503906 - 280.000000000 - 500.000000000 + -7.074600220 + 1096.035888672 @@ -47,20 +137,20 @@ - -1210.000000000 - -338.000000000 + 3.000000000 + 1073.000000000 - 610.000000000 - -338.000000000 + 1941.000000000 + 1076.000000000 - 610.000000000 - 642.000000000 + 1866.319824219 + 1122.291748047 - -1210.000000000 - 642.000000000 + 43.563842773 + 1139.496093750 @@ -82,11 +172,56 @@ - none - none + fbo + stubborn_vectors - 1 + 0 + + + + + + 1868.000000000 + 2.000000000 + + + 1923.000000000 + 5.000000000 + + + 1928.000000000 + 1075.000000000 + + + 1893.000000000 + 1079.000000000 + + + + + 0.000000000 + 0.000000000 + + + 1.000000000 + 0.000000000 + + + 1.000000000 + 1.000000000 + + + 0.000000000 + 1.000000000 + + + + fbo + stubborn_vectors + + + 0 diff --git a/bin/data/shaders/p_depth.vert b/bin/data/shaders/p_depth.vert index a5c2828..049503e 100644 --- a/bin/data/shaders/p_depth.vert +++ b/bin/data/shaders/p_depth.vert @@ -5,6 +5,7 @@ uniform float gridSize; uniform vec2 resolution; uniform vec2 threshold; uniform sampler2DRect tex1; +uniform float depth_mult; in vec2 texcoord; in vec4 position; @@ -18,15 +19,21 @@ void main() //float depth = pow(texture(tex1, tex_c).r, 2); - vec2 depthCoord = texcoord * vec2(1920.0/2.0, 960.0); + vec2 depthCoord = texcoord * vec2(1920.0/2.0, 1080.0); float depth = texture(tex1, depthCoord).r; + // Apply a contrast adjustment only to the range [0.5, 1.0] + // if (depth > 0.4) { + // // Shift the range to [0, 1] and apply pow() for more contrast, then shift it back + // depth = pow((depth - 0.4) / (1.0 - 0.4), 4.0) * (1.0 - 0.4) + 0.4; + // } + // Normalize the depth value between 0 and 1 based on min/max float normalizedDepth = (depth); vec4 displaced = position; - displaced.z = normalizedDepth * 400.0; + displaced.z = normalizedDepth * depth_mult; // Transform the vertex position to clip space vec4 clipPos = modelViewProjectionMatrix * displaced; diff --git a/bin/image-to-mesh b/bin/image-to-mesh index bbfbf87..78073bf 100755 Binary files a/bin/image-to-mesh and b/bin/image-to-mesh differ diff --git a/src/Bullet.cpp b/src/Bullet.cpp index e4c10b1..6de0f2b 100644 --- a/src/Bullet.cpp +++ b/src/Bullet.cpp @@ -124,8 +124,8 @@ void Bullet::update(bool& is_controller_active, Node& chosen_node, vector // Smoothed zoom based on velocity float vel_magnitude = glm::length(camera_velocity); - static float current_zoom = 0.05f; - float target_zoom = ofMap(vel_magnitude, 0, MAX_VELOCITY, 0.05, 0.3, true); + static float current_zoom = 0.02f; + float target_zoom = ofMap(vel_magnitude, 0, MAX_VELOCITY, 0.02, 0.3, true); current_zoom = current_zoom + (target_zoom - current_zoom) * 0.05f; // Calculate scale-out factor if we're in random walk mode @@ -233,7 +233,7 @@ void Bullet::draw(){ shader.setUniform1f("greyscale", is_nearest); shader.setUniform1f("gridSize", gridSize); - shader.setUniform2f("resolution", glm::vec2(1920.0/2.0, 960.0)); + shader.setUniform2f("resolution", glm::vec2(1920.0/2.0, 1080.0)); shader.setUniformTexture("tex0", n.tex, 0); n.collider->transformGL(); ofScale(n.scale); diff --git a/src/Bullet.h b/src/Bullet.h index c857e39..b3d21b8 100644 --- a/src/Bullet.h +++ b/src/Bullet.h @@ -11,6 +11,7 @@ struct Node { ofMesh col_mesh; ofMesh mesh; ofImage img; + std::string img_path; ofTexture tex; glm::vec3 scale; glm::vec3 tsne_position; @@ -18,11 +19,19 @@ struct Node { int error; }; +enum CameraMode{ + ORTHO, + PERSP +}; + struct CameraPosition { + + CameraMode mode; glm::vec3 position; glm::quat rotation; float scale; float time; + float depth_mult; }; class Bullet{ diff --git a/src/network/Server.cpp b/src/network/Server.cpp index fb82cbc..f201aa3 100644 --- a/src/network/Server.cpp +++ b/src/network/Server.cpp @@ -173,6 +173,43 @@ void Server::sendOSCMessage(){ bang_new_mesh.addTriggerArg(); messages.push_back(bang_new_mesh); + std::string& path = (*chosen_node).img_path; + std::string folder = ""; + + size_t f_slash = path.find('/'); + size_t s_slash = path.find('/', f_slash + 1); + + if(!f_slash != string::npos && s_slash != string::npos){ + folder = path.substr(f_slash + 1, s_slash - f_slash -1); + } + + ofLog() << folder; + + std::string audio_file = folder; + + // Check if file exists in a given dir + ofFile file("/home/cailean/Desktop/rave/all_wav_files/" + audio_file + ".wav"); + + if((audio_file != past_audio_file) && file.exists()){ + me_file.setAddress("/emote/filename"); + me_file.addStringArg(audio_file + ".wav"); + messages.push_back(me_file); + past_audio_file = audio_file; + } + + me_0.setAddress("/emote/0"); + me_0.addFloatArg(embedding.emotions["neutral"]); + messages.push_back(me_0); + me_1.setAddress("/emote/1"); + me_1.addFloatArg(embedding.emotions["happy"]); + messages.push_back(me_1); + me_2.setAddress("/emote/2"); + me_2.addFloatArg(embedding.emotions["sad"]); + messages.push_back(me_2); + me_3.setAddress("/emote/3"); + me_3.addFloatArg(embedding.emotions["angry"]); + messages.push_back(me_3); + for (auto& msg : messages){ osc_sender.sendMessage(msg, false); } diff --git a/src/ofApp.cpp b/src/ofApp.cpp index 66ca9f6..d2d61e1 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -2,7 +2,7 @@ //-------------------------------------------------------------- void ofApp::setup(){ - + draw_face = true; /* allocated fbo's */ map_fbo.allocate(ofGetWindowWidth() / 2, map_h, GL_RGB); map_fbo_alpha.allocate(map_fbo.getWidth(), map_h, GL_RGBA); @@ -32,6 +32,14 @@ void ofApp::setup(){ alpha_demo.allocate(portrait_pre_fbo_alpha.getWidth(), portrait_pre_fbo_alpha.getHeight(), OF_IMAGE_COLOR_ALPHA); + /* black pi mapper bg */ + mapper_black.allocate(300, 1080, GL_RGB); + mapper_black.begin(); + ofClear(0, 0, 0, 255); + mapper_black.end(); + + mapper_black.getTexture().setTextureMinMagFilter(GL_NEAREST, GL_NEAREST); + /* allocated ofImages in esp_images */ for(int i = 0; i < 4; i++){ ofFbo temp; @@ -44,7 +52,6 @@ void ofApp::setup(){ ofSetVerticalSync(true); ofDisableArbTex(); ofEnableDepthTest(); - ofDisableAntiAliasing(); ofEnableAlphaBlending(); /* load */ @@ -64,9 +71,10 @@ void ofApp::setup(){ bullet.setup(); /* pi mapper setup */ - // mapper.registerFboSource(projection_src); - // mapper.setup(); - // projection_src.setFbo(&comp_fbo); + mapper.registerFboSource(projection_src); + mapper.setup(); + projection_src.setFbo(&comp_fbo); + projection_src.setFbo(&mapper_black); /* onnx setup */ depth_onnx.Setup(modelPath, false, true); @@ -85,25 +93,35 @@ void ofApp::setup(){ cp.position = glm::vec3(-131.228, 502.624, 706.1); cp.rotation = glm::quat(0.898726, -0.436474, -0.0351978, -0.0170607); cp.scale = 0.313542; + cp.mode = CameraMode::ORTHO; + cp.depth_mult = 400; cam_positions.push_back(cp); - cp.position = glm::vec3(7.987, -195.329, 813.56); - cp.rotation = glm::quat(0.993, 0.117, 0.005, -0.00057); + cp.position = glm::vec3(-55.0256, -561.005, 750.691); + cp.rotation = glm::quat(0.938642, 0.340495, 0.0493511, -0.0177967); + cp.scale = 0.313542; + cp.mode = CameraMode::PERSP; + cp.depth_mult = 600; cam_positions.push_back(cp); - cp.position = glm::vec3(160.81, -87.86, 731.575); - cp.rotation = glm::quat(0.980, 0.027, 0.1934, -0.0053); - cam_positions.push_back(cp); + cp.position = glm::vec3(-13.9031, -573.818, 730.531); + cp.rotation = glm::quat(0.902208, 0.429743, 0.029696, -0.0140357); + cp.scale = 0.313542; + cp.mode =CameraMode::PERSP; + cp.depth_mult = 600; + //cam_positions.push_back(cp); /* settings */ portrait_camera.enableOrtho(); portrait_camera.setNearClip(-10000); portrait_camera.setFarClip(10000); - portrait_camera.setPosition(cam_positions[0].position); - portrait_camera.setOrientation(cam_positions[0].rotation); - portrait_camera.setScale(cam_positions[0].scale); - portrait_camera.removeAllInteractions(); - portrait_camera.disableMouseInput(); + current_cp = cam_positions[1]; + portrait_camera.setPosition(current_cp.position); + portrait_camera.setOrientation(current_cp.rotation); + portrait_camera.setScale(current_cp.scale); + updateCurrentCameraMode(); + // portrait_camera.removeAllInteractions(); + // portrait_camera.disableMouseInput(); createNodes("data/json/sv_embeddings.json"); @@ -132,6 +150,7 @@ void ofApp::update(){ if(n.tsne_position != last_chosen_node.tsne_position) { portrait_needs_update = true; last_chosen_node = n; + //current_cp = cam_positions[ofRandom(cam_positions.size())]; } else { portrait_needs_update = false; } @@ -178,7 +197,7 @@ void ofApp::update(){ std::cout << "Model did not run" << std::endl; } - //mapper.update(); + mapper.update(); bullet.update(server->is_active, server->getChosenNode(), nn_nodes); if(tsne_update_complete) { @@ -190,6 +209,8 @@ void ofApp::update(){ tsne_iter_idx = (tsne_iter_idx + 1) % 3; updateTSNEPositions(nodes); } + + //updateCurrentCameraMode(); } //-------------------------------------------------------------- @@ -246,15 +267,15 @@ void ofApp::draw(){ shaders.setUniform1i("frame", ofGetFrameNum()); map_fbo_post.draw(0, 0); map_fbo_alpha.draw(0,0); + //alpha_demo.draw(0, 0); shaders.end(); comp_fbo.end(); comp_fbo.getTexture().setTextureMinMagFilter(GL_NEAREST, GL_NEAREST); - //mapper.draw(); - - ofTranslate(0, 60); + ofTranslate(0, 0); comp_fbo.draw(0, 0); + mapper.draw(); if(bullet.print_debug) server->print(); @@ -316,7 +337,7 @@ void ofApp::drawPortrait(){ if(yolo.checkInferenceComplete()) { /* check if face is detected, and crop potrait */ - if(detected_faces.size() > 0){ + if(detected_faces.size() > 0 && draw_face){ for(auto& f : detected_faces){ float crop_width = f.box.x2 - f.box.x1; float crop_height = f.box.y2 - f.box.y1; @@ -405,7 +426,7 @@ void ofApp::drawPortraitZ(){ /* if no faces found use full portrait texture */ ofTexture tex_color; - if(detected_faces.size() > 0){ + if(detected_faces.size() > 0 && draw_face){ tex_color = portrait_cropped_alpha.getTexture(); } else { tex_color = portrait_pre_fbo_alpha.getTexture(); @@ -438,44 +459,45 @@ void ofApp::drawPortraitZ(){ - // Second plane with offset - ofPushMatrix(); - p_depth.begin(); - p_depth.setUniform1f("gridSize", g_size); - p_depth.setUniform2f("resolution", glm::vec2(portrait_fbo.getWidth(), portrait_fbo.getHeight())); - p_depth.setUniform2f("threshold", glm::vec2(minDepth, max_r)); - p_depth.setUniformTexture("tex0", tex_color, 0); - p_depth.setUniformTexture("tex1", tex_depth, 1); - ofFill(); - // Offset the second plane - adjust these values as needed - ofTranslate(p_noise_x - 50, p_noise_y - 0, -150); // moved right (+200), up (+100) and back (-300) - ofRotateDeg(60, 0, 1, 0); - plane.draw(); - p_depth.end(); - ofPopMatrix(); - ofDisableDepthTest(); - - // Second plane with offset - ofPushMatrix(); - p_depth.begin(); - p_depth.setUniform1f("gridSize", g_size); - p_depth.setUniform2f("resolution", glm::vec2(portrait_fbo.getWidth(), portrait_fbo.getHeight())); - p_depth.setUniform2f("threshold", glm::vec2(minDepth, max_r)); - p_depth.setUniformTexture("tex0", tex_color, 0); - p_depth.setUniformTexture("tex1", tex_depth, 1); - ofFill(); - // Offset the second plane - adjust these values as needed - ofTranslate(p_noise_x - 140, p_noise_y - 0, -150); // moved right (+200), up (+100) and back (-300) - ofRotateDeg(-60, 0, 1, 0); - plane.draw(); - p_depth.end(); - ofPopMatrix(); - ofDisableDepthTest(); + // // Second plane with offset + // ofPushMatrix(); + // p_depth.begin(); + // p_depth.setUniform1f("gridSize", g_size); + // p_depth.setUniform2f("resolution", glm::vec2(portrait_fbo.getWidth(), portrait_fbo.getHeight())); + // p_depth.setUniform2f("threshold", glm::vec2(minDepth, max_r)); + // p_depth.setUniformTexture("tex0", tex_color, 0); + // p_depth.setUniformTexture("tex1", tex_depth, 1); + // ofFill(); + // // Offset the second plane - adjust these values as needed + // ofTranslate(p_noise_x - 50, p_noise_y - 0, -150); // moved right (+200), up (+100) and back (-300) + // ofRotateDeg(60, 0, 1, 0); + // plane.draw(); + // p_depth.end(); + // ofPopMatrix(); + // ofDisableDepthTest(); + + // // Second plane with offset + // ofPushMatrix(); + // p_depth.begin(); + // p_depth.setUniform1f("gridSize", g_size); + // p_depth.setUniform2f("resolution", glm::vec2(portrait_fbo.getWidth(), portrait_fbo.getHeight())); + // p_depth.setUniform2f("threshold", glm::vec2(minDepth, max_r)); + // p_depth.setUniformTexture("tex0", tex_color, 0); + // p_depth.setUniformTexture("tex1", tex_depth, 1); + // ofFill(); + // // Offset the second plane - adjust these values as needed + // ofTranslate(p_noise_x - 140, p_noise_y - 0, -150); // moved right (+200), up (+100) and back (-300) + // ofRotateDeg(-60, 0, 1, 0); + // plane.draw(); + // p_depth.end(); + // ofPopMatrix(); + // ofDisableDepthTest(); ofPushMatrix(); p_depth.begin(); p_depth.setUniform1f("gridSize", g_size); + p_depth.setUniform1f("depth_mult", current_cp.depth_mult); p_depth.setUniform2f("resolution", glm::vec2(portrait_fbo.getWidth(), portrait_fbo.getHeight())); p_depth.setUniform2f("threshold", glm::vec2(minDepth, max_r)); p_depth.setUniformTexture("tex0", tex_color, 0); @@ -485,6 +507,7 @@ void ofApp::drawPortraitZ(){ plane.draw(); p_depth.end(); ofPopMatrix(); + ofDisableDepthTest(); portrait_camera.end(); } @@ -562,6 +585,7 @@ void ofApp::createNodes(std::string json_path){ if(addNode) { // Only process when flag is true Node n; n.img.load(j["image"]); + n.img_path = (j["image"]); n.tex = n.img.getTexture(); n.error = j["error"]; std::vector t_embedding; @@ -741,7 +765,7 @@ void ofApp::keyPressed(int key) { portrait_camera.setPosition(cam_positions[cam_pos_idx].position); portrait_camera.setOrientation(cam_positions[cam_pos_idx].rotation); portrait_camera.setScale(cam_positions[cam_pos_idx].scale); - case 'p': + case 'n': if(bullet.print_debug){ bullet.print_debug = false; } else { @@ -752,8 +776,18 @@ void ofApp::keyPressed(int key) { case 't': tsne_iter_idx++; updateTSNEPositions(nodes); + case 'v': + ofToggleFullscreen(); + case 'o': + if(draw_face){ + draw_face = false; + } else { + draw_face = true; + } + case 'e': + current_cp = cam_positions[ofRandom(cam_positions.size())]; } - //mapper.keyPressed(key); + mapper.keyPressed(key); } // Your main thread function: @@ -766,22 +800,43 @@ void ofApp::onTSNEUpdateComplete() { } void ofApp::keyReleased(int key){ - //mapper.keyReleased(key); + mapper.keyReleased(key); } //-------------------------------------------------------------- void ofApp::mouseDragged(int x, int y, int button){ - //mapper.mouseDragged(x, y, button); + mapper.mouseDragged(x, y, button); } //-------------------------------------------------------------- void ofApp::mousePressed(int x, int y, int button){ - //mapper.mousePressed(x, y, button); + mapper.mousePressed(x, y, button); } //-------------------------------------------------------------- void ofApp::mouseReleased(int x, int y, int button){ - //mapper.mouseReleased(x, y, button); + mapper.mouseReleased(x, y, button); +} + +void ofApp::updateCurrentCameraMode(){ + if(current_cp.mode == CameraMode::ORTHO){ + portrait_camera.enableOrtho(); + portrait_camera.setNearClip(-10000); + portrait_camera.setFarClip(10000); + draw_face = true; + plane_size = 70; + } else { + portrait_camera.disableOrtho(); + portrait_camera.setNearClip(0); + portrait_camera.setFarClip(0); + draw_face = false; + plane_size = 55; + } + portrait_camera.setPosition(current_cp.position); + portrait_camera.setOrientation(current_cp.rotation); + portrait_camera.setScale(current_cp.scale); + + } ofMesh ofApp::createCustomPlane(float width, float height, int numX, int numY) { diff --git a/src/ofApp.h b/src/ofApp.h index 5022f5c..2880aa7 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -36,6 +36,7 @@ class ofApp : public ofBaseApp{ void mouseReleased(int x, int y, int button); void mouseDragged(int x, int y, int button); void onTSNEUpdateComplete(); + void updateCurrentCameraMode(); void exit(); @@ -61,6 +62,8 @@ class ofApp : public ofBaseApp{ ofFbo model_esp_out_fbo; ofFbo model_portrait_out_fbo; + ofFbo mapper_black; + ofFbo esp_comp_fbo; ofFbo kd_out; ofImage model_image_esp; @@ -133,12 +136,13 @@ class ofApp : public ofBaseApp{ /* pi mapper */ ofxPiMapper mapper; QuadSource projection_src; - int map_h = 960; + int map_h = 1080; int map_w = 1920; /* camera positions */ vector cam_positions; float cam_pos_idx = 0; + CameraPosition current_cp; Node last_chosen_node; bool portrait_needs_update; @@ -160,6 +164,8 @@ class ofApp : public ofBaseApp{ const float TNSE_DURATION = 360.0f; float tsne_start_time; int tsne_iter_idx = 0; + + bool draw_face; @@ -172,4 +178,5 @@ class ofApp : public ofBaseApp{ void updateTSNEPositions(vector& _nodes); void updateTSNEPositionsThreaded(vector& _nodes); + }; diff --git a/src/utils/QuadSource.cpp b/src/utils/QuadSource.cpp index 98bf92e..38d1eee 100644 --- a/src/utils/QuadSource.cpp +++ b/src/utils/QuadSource.cpp @@ -5,7 +5,7 @@ void QuadSource::setup(){ name = "stubborn_vectors"; // allocate size - allocate(1920, 960); + allocate(1920, 1080); }