mesh extrusion depth & others

bin/data/shaders/dither.frag

@@ -21,7 +21,8 @@ const mat4 ditherMatrix = mat4(
 
 void main() {
     vec2 uv = varyingtexcoord / resolution;
-    vec3 t = texture(tex0, varyingtexcoord).rgb;
+    vec4 texColor = texture(tex0, varyingtexcoord);
+    vec3 t = texColor.rgb;
     
     vec2 fragCoord = gl_FragCoord.xy;
     
@@ -37,5 +38,5 @@ void main() {
     // Quantize the color
     col = (floor(col * 8.0) + 0.5) / 8.0;
     
-    fragColor = vec4(col, 1.0);
+    fragColor = vec4(col, texColor.a);
 }

bin/data/shaders/p_depth.frag

@@ -0,0 +1,19 @@
+#version 150
+
+uniform sampler2DRect tex0;
+
+in vec2 tex_c;
+
+out vec4 outputColor;
+
+void main()
+{
+    vec4 texColor = texture(tex0, tex_c);    
+
+    // Discard fragments with alpha below a threshold
+    if (texColor.a < 0.1) {
+        discard;
+    }
+    
+    outputColor = texColor;
+}

bin/data/shaders/p_depth.vert

@@ -0,0 +1,50 @@
+#version 150
+
+uniform mat4 modelViewProjectionMatrix;
+uniform float gridSize;
+uniform vec2 resolution;
+uniform sampler2DRect tex1;
+
+in vec2 texcoord;
+in vec4 position;
+
+
+out vec2 tex_c;
+
+void main()
+{
+    tex_c = texcoord;
+
+    // vec4 t_sample = texture(tex0, tex_c);
+
+    float depth = texture(tex1, tex_c).r;
+
+    vec4 displaced = position;
+
+    displaced.z = depth * 400.0;
+
+
+    //vec2 resolution = vec2(1280.0, 960.0);
+
+    // Transform the vertex position to clip space
+    vec4 clipPos = modelViewProjectionMatrix * displaced;
+    
+    // Perform perspective division
+    vec3 ndcPos = clipPos.xyz / clipPos.w;
+    
+    // Convert to screen space
+    vec2 screenPos = (ndcPos.xy + 1.0) * 0.5 * resolution;
+    
+    // Snap to grid
+    screenPos = floor(screenPos / gridSize) * gridSize;
+    
+    // Convert back to NDC space
+    ndcPos.xy = (screenPos / resolution) * 2.0 - 1.0;
+    
+    // Reconstruct the clip space position
+    clipPos = vec4(ndcPos * clipPos.w, clipPos.w);
+    
+    gl_Position = clipPos;
+
+    //gl_Position = modelViewProjectionMatrix * position;
+}

bin/data/shaders/vertex_snap.vert

@@ -2,6 +2,7 @@
 
 uniform mat4 modelViewProjectionMatrix;
 uniform float gridSize;
+uniform vec2 resolution;
 
 in vec2 texcoord;
 in vec4 position;
@@ -12,7 +13,7 @@ void main()
 {
     tex_c = texcoord;
 
-    vec2 resolution = vec2(1280.0, 960.0);
+    //vec2 resolution = vec2(1280.0, 960.0);
 
     // Transform the vertex position to clip space
     vec4 clipPos = modelViewProjectionMatrix * position;

image-to-mesh.code-workspace

@@ -109,7 +109,8 @@
 			"executor": "cpp",
 			"timer": "cpp",
 			"buffer": "cpp",
-			"socket": "cpp"
+			"socket": "cpp",
+			"*.glsl": "cpp"
 		}
 	}
 }

src/Bullet.cpp

@@ -25,9 +25,6 @@ void Bullet::setup(vector<Node>& _nodes){
 
     shader.load("shaders/vertex_snap");
 
-    /* assign nodes */
-    //nodes = _nodes;
-
     std::cout << workerThreads.size() << std::endl;
 }
 
@@ -68,7 +65,7 @@ void Bullet::draw(){
     glEnable( GL_DEPTH_TEST );
     
     ofSetLineWidth(10.f);
-    ofDrawGrid(20, 10, true, false, false, true);
+    ofDrawGrid(20, 100, true, false, false, true);
 
     ofNoFill();
     ofSetColor(ofColor::yellow);
@@ -98,6 +95,7 @@ void Bullet::draw(){
             ofPushMatrix();
             shader.begin();
             shader.setUniform1f("gridSize", gridSize);
+            shader.setUniform2f("resolution", glm::vec2(1280.0, 960.0));
             shader.setUniformTexture("tex0", n.tex, 0);
             n.collider->transformGL();        
             ofScale(n.scale);
@@ -255,7 +253,7 @@ glm::vec4 Bullet::getCameraBounds(const ofCamera& cam){
     
     glm::vec3 cam_pos = cam.getPosition();
     float cam_scale = cam.getScale().x;
-    float bounds = ofMap(cam_scale, 0.01, 0.2, 10, 120) + 20;
+    float bounds = ofMap(cam_scale, 0.01, 0.2, 10, 240) + 20;
     return glm::vec4(cam_pos.x + bounds, cam_pos.x - bounds, cam_pos.y + bounds, cam_pos.y - bounds);
     
 }

src/network/Server.cpp

@@ -14,6 +14,9 @@ void Server::start(){
     for(auto& p : esp_pixels){
         p.allocate(128, 168, GL_RGB);
     }
+
+    /* set initial chosen node */
+    chosen_node = &nodes[0];
 }
 
 void Server::update(ofFbo& esp_comp){
@@ -158,6 +161,7 @@ void Server::checkActivity(){
 
         if (duration >= 2) {
             is_active = false;
+            //ofLogNotice() << "Inactive";
         }
     }
 }
@@ -245,6 +249,10 @@ void Server::sendImagesToClients(vector<ofImage>& imgs){
     ofSetLogLevel(OF_LOG_NOTICE);
 }
 
+Node& Server::getChosenNode(){
+    return *chosen_node;
+}
+
 void Server::close(){
     server.close();
 }

src/network/Server.h

@@ -37,6 +37,9 @@ class Server{
     void close();
     std::vector<std::vector<double>> generateRandomVectors(int count, int dimension);
 
+    /* getters */
+    Node& getChosenNode();
+    
     int port;
     ofxTCPServer server;
     std::unordered_map<int, ClientInfo> clients;

src/ofApp.cpp

@@ -4,11 +4,16 @@
 void ofApp::setup(){
 
     /* allocated fbo's */
-    map_fbo.allocate((ofGetWindowWidth() / 3) * 2, ofGetWindowHeight(), GL_RGB);
+    map_fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGB);
-    portrait_fbo.allocate((ofGetWindowWidth() / 3) * 1, ofGetWindowHeight(), GL_RGBA);
+    map_fbo_alpha.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA);
+    portrait_fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA);
+    portrait_pre_fbo.allocate((ofGetWindowWidth() / 3) * 1, ofGetWindowHeight(), GL_RGB);
+    portrait_pre_fbo_alpha.allocate((ofGetWindowWidth() / 3) * 1, ofGetWindowHeight(), GL_RGBA);
     comp_fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA);
-    model_outptut_fbo.allocate((ofGetWindowWidth() / 3) * 2, ofGetWindowHeight(), GL_RGB);
+
+    model_outptut_fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGB);
     model_esp_out_fbo.allocate(128 * 2, 168 * 2, GL_RGB);
+    model_portrait_out_fbo.allocate(portrait_pre_fbo_alpha.getWidth(), portrait_pre_fbo_alpha.getHeight(), GL_RGB);
 
     /* k-d image comp (4-images) */
     kd_out.allocate(128 * 2, 168 * 2, GL_RGB);
@@ -16,8 +21,10 @@ void ofApp::setup(){
 
     /* input images for model */
     model_image_esp.allocate(128 * 2, 168 * 2, OF_IMAGE_COLOR);
-    model_image.allocate((ofGetWindowWidth() / 3) * 2, ofGetWindowHeight(), OF_IMAGE_COLOR);
+    model_image.allocate(ofGetWindowWidth(), ofGetWindowHeight(), OF_IMAGE_COLOR);
+    model_image_portrait.allocate(portrait_pre_fbo_alpha.getWidth(), portrait_pre_fbo_alpha.getHeight(), OF_IMAGE_COLOR);
 
+    map_pixels.allocate(ofGetWindowWidth(), ofGetWindowHeight(), OF_PIXELS_RGB);
 
     /* allocated ofImages in esp_images */
     for(int i = 0; i < 4; i++){
@@ -34,22 +41,33 @@ void ofApp::setup(){
     ofEnableAlphaBlending();
 
     /* load */
+    ofLoadImage(bayer, "images/bayer.png");
+    bayer.setTextureWrap(GL_REPEAT, GL_REPEAT);
     shaders.load("shaders/dither");
     esp_shader.load("shaders/espDepth");
+    vert_snap.load("shaders/vertex_snap");
+    p_depth.load("shaders/p_depth");
    
     ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx";
+    ORTCHAR_T* modelPath_Small = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx";
 
     /* setup */
     bullet.setup(nodes);
 
-    depth_onnx.Setup(modelPath, true, true);
+    depth_onnx.Setup(modelPath, false, true);
-    depth_onnx_esp.Setup(modelPath, true, true);
+    depth_onnx_esp.Setup(modelPath_Small, false, true);
+    depth_onnx_portrait.Setup(modelPath_Small, false, true);
 
     depth_thread.setup(&model_image, &model_outptut_fbo, &depth_onnx);
     depth_esp.setup(&model_image_esp, &model_esp_out_fbo, &depth_onnx_esp);
+    depth_portrait.setup(&model_image_portrait, &model_portrait_out_fbo, &depth_onnx_portrait);
+
+    /* camera settings for portrait */
+    portrait_camera.setFov(90);
+    portrait_camera.setPosition(0, 150, 480);
+    portrait_camera.removeAllInteractions();
+    portrait_camera.disableMouseInput();
     
-    ofLoadImage(bayer, "images/bayer.png");
-    bayer.setTextureWrap(GL_REPEAT, GL_REPEAT);
 
     createNodes("data/json/embeddings.json");
 
@@ -82,20 +100,28 @@ void ofApp::update(){
     if(ofGetFrameNum() < 1){
         depth_thread.start();
         depth_esp.start();
+        depth_portrait.start();
     }
 
     /* write pixels to model input image */
-    map_fbo.readToPixels(map_pixels);
+    try{
-    model_image.setFromPixels(map_pixels);
+        map_fbo.readToPixels(map_pixels);
+        model_image.setFromPixels(map_pixels);
+    } catch (exception e) {
+        ofLogError() << "couldn't set pixels!";
+    }
+    
 
 
     try{
         depth_thread.update();
         depth_esp.update();
+        depth_portrait.update();
 
         /* set output to fbo's */
         depth_onnx.SetPixels(model_outptut_fbo);
         depth_onnx_esp.SetPixels(model_esp_out_fbo);
+        depth_onnx_portrait.SetPixels(model_portrait_out_fbo);
 
     } catch (exception e){
         std::cout << "Model did not run" << std::endl;
@@ -108,67 +134,124 @@ void ofApp::update(){
 void ofApp::draw(){
    
     ofPushStyle();
-    map_fbo.begin();
+    map_fbo_alpha.begin();
-    ofClear(ofColor::grey);
+    ofClear(0, 0, 0, 0);
     bullet.draw();
-    map_fbo.end();
+    map_fbo_alpha.end();
     ofPopStyle();
 
+    map_fbo.begin();
+    ofClear(0, 0, 0, 0);
+    map_fbo_alpha.draw(0, 0);
+    map_fbo.end();
+
     ofTexture t = map_fbo.getTexture();
+    ofTexture d = model_outptut_fbo.getTexture();
+
+    //model_outptut_fbo.draw(ofGetWindowWidth() / 2, 0);
+
+
+    esp_comp_fbo.begin();
 
+    esp_shader.begin();
+    esp_shader.setUniformTexture("tex0", model_esp_out_fbo.getTexture(), 0);
+    model_esp_out_fbo.draw(0, 0);
+    esp_comp_fbo.end();
+    esp_shader.end();
+
+    portrait_fbo.begin();
+
+    ofBackground(0, 0, 0, 0);
+
+    drawPortraitZ();
+
+    portrait_fbo.end();
+
+    drawPortrait();
 
     comp_fbo.begin();
+    ofClear(0, 0, 0, 0);
+    
+
     shaders.begin();
     shaders.setUniformTexture("tex0", t, 0);
     shaders.setUniformTexture("tex1", bayer, 1);
     shaders.setUniform2f("resolution", ofGetWidth(), ofGetHeight());
     shaders.setUniform1f("time", ofGetElapsedTimef());
     shaders.setUniform1i("frame", ofGetFrameNum());
-    ofClear(0, 0, 0, 0);
+    model_outptut_fbo.draw(0, 0);
-    map_fbo.draw(0,0);
+    portrait_fbo.draw(0, 0);
-
+    map_fbo_alpha.draw(0,0);
     shaders.end();
     comp_fbo.end();
 
 
     comp_fbo.draw(0,0);
-    model_outptut_fbo.draw(ofGetWindowWidth() / 2, 0);
+}
+
+void ofApp::drawPortrait(){
+
+    Node n = server->getChosenNode();
+    portrait_pre_fbo.begin();
+    float scale = min(
+        (float)portrait_pre_fbo.getWidth() / n.img.getWidth(),
+        (float)portrait_pre_fbo.getHeight() / n.img.getHeight()
+    );
+    float scaledWidth = n.img.getWidth() * scale;
+    float scaledHeight = n.img.getHeight() * scale;
+    float xPos = (portrait_pre_fbo.getWidth() - scaledWidth) / 2;
+    float yPos = (portrait_pre_fbo.getHeight() - scaledHeight) / 2;
+    n.img.draw(xPos, yPos, scaledWidth, scaledHeight);
+    portrait_pre_fbo.end();
+
+    portrait_pre_fbo_alpha.begin();
+    ofClear(0, 0, 0, 0);
+    n.img.draw(xPos, yPos, scaledWidth, scaledHeight);
+    portrait_pre_fbo_alpha.end();
 
+    ofPixels pix;
+    portrait_pre_fbo.readToPixels(pix);
+    model_image_portrait.setFromPixels(pix);
+}
 
-    esp_comp_fbo.begin();
+void ofApp::drawPortraitZ(){
+    ofTexture tex_color = portrait_pre_fbo_alpha.getTexture();
+    ofTexture tex_depth = model_portrait_out_fbo.getTexture();
 
-    esp_shader.begin();
+    portrait_camera.begin();
-    esp_shader.setUniformTexture("tex0", model_esp_out_fbo.getTexture(), 0);
+    ofEnableDepthTest();
-    model_esp_out_fbo.draw(0, 0);
-    esp_comp_fbo.end();
-    esp_shader.end();
 
-    esp_comp_fbo.draw(0, 0);
+    float time = ofGetElapsedTimef() / 10;
+    float p_noise_x = ofSignedNoise(time, time) * 20;
+    float p_noise_y = ofSignedNoise(time + 100, time + 100) * 20;
 
-    //server->print();
+    ofPushMatrix();
-    //shader_fbo.draw(0, 0);
-    // float planeScale = 0.75;
-    // int planeWidth = ofGetWidth() * planeScale;
-    // int planeHeight = ofGetHeight() * planeScale;
-    // int planeGridSize = 20;
-    // int planeColumns = planeWidth / planeGridSize;
-    // int planeRows = planeHeight / planeGridSize;
     
-    // plane.set(planeWidth, planeHeight, planeColumns, planeRows, OF_PRIMITIVE_TRIANGLES);
+    float planeScale = 1.0;
+    int planeWidth = portrait_pre_fbo_alpha.getWidth() * planeScale;
+    int planeHeight = portrait_pre_fbo_alpha.getHeight() * planeScale;
+    int planeGridSize = plane_size;
+    int planeColumns = planeWidth / planeGridSize;
+    int planeRows = planeHeight / planeGridSize;
 
-    // plane.mapTexCoordsFromTexture(t);
+    plane.set(planeWidth, planeHeight, planeColumns, planeRows, OF_PRIMITIVE_TRIANGLES);
+    plane.mapTexCoords(0, 0, planeWidth, planeHeight);
 
-    // t.bind();
+    p_depth.begin();
-    // shaders.begin();
+    p_depth.setUniform1f("gridSize", g_size);
-    // ofPushMatrix();
+    p_depth.setUniform2f("resolution", glm::vec2(portrait_fbo.getWidth(), portrait_fbo.getHeight()));
-    // ofTranslate(ofGetWidth()/2, ofGetHeight()/2);
+    p_depth.setUniformTexture("tex0", tex_color, 0);
-    // plane.draw();
+    p_depth.setUniformTexture("tex1", tex_depth, 1);
-    // ofPopMatrix();
+    ofFill();
-    // shaders.end();
+    ofTranslate(p_noise_x, p_noise_y, 0);
-    // t.unbind();
+    plane.draw();
+    p_depth.end();
 
 
-    //t.draw(ofGetWindowWidth() / 2, 0);
+    ofPopMatrix();
+    ofDisableDepthTest();
+    
+    portrait_camera.end();
 }
 
 /* creates an fbo with four cropped images, to preprare for model input */
@@ -372,7 +455,27 @@ void ofApp::queryKD(glm::vec3& _position, int k){
 
     size_t index = kd_result[0].second;
 
-    std::cout << "camera positions: " << _position << ", nearest node: "<< nodes[index].tsne_position << " " << index << std::endl;
+    //std::cout << "camera positions: " << _position << ", nearest node: "<< nodes[index].tsne_position << " " << index << std::endl;
 
 }
 
+void ofApp::keyPressed(int key) {
+    switch(key) {
+        case OF_KEY_UP:
+            g_size += 10;
+            ofLog() << "Grid Size: " << g_size;
+            break;
+        case OF_KEY_DOWN:
+            g_size -= 10;
+            ofLog() << "Grid Size: " << g_size;
+            break;
+        case 'q':
+            plane_size += 10;
+            ofLog() << "Plane Size: " << plane_size;
+            break;
+        case 'w':
+            plane_size -=10;
+            ofLog() << "Plane Size: " << plane_size;
+            break;
+    }
+}

src/ofApp.h

@@ -25,9 +25,12 @@ class ofApp : public ofBaseApp{
 		void buildKDTree();
 		void queryKD(glm::vec3& _position, int k);
 		void buildMeshes();
+		void drawPortrait();
+		void drawPortraitZ();
+		void keyPressed(int key);
 		void exit();
 
-		ofEasyCam cam;
+		ofEasyCam portrait_camera;
 		ofImage img;
 		ofMesh mesh;
 		MeshGenerator mesh_generator;
@@ -35,17 +38,24 @@ class ofApp : public ofBaseApp{
 		vector<ofImage> images;
 		
 		ofFbo map_fbo;
+		ofFbo map_fbo_alpha;
 		ofFbo portrait_fbo;
+		ofFbo portrait_pre_fbo;
+		ofFbo portrait_pre_fbo_alpha;
 		ofFbo comp_fbo;
 		ofFbo model_outptut_fbo;
 		ofFbo model_esp_out_fbo;
+		ofFbo model_portrait_out_fbo;
 
 		ofFbo esp_comp_fbo;
 		ofFbo kd_out;
 		ofImage model_image_esp;
+		ofImage model_image_portrait;
 
 		ofShader shaders;
 		ofShader esp_shader;
+		ofShader vert_snap;
+		ofShader p_depth;
 
 		ofPlanePrimitive plane;
 		ofTexture bayer;
@@ -55,8 +65,10 @@ class ofApp : public ofBaseApp{
 		/* onnx */
 		Onnx depth_onnx;
 		Onnx depth_onnx_esp;
+		Onnx depth_onnx_portrait;
 		ModelThread depth_thread;
 		ModelThread depth_esp;
+		ModelThread depth_portrait;
 		ofImage model_image;
 		ofPixels map_pixels;
 
@@ -87,4 +99,7 @@ class ofApp : public ofBaseApp{
 		std::unique_ptr<KDTree> kd_tree;
 		std::vector<pointIndex> kd_result;
 
+		float g_size = 10;
+		float plane_size = 50;
+		
 };