Browse Source

mesh extrusion depth & others

master
cailean 2 months ago
parent
commit
fdf75913ac
  1. BIN
      bin/data/new_tree.bin
  2. 5
      bin/data/shaders/dither.frag
  3. 19
      bin/data/shaders/p_depth.frag
  4. 50
      bin/data/shaders/p_depth.vert
  5. 3
      bin/data/shaders/vertex_snap.vert
  6. BIN
      bin/data/vp-tree.bin
  7. BIN
      bin/image-to-mesh
  8. 3
      image-to-mesh.code-workspace
  9. 8
      src/Bullet.cpp
  10. 8
      src/network/Server.cpp
  11. 3
      src/network/Server.h
  12. 191
      src/ofApp.cpp
  13. 17
      src/ofApp.h

BIN
bin/data/new_tree.bin

Binary file not shown.

5
bin/data/shaders/dither.frag

@ -21,7 +21,8 @@ const mat4 ditherMatrix = mat4(
void main() {
vec2 uv = varyingtexcoord / resolution;
vec3 t = texture(tex0, varyingtexcoord).rgb;
vec4 texColor = texture(tex0, varyingtexcoord);
vec3 t = texColor.rgb;
vec2 fragCoord = gl_FragCoord.xy;
@ -37,5 +38,5 @@ void main() {
// Quantize the color
col = (floor(col * 8.0) + 0.5) / 8.0;
fragColor = vec4(col, 1.0);
fragColor = vec4(col, texColor.a);
}

19
bin/data/shaders/p_depth.frag

@ -0,0 +1,19 @@
#version 150
uniform sampler2DRect tex0;
in vec2 tex_c;
out vec4 outputColor;
void main()
{
vec4 texColor = texture(tex0, tex_c);
// Discard fragments with alpha below a threshold
if (texColor.a < 0.1) {
discard;
}
outputColor = texColor;
}

50
bin/data/shaders/p_depth.vert

@ -0,0 +1,50 @@
#version 150
uniform mat4 modelViewProjectionMatrix;
uniform float gridSize;
uniform vec2 resolution;
uniform sampler2DRect tex1;
in vec2 texcoord;
in vec4 position;
out vec2 tex_c;
void main()
{
tex_c = texcoord;
// vec4 t_sample = texture(tex0, tex_c);
float depth = texture(tex1, tex_c).r;
vec4 displaced = position;
displaced.z = depth * 400.0;
//vec2 resolution = vec2(1280.0, 960.0);
// Transform the vertex position to clip space
vec4 clipPos = modelViewProjectionMatrix * displaced;
// Perform perspective division
vec3 ndcPos = clipPos.xyz / clipPos.w;
// Convert to screen space
vec2 screenPos = (ndcPos.xy + 1.0) * 0.5 * resolution;
// Snap to grid
screenPos = floor(screenPos / gridSize) * gridSize;
// Convert back to NDC space
ndcPos.xy = (screenPos / resolution) * 2.0 - 1.0;
// Reconstruct the clip space position
clipPos = vec4(ndcPos * clipPos.w, clipPos.w);
gl_Position = clipPos;
//gl_Position = modelViewProjectionMatrix * position;
}

3
bin/data/shaders/vertex_snap.vert

@ -2,6 +2,7 @@
uniform mat4 modelViewProjectionMatrix;
uniform float gridSize;
uniform vec2 resolution;
in vec2 texcoord;
in vec4 position;
@ -12,7 +13,7 @@ void main()
{
tex_c = texcoord;
vec2 resolution = vec2(1280.0, 960.0);
//vec2 resolution = vec2(1280.0, 960.0);
// Transform the vertex position to clip space
vec4 clipPos = modelViewProjectionMatrix * position;

BIN
bin/data/vp-tree.bin

Binary file not shown.

BIN
bin/image-to-mesh

Binary file not shown.

3
image-to-mesh.code-workspace

@ -109,7 +109,8 @@
"executor": "cpp",
"timer": "cpp",
"buffer": "cpp",
"socket": "cpp"
"socket": "cpp",
"*.glsl": "cpp"
}
}
}

8
src/Bullet.cpp

@ -25,9 +25,6 @@ void Bullet::setup(vector<Node>& _nodes){
shader.load("shaders/vertex_snap");
/* assign nodes */
//nodes = _nodes;
std::cout << workerThreads.size() << std::endl;
}
@ -68,7 +65,7 @@ void Bullet::draw(){
glEnable( GL_DEPTH_TEST );
ofSetLineWidth(10.f);
ofDrawGrid(20, 10, true, false, false, true);
ofDrawGrid(20, 100, true, false, false, true);
ofNoFill();
ofSetColor(ofColor::yellow);
@ -98,6 +95,7 @@ void Bullet::draw(){
ofPushMatrix();
shader.begin();
shader.setUniform1f("gridSize", gridSize);
shader.setUniform2f("resolution", glm::vec2(1280.0, 960.0));
shader.setUniformTexture("tex0", n.tex, 0);
n.collider->transformGL();
ofScale(n.scale);
@ -255,7 +253,7 @@ glm::vec4 Bullet::getCameraBounds(const ofCamera& cam){
glm::vec3 cam_pos = cam.getPosition();
float cam_scale = cam.getScale().x;
float bounds = ofMap(cam_scale, 0.01, 0.2, 10, 120) + 20;
float bounds = ofMap(cam_scale, 0.01, 0.2, 10, 240) + 20;
return glm::vec4(cam_pos.x + bounds, cam_pos.x - bounds, cam_pos.y + bounds, cam_pos.y - bounds);
}

8
src/network/Server.cpp

@ -14,6 +14,9 @@ void Server::start(){
for(auto& p : esp_pixels){
p.allocate(128, 168, GL_RGB);
}
/* set initial chosen node */
chosen_node = &nodes[0];
}
void Server::update(ofFbo& esp_comp){
@ -158,6 +161,7 @@ void Server::checkActivity(){
if (duration >= 2) {
is_active = false;
//ofLogNotice() << "Inactive";
}
}
}
@ -245,6 +249,10 @@ void Server::sendImagesToClients(vector<ofImage>& imgs){
ofSetLogLevel(OF_LOG_NOTICE);
}
Node& Server::getChosenNode(){
return *chosen_node;
}
void Server::close(){
server.close();
}

3
src/network/Server.h

@ -37,6 +37,9 @@ class Server{
void close();
std::vector<std::vector<double>> generateRandomVectors(int count, int dimension);
/* getters */
Node& getChosenNode();
int port;
ofxTCPServer server;
std::unordered_map<int, ClientInfo> clients;

191
src/ofApp.cpp

@ -4,11 +4,16 @@
void ofApp::setup(){
/* allocated fbo's */
map_fbo.allocate((ofGetWindowWidth() / 3) * 2, ofGetWindowHeight(), GL_RGB);
portrait_fbo.allocate((ofGetWindowWidth() / 3) * 1, ofGetWindowHeight(), GL_RGBA);
map_fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGB);
map_fbo_alpha.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA);
portrait_fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA);
portrait_pre_fbo.allocate((ofGetWindowWidth() / 3) * 1, ofGetWindowHeight(), GL_RGB);
portrait_pre_fbo_alpha.allocate((ofGetWindowWidth() / 3) * 1, ofGetWindowHeight(), GL_RGBA);
comp_fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA);
model_outptut_fbo.allocate((ofGetWindowWidth() / 3) * 2, ofGetWindowHeight(), GL_RGB);
model_outptut_fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGB);
model_esp_out_fbo.allocate(128 * 2, 168 * 2, GL_RGB);
model_portrait_out_fbo.allocate(portrait_pre_fbo_alpha.getWidth(), portrait_pre_fbo_alpha.getHeight(), GL_RGB);
/* k-d image comp (4-images) */
kd_out.allocate(128 * 2, 168 * 2, GL_RGB);
@ -16,8 +21,10 @@ void ofApp::setup(){
/* input images for model */
model_image_esp.allocate(128 * 2, 168 * 2, OF_IMAGE_COLOR);
model_image.allocate((ofGetWindowWidth() / 3) * 2, ofGetWindowHeight(), OF_IMAGE_COLOR);
model_image.allocate(ofGetWindowWidth(), ofGetWindowHeight(), OF_IMAGE_COLOR);
model_image_portrait.allocate(portrait_pre_fbo_alpha.getWidth(), portrait_pre_fbo_alpha.getHeight(), OF_IMAGE_COLOR);
map_pixels.allocate(ofGetWindowWidth(), ofGetWindowHeight(), OF_PIXELS_RGB);
/* allocated ofImages in esp_images */
for(int i = 0; i < 4; i++){
@ -34,22 +41,33 @@ void ofApp::setup(){
ofEnableAlphaBlending();
/* load */
ofLoadImage(bayer, "images/bayer.png");
bayer.setTextureWrap(GL_REPEAT, GL_REPEAT);
shaders.load("shaders/dither");
esp_shader.load("shaders/espDepth");
vert_snap.load("shaders/vertex_snap");
p_depth.load("shaders/p_depth");
ORTCHAR_T* modelPath = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx";
ORTCHAR_T* modelPath_Small = "/home/cailean/Desktop/openframeworks/of_v0.12.0_linux64gcc6_release/apps/myApps/image-to-mesh/bin/data/models/depth_anything_v2_vits.onnx";
/* setup */
bullet.setup(nodes);
depth_onnx.Setup(modelPath, true, true);
depth_onnx_esp.Setup(modelPath, true, true);
depth_onnx.Setup(modelPath, false, true);
depth_onnx_esp.Setup(modelPath_Small, false, true);
depth_onnx_portrait.Setup(modelPath_Small, false, true);
depth_thread.setup(&model_image, &model_outptut_fbo, &depth_onnx);
depth_esp.setup(&model_image_esp, &model_esp_out_fbo, &depth_onnx_esp);
depth_portrait.setup(&model_image_portrait, &model_portrait_out_fbo, &depth_onnx_portrait);
/* camera settings for portrait */
portrait_camera.setFov(90);
portrait_camera.setPosition(0, 150, 480);
portrait_camera.removeAllInteractions();
portrait_camera.disableMouseInput();
ofLoadImage(bayer, "images/bayer.png");
bayer.setTextureWrap(GL_REPEAT, GL_REPEAT);
createNodes("data/json/embeddings.json");
@ -82,20 +100,28 @@ void ofApp::update(){
if(ofGetFrameNum() < 1){
depth_thread.start();
depth_esp.start();
depth_portrait.start();
}
/* write pixels to model input image */
map_fbo.readToPixels(map_pixels);
model_image.setFromPixels(map_pixels);
try{
map_fbo.readToPixels(map_pixels);
model_image.setFromPixels(map_pixels);
} catch (exception e) {
ofLogError() << "couldn't set pixels!";
}
try{
depth_thread.update();
depth_esp.update();
depth_portrait.update();
/* set output to fbo's */
depth_onnx.SetPixels(model_outptut_fbo);
depth_onnx_esp.SetPixels(model_esp_out_fbo);
depth_onnx_portrait.SetPixels(model_portrait_out_fbo);
} catch (exception e){
std::cout << "Model did not run" << std::endl;
@ -108,67 +134,124 @@ void ofApp::update(){
void ofApp::draw(){
ofPushStyle();
map_fbo.begin();
ofClear(ofColor::grey);
map_fbo_alpha.begin();
ofClear(0, 0, 0, 0);
bullet.draw();
map_fbo.end();
map_fbo_alpha.end();
ofPopStyle();
map_fbo.begin();
ofClear(0, 0, 0, 0);
map_fbo_alpha.draw(0, 0);
map_fbo.end();
ofTexture t = map_fbo.getTexture();
ofTexture d = model_outptut_fbo.getTexture();
//model_outptut_fbo.draw(ofGetWindowWidth() / 2, 0);
esp_comp_fbo.begin();
esp_shader.begin();
esp_shader.setUniformTexture("tex0", model_esp_out_fbo.getTexture(), 0);
model_esp_out_fbo.draw(0, 0);
esp_comp_fbo.end();
esp_shader.end();
portrait_fbo.begin();
ofBackground(0, 0, 0, 0);
drawPortraitZ();
portrait_fbo.end();
drawPortrait();
comp_fbo.begin();
ofClear(0, 0, 0, 0);
shaders.begin();
shaders.setUniformTexture("tex0", t, 0);
shaders.setUniformTexture("tex1", bayer, 1);
shaders.setUniform2f("resolution", ofGetWidth(), ofGetHeight());
shaders.setUniform1f("time", ofGetElapsedTimef());
shaders.setUniform1i("frame", ofGetFrameNum());
ofClear(0, 0, 0, 0);
map_fbo.draw(0,0);
model_outptut_fbo.draw(0, 0);
portrait_fbo.draw(0, 0);
map_fbo_alpha.draw(0,0);
shaders.end();
comp_fbo.end();
comp_fbo.draw(0,0);
model_outptut_fbo.draw(ofGetWindowWidth() / 2, 0);
}
void ofApp::drawPortrait(){
Node n = server->getChosenNode();
portrait_pre_fbo.begin();
float scale = min(
(float)portrait_pre_fbo.getWidth() / n.img.getWidth(),
(float)portrait_pre_fbo.getHeight() / n.img.getHeight()
);
float scaledWidth = n.img.getWidth() * scale;
float scaledHeight = n.img.getHeight() * scale;
float xPos = (portrait_pre_fbo.getWidth() - scaledWidth) / 2;
float yPos = (portrait_pre_fbo.getHeight() - scaledHeight) / 2;
n.img.draw(xPos, yPos, scaledWidth, scaledHeight);
portrait_pre_fbo.end();
portrait_pre_fbo_alpha.begin();
ofClear(0, 0, 0, 0);
n.img.draw(xPos, yPos, scaledWidth, scaledHeight);
portrait_pre_fbo_alpha.end();
ofPixels pix;
portrait_pre_fbo.readToPixels(pix);
model_image_portrait.setFromPixels(pix);
}
esp_comp_fbo.begin();
void ofApp::drawPortraitZ(){
ofTexture tex_color = portrait_pre_fbo_alpha.getTexture();
ofTexture tex_depth = model_portrait_out_fbo.getTexture();
esp_shader.begin();
esp_shader.setUniformTexture("tex0", model_esp_out_fbo.getTexture(), 0);
model_esp_out_fbo.draw(0, 0);
esp_comp_fbo.end();
esp_shader.end();
portrait_camera.begin();
ofEnableDepthTest();
esp_comp_fbo.draw(0, 0);
float time = ofGetElapsedTimef() / 10;
float p_noise_x = ofSignedNoise(time, time) * 20;
float p_noise_y = ofSignedNoise(time + 100, time + 100) * 20;
//server->print();
//shader_fbo.draw(0, 0);
// float planeScale = 0.75;
// int planeWidth = ofGetWidth() * planeScale;
// int planeHeight = ofGetHeight() * planeScale;
// int planeGridSize = 20;
// int planeColumns = planeWidth / planeGridSize;
// int planeRows = planeHeight / planeGridSize;
ofPushMatrix();
// plane.set(planeWidth, planeHeight, planeColumns, planeRows, OF_PRIMITIVE_TRIANGLES);
float planeScale = 1.0;
int planeWidth = portrait_pre_fbo_alpha.getWidth() * planeScale;
int planeHeight = portrait_pre_fbo_alpha.getHeight() * planeScale;
int planeGridSize = plane_size;
int planeColumns = planeWidth / planeGridSize;
int planeRows = planeHeight / planeGridSize;
// plane.mapTexCoordsFromTexture(t);
plane.set(planeWidth, planeHeight, planeColumns, planeRows, OF_PRIMITIVE_TRIANGLES);
plane.mapTexCoords(0, 0, planeWidth, planeHeight);
// t.bind();
// shaders.begin();
// ofPushMatrix();
// ofTranslate(ofGetWidth()/2, ofGetHeight()/2);
// plane.draw();
// ofPopMatrix();
// shaders.end();
// t.unbind();
p_depth.begin();
p_depth.setUniform1f("gridSize", g_size);
p_depth.setUniform2f("resolution", glm::vec2(portrait_fbo.getWidth(), portrait_fbo.getHeight()));
p_depth.setUniformTexture("tex0", tex_color, 0);
p_depth.setUniformTexture("tex1", tex_depth, 1);
ofFill();
ofTranslate(p_noise_x, p_noise_y, 0);
plane.draw();
p_depth.end();
//t.draw(ofGetWindowWidth() / 2, 0);
ofPopMatrix();
ofDisableDepthTest();
portrait_camera.end();
}
/* creates an fbo with four cropped images, to preprare for model input */
@ -372,7 +455,27 @@ void ofApp::queryKD(glm::vec3& _position, int k){
size_t index = kd_result[0].second;
std::cout << "camera positions: " << _position << ", nearest node: "<< nodes[index].tsne_position << " " << index << std::endl;
//std::cout << "camera positions: " << _position << ", nearest node: "<< nodes[index].tsne_position << " " << index << std::endl;
}
void ofApp::keyPressed(int key) {
switch(key) {
case OF_KEY_UP:
g_size += 10;
ofLog() << "Grid Size: " << g_size;
break;
case OF_KEY_DOWN:
g_size -= 10;
ofLog() << "Grid Size: " << g_size;
break;
case 'q':
plane_size += 10;
ofLog() << "Plane Size: " << plane_size;
break;
case 'w':
plane_size -=10;
ofLog() << "Plane Size: " << plane_size;
break;
}
}

17
src/ofApp.h

@ -25,9 +25,12 @@ class ofApp : public ofBaseApp{
void buildKDTree();
void queryKD(glm::vec3& _position, int k);
void buildMeshes();
void drawPortrait();
void drawPortraitZ();
void keyPressed(int key);
void exit();
ofEasyCam cam;
ofEasyCam portrait_camera;
ofImage img;
ofMesh mesh;
MeshGenerator mesh_generator;
@ -35,17 +38,24 @@ class ofApp : public ofBaseApp{
vector<ofImage> images;
ofFbo map_fbo;
ofFbo map_fbo_alpha;
ofFbo portrait_fbo;
ofFbo portrait_pre_fbo;
ofFbo portrait_pre_fbo_alpha;
ofFbo comp_fbo;
ofFbo model_outptut_fbo;
ofFbo model_esp_out_fbo;
ofFbo model_portrait_out_fbo;
ofFbo esp_comp_fbo;
ofFbo kd_out;
ofImage model_image_esp;
ofImage model_image_portrait;
ofShader shaders;
ofShader esp_shader;
ofShader vert_snap;
ofShader p_depth;
ofPlanePrimitive plane;
ofTexture bayer;
@ -55,8 +65,10 @@ class ofApp : public ofBaseApp{
/* onnx */
Onnx depth_onnx;
Onnx depth_onnx_esp;
Onnx depth_onnx_portrait;
ModelThread depth_thread;
ModelThread depth_esp;
ModelThread depth_portrait;
ofImage model_image;
ofPixels map_pixels;
@ -87,4 +99,7 @@ class ofApp : public ofBaseApp{
std::unique_ptr<KDTree> kd_tree;
std::vector<pointIndex> kd_result;
float g_size = 10;
float plane_size = 50;
};

Loading…
Cancel
Save