#pragma once #include "ofMain.h" #include "utils/MeshGenerator.h" #include "Bullet.h" #include "vp/VP.h" #include #include "onnx/Onnx.h" #include "onnx/Yolo.h" #include "onnx/ModelThread.h" #include "network/Request.h" #include "network/Server.h" #include "ofxTSNE.h" #include "utils/KDTree.hpp" #include "utils/QuadSource.h" #include "ofxPiMapper.h" class ofApp : public ofBaseApp{ public: void setup(); void update(); void draw(); void getNearestImages(); void createNodes(std::string json_path); std::vector> createDoubleVectorFromNodes(const std::vector& nodes); void buildKDTree(); void queryKD(glm::vec3& _position, int k); void buildMeshes(); void drawPortrait(); void drawPortraitZ(); ofMesh createCustomPlane(float width, float height, int numX, int numY); void keyPressed(int key); void keyReleased(int key); void mousePressed(int x, int y, int button); void mouseReleased(int x, int y, int button); void mouseDragged(int x, int y, int button); void onTSNEUpdateComplete(); void exit(); ofEasyCam portrait_camera; ofImage img; ofMesh mesh; MeshGenerator mesh_generator; //Bullet bullet; vector images; ofFbo map_fbo; ofFbo map_fbo_alpha; ofFbo map_fbo_post; ofFbo portrait_fbo; ofFbo portrait_pre_fbo; ofFbo portrait_pre_fbo_alpha; ofFbo portrait_cropped; ofFbo portrait_cropped_alpha; ofFbo comp_fbo; ofFbo model_outptut_fbo; ofFbo model_esp_out_fbo; ofFbo model_portrait_out_fbo; ofFbo esp_comp_fbo; ofFbo kd_out; ofImage model_image_esp; ofImage model_image_portrait; ofImage model_image_portrait_cropped; ofShader shaders; ofShader esp_shader; ofShader vert_snap; ofShader p_depth; ofShader map_depth; ofPlanePrimitive plane; ofTexture bayer; VP vp_tree; /* onnx */ Onnx depth_onnx; Onnx depth_onnx_esp; Onnx depth_onnx_portrait; Onnx yolo_onnx; ModelThread depth_thread; ModelThread depth_esp; ModelThread depth_portrait; ModelThread yolo; Yolo face_detector; ofImage model_image; ofImage yolo_t_image; ofxCvColorImage emote_image; ofPixels map_pixels; std::vector detected_faces; std::vector croppedFaces; int emotionImageMaxBatchSize = 5; /* the current embedding */ Embedding embed; /* server */ std::unique_ptr server; /* esp_image fbos */ vector esp_images; ofPixels esp_comp_pixels; float last_updated_time; float last_updated_time_esp; /* nodes */ ofJson json; vector nodes; ofxTSNE tsne; int dims = 2; float perplexity = 20; float theta = 0.2; bool normalise = true; bool runManually = true; float tsne_scale = 500; vector>> point_iterations; /* kd tree */ std::unique_ptr kd_tree; std::vector kd_result; /* settings */ float g_size = 5; float plane_size = 70; float past_plane_size; /* pi mapper */ ofxPiMapper mapper; QuadSource projection_src; int map_h = 960; int map_w = 1920; /* camera positions */ vector cam_positions; float cam_pos_idx = 0; Node last_chosen_node; bool portrait_needs_update; ofMesh custom_mesh; bool depth_ready = false; bool mesh_ready = false; float last_process_time = 0; bool waiting_for_yolo; ofImage alpha_demo; ofPixels alpha_pixels; /* n-neighbour */ vector nn_nodes; Bullet bullet; /* simulation update timer */ const float TNSE_DURATION = 360.0f; float tsne_start_time; int tsne_iter_idx = 0; private: std::thread tsne_thread; std::atomic tsne_updating{false}; std::mutex nodes_mutex; bool tsne_update_complete = false; void updateTSNEPositions(vector& _nodes); void updateTSNEPositionsThreaded(vector& _nodes); };