Browse Source

right up until shader!

tsns-map
cailean 3 months ago
parent
commit
5104e7af27
  1. 132
      src/Map.cpp
  2. 29
      src/Map.h
  3. 14
      src/Onnx.cpp
  4. 4
      src/Onnx.h
  5. 4
      src/Player.cpp
  6. 20
      src/ofApp.cpp
  7. 2
      src/ofApp.h

132
src/Map.cpp

@ -1,4 +1,5 @@
#include "Map.h"
#include "algorithm"
Map::Map(){
@ -6,6 +7,7 @@ Map::Map(){
void Map::Setup(){
ofDisableArbTex();
scale = 10000;
isDebug = true;
json_embeddings = ofLoadJson(jsonPath);
@ -17,21 +19,107 @@ void Map::Setup(){
SetupTSNE();
}
mapFbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA);
mapFbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGB);
fboImage.allocate(ofGetWindowWidth(), ofGetWindowHeight(), OF_IMAGE_COLOR);
Setup3D();
SetupNodes();
}
void Map::Update(){
time = ofGetElapsedTimef();
for(auto& n : nodes){
glm::vec2 offset = n.amplitude * sin(n.speed * time);
n.offset = glm::vec3(offset.x, offset.y, 0);
}
}
void Map::Draw(){
mapFbo.begin();
cam.begin();
ofClear(0, 0, 0, 1);
for (auto& n :nodes){
n.texture.getTexture().bind();
ofPushMatrix();
ofFill();
n.geom.setPosition(n.position + n.offset);
n.geom.draw();
ofPopMatrix();
n.texture.getTexture().unbind();
}
cam.end();
mapFbo.end();
mapFbo.readToPixels(fboPixels);
fboImage.setFromPixels(fboPixels);
//mapFbo.draw(0, 0);
}
/*
Setup texture for each node
*/
void Map::SetupNodes(){
std::cout << "Setting up nodes.." << std::endl;
for (auto& node : nodes){
ofImage img;
ofPlanePrimitive plane;
img.load("data/" + node.image_path);
int imageW = img.getWidth();
int imageH = img.getHeight();
int maxWidth = 100;
int maxHeight = 100;
float aspectRatio = (float)imageW / (float)imageH;
// Determine plane dimensions based on aspect ratio and constraints
float planeW, planeH;
if (aspectRatio > (float)maxWidth / maxHeight) {
// Image is wider relative to the max constraints
planeW = maxWidth;
planeH = maxWidth / aspectRatio;
} else {
// Image is taller relative to the max constraints
planeH = maxHeight;
planeW = maxHeight * aspectRatio;
}
// Ensure that dimensions do not exceed constraints
if (planeH > maxHeight) {
planeH = maxHeight;
planeW = maxHeight * aspectRatio;
}
if (planeW > maxWidth) {
planeW = maxWidth;
planeH = maxWidth / aspectRatio;
}
plane.set(planeW, planeH);
plane.setPosition(node.position);
plane.setResolution(2, 2);
plane.mapTexCoordsFromTexture(img.getTexture());
img.getTexture().setTextureWrap(GL_REPEAT, GL_REPEAT);
plane.setScale(1);
node.geom = plane;
node.texture = img;
node.speed = glm::vec2(ofRandom(0.1, 2), ofRandom(0.1, 2));
node.amplitude = glm::vec2(ofRandom(1, 5), ofRandom(1, 5));
node.phase = glm::vec2(ofRandom(0, TWO_PI), ofRandom(0, TWO_PI));
}
std::cout << "Finished setting up nodes!" << std::endl;
}
/*
Setup TSNE - reads JSON, converts to points, creates nodes & hashmap
*/
void Map::SetupTSNE(){
for (const auto& entry: json_embeddings) {
if (entry.contains("vector") && entry["vector"].is_array()) {
@ -58,13 +146,47 @@ void Map::SetupTSNE(){
}
std::cout << nodes.size() << std::endl;
std::cout << embeddings.size() << std::endl;
points = tsne.run(embeddings, dims, perplexity, theta, normalise, runManually);
for (size_t i = 0; i < points.size(); i++){
const auto& vec = points[i];
auto& n = nodes[i];
n.position = glm::vec3(vec[0], vec[1], vec[2]);
n.position = ( (glm::vec3(vec[0], vec[1], vec[2]) * scale) - (scale / 2) );
node_hash_map[n.image_path] = &n;
}
SortNodes();
}
/*
Setup 3D environment
*/
void Map::Setup3D(){
cam.setFov(20);
cam.removeAllInteractions();
cam.addInteraction(ofEasyCam::TRANSFORM_TRANSLATE_XY, OF_MOUSE_BUTTON_LEFT);
cam.addInteraction(ofEasyCam::TRANSFORM_TRANSLATE_Z, OF_MOUSE_BUTTON_MIDDLE);
}
/*
Query hashmap
*/
void Map::SearchMap(){
std::string t = "dataset/20230601_webexclusi-irelandsey_cl11536538/segmented_images/000000000_0.png";
if(node_hash_map.find(t) != node_hash_map.end()){
Node* n = node_hash_map[t];
std::cout << n->foldername << std::endl;
}
}
/*
Sort nodes by z position, for draw calls (lowest -> highest)
*/
void Map::SortNodes(){
std::cout << "Sorting Nodes.." << std::endl;
std::sort(nodes.begin(), nodes.end(), [](const Node& a, const Node&b){
return a.position.z < b.position.z;
});
std::cout << "Finished Sorting!" << std::endl;
}

29
src/Map.h

@ -9,7 +9,9 @@ struct Node {
glm::vec2 speed;
glm::vec2 amplitude;
glm::vec2 phase;
glm::vec3 offset;
ofImage texture;
ofPlanePrimitive geom;
glm::vec3 position;
std::string image_path;
std::string foldername;
@ -20,31 +22,30 @@ struct Node {
class Map {
public:
/*
Constructor
*/
Map();
/*
Methods
*/
Map();
void Setup();
void Update();
void Draw();
void SetupNodes();
void SetupTSNE();
void Setup3D();
void SearchMap();
void SortNodes();
/*
Variables
*/
bool isDebug;
float time;
ofFbo mapFbo;
vector<ofPlanePrimitive> geoms;
vector<Node> nodes;
float scale;
ofxTSNE tsne;
vector<vector<double>> points;
@ -57,9 +58,19 @@ class Map {
ofJson json_embeddings;
std::string jsonPath = "data/json/embeddings.json";
private:
std::unordered_map<std::string, Node*> node_hash_map;
/*
3D Variables
*/
ofLight light;
ofEasyCam cam;
float zoom;
ofImage fboImage;
ofPixels fboPixels;
private:
};
#endif

14
src/Onnx.cpp

@ -1,4 +1,5 @@
#include "Onnx.h"
#include <cmath>
// Setups the model. CUDA is enabled by default
void Onnx::Setup(ORTCHAR_T* modelPath, bool isLog, bool useCuda){
@ -75,14 +76,10 @@ std::vector<Ort::Value> Onnx::Run(ofImage &img){
std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char),
[&](const std::string& str) { return str.c_str(); });
std::cout << "Running model... ";
try {
auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(),
input_names_char.size(), output_names_char.data(), output_names_char.size());
std::cout << "Done!" << std::endl;
if (timeStamp) {
auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> elapsed = end - start;
@ -146,14 +143,10 @@ std::vector<Ort::Value> Onnx::RunBatch(std::vector<ofImage>& images){
std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char),
[&](const std::string& str) { return str.c_str(); });
std::cout << "Running model... ";
try {
auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(),
input_names_char.size(), output_names_char.data(), output_names_char.size());
std::cout << "Done!" << std::endl;
if (timeStamp) {
auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> elapsed = end - start;
@ -235,7 +228,7 @@ void Onnx::Normalize(float* data, size_t size, float min_value, float max_value)
}
// Coverts the output tensor data to a texture of a given ofFbo.
void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo){
void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo, ofShader& shader){
// Convert data into opencv mat
cv::Mat inputMat(height, width, CV_32FC1, const_cast<float*>(data));
@ -261,8 +254,11 @@ void Onnx::DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo)
// Update FBO with new pixels
fbo.begin();
shader.begin();
ofTexture& texture = fbo.getTexture();
texture.loadData(pixels);
shader.setUniformTexture("tex", texture, 0);
shader.end();
fbo.end();

4
src/Onnx.h

@ -31,9 +31,9 @@
float ReduceMin(const float* data, size_t size);
float ReduceMax(const float* data, size_t size);
void Normalize(float* data, size_t size, float min_value, float max_value);
void DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo);
void DataToFbo(const float* data, size_t width, size_t height, ofFbo& fbo, ofShader& shader);
void Softmax(float* data, size_t size);
bool timeStamp = true;
bool timeStamp = false;
bool log = false;
protected:

4
src/Player.cpp

@ -44,7 +44,9 @@ ofPixels Player::GetVideoPixels(){
void Player::SetVideo(std::string path, ofFbo &fbo){
videoPlayer.load(path);
videoPlayer.setFrame(800);
fbo.allocate(videoPlayer.getWidth(), videoPlayer.getHeight(), GL_RGB);
//fbo.allocate(videoPlayer.getWidth(), videoPlayer.getHeight(), GL_RGB);
// Just setting the video dims here for the tsne map!
fbo.allocate(1600, 800, GL_RGB);
}
// Sets a random frame in the active video

20
src/ofApp.cpp

@ -21,6 +21,8 @@ void ofApp::setup(){
yolo.Setup(modelPath2, false, true);
depth.Setup(modelPath, false, true);
emotion.Setup(modelPath3, false, true);
depthToColourShader.load("data/shader/rampShader.vert", "data/shader/rampShader.frag");
}
@ -35,13 +37,21 @@ void ofApp::update(){
/* Clear detetced face list */
detected_faces.clear();
/* tsnemap fbo - first run to allocate fbo */
map.Update();
if(firstRun){
map.Draw();
}
/* Setup model input using ofImage */
player.Update(img);
img.setFromPixels(player.GetVideoPixels());
/* Run Models */
try{
auto output_tensors = depth.Run(img);
auto output_tensors = depth.Run(map.fboImage);
float* output_ptr = output_tensors.front().GetTensorMutableData<float>();
size_t num_elements = output_tensors.front().GetTensorTypeAndShapeInfo().GetElementCount();
@ -50,9 +60,9 @@ void ofApp::update(){
depth.Normalize(output_ptr, num_elements, min_value, max_value);
depth.DataToFbo(output_ptr, 518, 518, fbo);
depth.DataToFbo(output_ptr, 518, 518, fbo, depthToColourShader);
auto output_tensors_face = yolo.Run(img);
auto output_tensors_face = yolo.Run(map.fboImage);
auto output_faces = output_tensors_face.front().GetTensorTypeAndShapeInfo().GetShape();
@ -100,13 +110,15 @@ void ofApp::update(){
//--------------------------------------------------------------
void ofApp::draw(){
map.Draw();
fbo.draw(0, 0);
if(!firstRun){
faceDetector.DrawBox(detected_faces);
faceDetector.DrawCenter(detected_faces);
}
// emoteImage.draw(640, 0);
// for(auto& face : detected_faces){
// ofDrawBitmapString(std::to_string(face.box.emotional_state.emotions[0]), 700, 300);

2
src/ofApp.h

@ -49,4 +49,6 @@ class ofApp : public ofBaseApp{
std::vector<types::BoxfWithLandmarks> detected_faces;
Map map;
ofShader depthToColourShader;
};

Loading…
Cancel
Save