30 changed files with 1197 additions and 38 deletions
|
After Width: | Height: | Size: 3.2 KiB |
@ -0,0 +1,41 @@ |
|||
OF_GLSL_SHADER_HEADER |
|||
precision mediump float; |
|||
|
|||
uniform sampler2DRect tex0; // Main texture |
|||
uniform sampler2D tex1; // Bayer texture |
|||
uniform vec2 resolution; |
|||
uniform float time; |
|||
uniform int frame; |
|||
|
|||
in vec2 varyingtexcoord; |
|||
out vec4 fragColor; |
|||
|
|||
const float SIZE = 256.0; |
|||
|
|||
const mat4 ditherMatrix = mat4( |
|||
1.0/17.0, 9.0/17.0, 3.0/17.0, 11.0/17.0, |
|||
13.0/17.0, 5.0/17.0, 15.0/17.0, 7.0/17.0, |
|||
4.0/17.0, 12.0/17.0, 2.0/17.0, 10.0/17.0, |
|||
16.0/17.0, 8.0/17.0, 14.0/17.0, 6.0/17.0 |
|||
); |
|||
|
|||
void main() { |
|||
vec2 uv = varyingtexcoord / resolution; |
|||
vec3 t = texture(tex0, varyingtexcoord).rgb; |
|||
|
|||
vec2 fragCoord = gl_FragCoord.xy; |
|||
|
|||
// color to display, use floating point precision here |
|||
vec3 col = t; |
|||
|
|||
// Sample the Bayer 8x8 texture for the dither pattern |
|||
float ditherValue = texture(tex1, fragCoord / 8.0).r; |
|||
|
|||
// Apply dithering with increased effect |
|||
col += (ditherValue - 0.5) / 2.0 * 2.0; |
|||
|
|||
// Quantize the color |
|||
col = (floor(col * 8.0) + 0.5) / 8.0; |
|||
|
|||
fragColor = vec4(col, 1.0); |
|||
} |
|||
@ -0,0 +1,16 @@ |
|||
OF_GLSL_SHADER_HEADER |
|||
|
|||
uniform mat4 modelViewMatrix; |
|||
uniform mat4 projectionMatrix; |
|||
uniform mat4 textureMatrix; |
|||
uniform mat4 modelViewProjectionMatrix; |
|||
|
|||
in vec4 position; |
|||
in vec2 texcoord; |
|||
|
|||
out vec2 varyingtexcoord; |
|||
void main() |
|||
{ |
|||
varyingtexcoord = texcoord; |
|||
gl_Position = modelViewProjectionMatrix * position; |
|||
} |
|||
@ -0,0 +1,25 @@ |
|||
#version 150 |
|||
|
|||
uniform sampler2D tex0; |
|||
|
|||
in vec2 tex_c; |
|||
|
|||
out vec4 outputColor; |
|||
|
|||
float hash13(vec3 p3) { |
|||
p3 = fract(p3 * .1031); |
|||
p3 += dot(p3, p3.yzx + 19.19); |
|||
return fract((p3.x + p3.y) * p3.z); |
|||
} |
|||
|
|||
void main() |
|||
{ |
|||
vec4 texColor = texture(tex0, tex_c); |
|||
|
|||
// Discard fragments with alpha below a threshold |
|||
if (texColor.a < 0.1) { |
|||
discard; |
|||
} |
|||
|
|||
outputColor = texColor; |
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
#version 150 |
|||
|
|||
uniform mat4 modelViewProjectionMatrix; |
|||
uniform float gridSize; |
|||
|
|||
in vec2 texcoord; |
|||
in vec4 position; |
|||
|
|||
out vec2 tex_c; |
|||
|
|||
void main() |
|||
{ |
|||
tex_c = texcoord; |
|||
|
|||
vec2 resolution = vec2(1280.0, 960.0); |
|||
|
|||
// Transform the vertex position to clip space |
|||
vec4 clipPos = modelViewProjectionMatrix * position; |
|||
|
|||
// Perform perspective division |
|||
vec3 ndcPos = clipPos.xyz / clipPos.w; |
|||
|
|||
// Convert to screen space |
|||
vec2 screenPos = (ndcPos.xy + 1.0) * 0.5 * resolution; |
|||
|
|||
// Snap to grid |
|||
screenPos = floor(screenPos / gridSize) * gridSize; |
|||
|
|||
// Convert back to NDC space |
|||
ndcPos.xy = (screenPos / resolution) * 2.0 - 1.0; |
|||
|
|||
// Reconstruct the clip space position |
|||
clipPos = vec4(ndcPos * clipPos.w, clipPos.w); |
|||
|
|||
gl_Position = clipPos; |
|||
} |
|||
Binary file not shown.
Binary file not shown.
@ -0,0 +1,40 @@ |
|||
#include "Request.h" |
|||
|
|||
/* setup http server */ |
|||
void Request::setup(std::string ip, int port, std::string page){ |
|||
std::cout << "Initialising HTTP Server" << std::endl; |
|||
|
|||
req.method = ofHttpRequest::POST; |
|||
req.url = "http://" + ip + ":" + ofToString(port) + "/" + page; |
|||
req.headers["Content-Type"] = "application/json"; |
|||
} |
|||
|
|||
/* send a request to vp_server & return frame/video/folder */ |
|||
VPResp Request::query(Embedding& in){ |
|||
VPResp vp_resp; |
|||
try { |
|||
req.body = "{\"vector\": [" + |
|||
ofToString(in.emotions["angry"]) + "," + |
|||
ofToString(in.emotions["disgust"]) + "," + |
|||
ofToString(in.emotions["fear"]) + "," + |
|||
ofToString(in.emotions["happy"]) + "," + |
|||
ofToString(in.emotions["sad"]) + "," + |
|||
ofToString(in.emotions["surprise"]) + "," + |
|||
ofToString(in.emotions["neutral"]) + "]}"; |
|||
|
|||
auto resp = http.handleRequest(req); |
|||
json_resp = ofJson::parse(resp.data.getText()); |
|||
vp_resp.folder = json_resp["folder"]; |
|||
vp_resp.image = json_resp["image"]; |
|||
vp_resp.video = json_resp["video"]; |
|||
vp_resp.frame = json_resp["frame"]; |
|||
vp_resp.lost = json_resp["lost"]; |
|||
|
|||
past_vp_resp = vp_resp; |
|||
|
|||
return vp_resp; |
|||
} catch (exception e) { |
|||
// Some issue happening here when plugging in controllers, or when they initially connect
|
|||
return past_vp_resp; |
|||
} |
|||
} |
|||
@ -0,0 +1,44 @@ |
|||
#pragma once |
|||
|
|||
#include "ofMain.h" |
|||
#include <string> |
|||
#include <unordered_map> |
|||
|
|||
struct Embedding { |
|||
std::unordered_map<std::string, float> emotions; |
|||
|
|||
Embedding() { |
|||
// Initialize with default values
|
|||
emotions["angry"] = 0.0f; |
|||
emotions["disgust"] = 0.0f; |
|||
emotions["fear"] = 0.0f; |
|||
emotions["happy"] = 0.0f; |
|||
emotions["sad"] = 0.0f; |
|||
emotions["surprise"] = 0.0f; |
|||
emotions["neutral"] = 0.0f; |
|||
} |
|||
|
|||
bool operator!=(const Embedding &other) const { |
|||
return emotions != other.emotions; |
|||
} |
|||
}; |
|||
|
|||
/* Vantage point query structure */ |
|||
struct VPResp{ |
|||
std::string folder; |
|||
std::string video; |
|||
std::string image; |
|||
std::string frame; |
|||
int lost; |
|||
}; |
|||
|
|||
class Request{ |
|||
public: |
|||
void setup(std::string ip, int port, std::string page); |
|||
VPResp query(Embedding& in); |
|||
|
|||
ofHttpRequest req; |
|||
ofURLFileLoader http; |
|||
ofJson json_resp; |
|||
VPResp past_vp_resp; |
|||
}; |
|||
@ -0,0 +1,152 @@ |
|||
#include "Server.h" |
|||
|
|||
void Server::start(){ |
|||
std::cout << "Initialising TCP sever" << std::endl; |
|||
server.setup(port); |
|||
osc_sender.setup(OSC_HOST, OSC_PORT); |
|||
http.setup(http_ip, http_port, http_page); |
|||
is_active = true; |
|||
previous_embedding = embedding; |
|||
last_change_time = std::chrono::steady_clock::now(); |
|||
} |
|||
|
|||
void Server::update(){ |
|||
|
|||
for ( int i = 0; i < server.getLastID(); i++){ |
|||
if (server.isClientConnected(i)) { |
|||
const int buffer_size = 8; |
|||
char buffer[buffer_size]; |
|||
int bytes_recieved = server.receiveRawBytes(i, buffer, buffer_size); |
|||
|
|||
if (bytes_recieved == buffer_size){ |
|||
float value; |
|||
int id; |
|||
memcpy(&value, buffer, sizeof(float)); |
|||
memcpy(&id, buffer + sizeof(float), sizeof(int)); |
|||
|
|||
std::string ip_address = server.getClientIP(i); |
|||
|
|||
addOrUpdateClient(id, value, ip_address); |
|||
} |
|||
} |
|||
} |
|||
|
|||
updateEmbedding(); |
|||
checkActivity(); |
|||
sendOSCMessage(); |
|||
|
|||
if(debug){ |
|||
printClients(); |
|||
} |
|||
} |
|||
|
|||
void Server::addOrUpdateClient(int client_id, float value, const std::string& ip_address){ |
|||
ClientInfo client; |
|||
|
|||
client.ip_address = ip_address; |
|||
client.value = value; |
|||
|
|||
clients[client_id] = client; |
|||
} |
|||
|
|||
void Server::updateEmbedding() { |
|||
std::vector<std::string> emotionNames = { |
|||
"angry", "disgust", "fear", "happy", "sad", "surprise", "neutral" |
|||
}; |
|||
|
|||
for (const auto& c : clients) { |
|||
const ClientInfo& info = c.second; |
|||
float val = std::round(info.value * 100.0f) / 100.0f; |
|||
|
|||
if (c.first >= 0 && c.first < emotionNames.size()) { |
|||
embedding.emotions[emotionNames[c.first]] = val; |
|||
} |
|||
|
|||
// Special case for neutral (index 6)
|
|||
if (c.first == 6) { |
|||
embedding.emotions["fear"] = ofRandom(0.1, 0.6); |
|||
embedding.emotions["angry"] = ofRandom(0.01, 0.99); |
|||
embedding.emotions["happy"] = ofRandom(0.01, 0.99); |
|||
} |
|||
} |
|||
} |
|||
|
|||
void Server::printClients(){ |
|||
for( const auto& c : clients){ |
|||
int id = c.first; |
|||
const ClientInfo& info = c.second; |
|||
std::cout << "id: " << id |
|||
<< ", value: " << info.value |
|||
<< ", IP: " << info.ip_address << std::endl; |
|||
} |
|||
|
|||
std::cout << is_active << std::endl; |
|||
} |
|||
|
|||
/* check if the controllers are in use */ |
|||
void Server::checkActivity(){ |
|||
if (previous_embedding.emotions["neutral"] != embedding.emotions["neutral"]) { // Check if embedding has changed
|
|||
last_change_time = std::chrono::steady_clock::now(); // Reset the timer if there is a change
|
|||
previous_embedding = embedding; // Update the previous embedding to the current one
|
|||
is_active = true; |
|||
sendHttpRequest(); |
|||
} else { |
|||
// Calculate the time since the last change
|
|||
auto now = std::chrono::steady_clock::now(); |
|||
auto duration = std::chrono::duration_cast<std::chrono::seconds>(now - last_change_time).count(); |
|||
|
|||
if (duration >= 2) { |
|||
is_active = false; |
|||
} |
|||
} |
|||
} |
|||
|
|||
/* send osc msg, check if audio file exists and it is different to the past audiofile */ |
|||
void Server::sendOSCMessage(){ |
|||
std::vector<ofxOscMessage> messages; |
|||
|
|||
ofxOscMessage me_0; |
|||
ofxOscMessage me_1; |
|||
ofxOscMessage me_2; |
|||
ofxOscMessage me_3; |
|||
ofxOscMessage me_file; |
|||
|
|||
std::string audio_file = vp_resp.folder; |
|||
|
|||
// Check if file exists in a given dir
|
|||
ofFile file("/home/cailean/Desktop/rave/all_wav_files/" + audio_file + ".wav"); |
|||
|
|||
if(!is_active && (audio_file != past_audio_file) && file.exists()){ |
|||
me_file.setAddress("/emote/filename"); |
|||
me_file.addStringArg(audio_file + ".wav"); |
|||
messages.push_back(me_file); |
|||
past_audio_file = audio_file; |
|||
} |
|||
|
|||
me_0.setAddress("/emote/0"); |
|||
me_0.addFloatArg(embedding.emotions["neutral"]); |
|||
messages.push_back(me_0); |
|||
me_1.setAddress("/emote/1"); |
|||
me_1.addFloatArg(embedding.emotions["neutral"]); |
|||
messages.push_back(me_1); |
|||
me_2.setAddress("/emote/2"); |
|||
me_2.addFloatArg(embedding.emotions["neutral"]); |
|||
messages.push_back(me_2); |
|||
me_3.setAddress("/emote/3"); |
|||
me_3.addFloatArg(embedding.emotions["neutral"]); |
|||
messages.push_back(me_3); |
|||
|
|||
for (auto& msg : messages){ |
|||
osc_sender.sendMessage(msg, false); |
|||
} |
|||
|
|||
} |
|||
|
|||
/* sends request to http server if is_active = true */ |
|||
void Server::sendHttpRequest(){ |
|||
vp_resp = http.query(embedding); |
|||
} |
|||
|
|||
std::vector<std::vector<double>> Server::generateRandomVectors(int count, int dimension){ |
|||
return tree.generateRandomVectors(count, dimension); |
|||
} |
|||
@ -0,0 +1,58 @@ |
|||
#pragma once |
|||
|
|||
#include "ofMain.h" |
|||
#include "ofxNetwork.h" |
|||
#include "Request.h" |
|||
#include <unordered_map> |
|||
#include <chrono> |
|||
#include "ofxOsc.h" |
|||
#include "../vp/VP.h" |
|||
|
|||
#define OSC_HOST "127.0.0.1" |
|||
#define OSC_PORT 9002 |
|||
|
|||
struct ClientInfo { |
|||
float value; |
|||
std::string ip_address; |
|||
}; |
|||
|
|||
class Server{ |
|||
public: |
|||
Server(int _port, Embedding _embedding, VP _tree, bool debug, std::string _http_ip, int _http_port, std::string _http_page) |
|||
: port(_port), embedding(_embedding), tree(_tree), debug(debug), http_ip(_http_ip), http_port(_http_port), http_page(_http_page) {} |
|||
|
|||
void start(); |
|||
void update(); |
|||
void addOrUpdateClient(int client_id, float value, const std::string& ip_address); |
|||
void printClients(); |
|||
void updateEmbedding(); |
|||
void checkActivity(); |
|||
void sendHttpRequest(); |
|||
void sendOSCMessage(); |
|||
std::vector<std::vector<double>> generateRandomVectors(int count, int dimension); |
|||
|
|||
|
|||
|
|||
int port; |
|||
ofxTCPServer server; |
|||
std::unordered_map<int, ClientInfo> clients; |
|||
bool debug; |
|||
bool is_active; |
|||
Embedding embedding; |
|||
|
|||
Request http; |
|||
std::string http_ip; |
|||
int http_port; |
|||
std::string http_page; |
|||
VPResp vp_resp; |
|||
|
|||
private: |
|||
Embedding previous_embedding; |
|||
std::chrono::time_point<std::chrono::steady_clock> last_change_time; |
|||
|
|||
ofxOscSender osc_sender; |
|||
std::string past_audio_file; |
|||
|
|||
/* vp tree */ |
|||
VP tree; |
|||
}; |
|||
@ -0,0 +1,99 @@ |
|||
#include "ofMain.h" |
|||
#include "Onnx.h" |
|||
#include "Yolo.h" |
|||
|
|||
class ModelThread : public ofThread |
|||
{ |
|||
public: |
|||
ofImage* img; |
|||
ofFbo* fbo; |
|||
Onnx* model; |
|||
Yolo* yolo; |
|||
std::vector<types::BoxfWithLandmarks>* detected_faces; |
|||
std::string model_type; |
|||
|
|||
// emotional recognition model
|
|||
std::vector<ofImage>* croppedFaces; |
|||
float* emotional_data; |
|||
|
|||
|
|||
~ModelThread(){ |
|||
stop(); |
|||
waitForThread(false); |
|||
} |
|||
|
|||
void setup(ofImage* _img, ofFbo* _fbo, Onnx* _model){ |
|||
std::lock_guard<std::mutex> lock(mutex); |
|||
this->img = _img; |
|||
this->fbo = _fbo; |
|||
this->model = _model; |
|||
this->model_type = "depth"; |
|||
} |
|||
|
|||
void setupYolo(ofImage* _img, std::vector<types::BoxfWithLandmarks>* _detected_faces, Onnx* _model, Yolo* _yolo){ |
|||
std::lock_guard<std::mutex> lock(mutex); |
|||
this->img = _img; |
|||
this->detected_faces = _detected_faces; |
|||
this->model_type = "yolo"; |
|||
this->model = _model; |
|||
this->yolo = _yolo; |
|||
} |
|||
|
|||
void start(){ |
|||
startThread(); |
|||
} |
|||
|
|||
void stop(){ |
|||
stopThread(); |
|||
condition.notify_all(); |
|||
} |
|||
|
|||
void threadedFunction(){ |
|||
while(isThreadRunning()){ |
|||
if(model_type == "depth"){ |
|||
std::unique_lock<std::mutex> lock(mutex); |
|||
inferDepthImage(fbo, img, model); |
|||
condition.wait(lock); |
|||
} else if(model_type == "yolo") { |
|||
std::unique_lock<std::mutex> lock(mutex); |
|||
inferYolo(); |
|||
condition.wait(lock); |
|||
} |
|||
|
|||
} |
|||
} |
|||
|
|||
void update(){ |
|||
std::lock_guard<std::mutex> lock(mutex); |
|||
condition.notify_one(); |
|||
} |
|||
|
|||
void inferYolo(){ |
|||
auto output_tensors_face = model->Run(*img); |
|||
|
|||
auto output_faces = output_tensors_face.front().GetTensorTypeAndShapeInfo().GetShape(); |
|||
|
|||
unsigned int num_anchors = output_faces[1]; // Number of anchors
|
|||
|
|||
float* output_face_ptr = output_tensors_face.front().GetTensorMutableData<float>(); |
|||
|
|||
yolo->ParseOutput(output_face_ptr, *detected_faces, num_anchors); |
|||
} |
|||
|
|||
void inferDepthImage(ofFbo* fbo, ofImage* img, Onnx* model){ |
|||
auto output_tensors = model->Run(*img); |
|||
float* output_ptr = output_tensors.front().GetTensorMutableData<float>(); |
|||
size_t num_elements = output_tensors.front().GetTensorTypeAndShapeInfo().GetElementCount(); |
|||
|
|||
float min_value = model->ReduceMin(output_ptr, num_elements); |
|||
float max_value = model->ReduceMax(output_ptr, num_elements); |
|||
|
|||
model->Normalize(output_ptr, num_elements, min_value, max_value); |
|||
|
|||
model->DataToFbo(output_ptr, 518, 518, *fbo); |
|||
} |
|||
|
|||
|
|||
protected: |
|||
std::condition_variable condition; |
|||
}; |
|||
@ -0,0 +1,272 @@ |
|||
#include "Onnx.h" |
|||
#include <cmath> |
|||
|
|||
// Setups the model. CUDA is enabled by default
|
|||
void Onnx::Setup(ORTCHAR_T* modelPath, bool isLog, bool useCuda){ |
|||
session_options.SetIntraOpNumThreads(1); |
|||
session_options.SetIntraOpNumThreads(1); |
|||
session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); |
|||
log = isLog; |
|||
|
|||
// cuda setup
|
|||
if(useCuda){ |
|||
OrtCUDAProviderOptions opts; |
|||
opts.device_id = 0; |
|||
opts.cudnn_conv_algo_search = OrtCudnnConvAlgoSearchExhaustive; |
|||
opts.do_copy_in_default_stream = 0; |
|||
opts.arena_extend_strategy = 0; |
|||
session_options.AppendExecutionProvider_CUDA(opts); |
|||
} |
|||
|
|||
ort_session = std::make_shared<Ort::Session>(ort_env, modelPath, session_options); |
|||
|
|||
Ort::AllocatorWithDefaultOptions allocator; |
|||
for (std::size_t i = 0; i < ort_session->GetInputCount(); i++) { |
|||
input_node_names.emplace_back(ort_session->GetInputNameAllocated(i, allocator).get()); |
|||
input_node_dims = ort_session->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); |
|||
if(log) |
|||
std::cout << "\t" << input_node_names.at(i) << " : " << PrintShape(input_node_dims) << std::endl; |
|||
} |
|||
|
|||
// some models might have negative shape values to indicate dynamic shape, e.g., for variable batch size.
|
|||
for (auto& s : input_node_dims) { |
|||
if (s < 0) { |
|||
s = 1; |
|||
if(log) |
|||
std::cout << "transfromed!" << std::endl; |
|||
} |
|||
} |
|||
|
|||
|
|||
|
|||
if(log) |
|||
std::cout << "Output Node Name/Shape (" << output_node_names.size() << "):" << std::endl; |
|||
for (std::size_t i = 0; i < ort_session->GetOutputCount(); i++) { |
|||
output_node_names.emplace_back(ort_session->GetOutputNameAllocated(i, allocator).get()); |
|||
auto output_shapes = ort_session->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); |
|||
if(log) |
|||
std::cout << "\t" << output_node_names.at(i) << " : " << PrintShape(output_shapes) << std::endl; |
|||
} |
|||
|
|||
} |
|||
|
|||
// Runs the model, given an image
|
|||
std::vector<Ort::Value> Onnx::Run(ofImage &img){ |
|||
auto start = std::chrono::high_resolution_clock::now(); |
|||
|
|||
TransformImage(img); |
|||
size_t input_tensor_size = image_array.total(); |
|||
std::vector<Ort::Value> input_tensors; |
|||
Ort::MemoryInfo mem_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); |
|||
input_tensors.emplace_back(Ort::Value::CreateTensor<float>(mem_info, (float*)image_array.data, |
|||
input_tensor_size, input_node_dims.data(), input_node_dims.size())); |
|||
|
|||
// double-check the dimensions of the input tensor
|
|||
assert(input_tensors[0].IsTensor() && input_tensors[0].GetTensorTypeAndShapeInfo().GetShape() == input_node_dims); |
|||
|
|||
//std::cout << "\ninput_tensor shape: " << PrintShape(input_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << std::endl;
|
|||
|
|||
// pass data through model
|
|||
std::vector<const char*> input_names_char(input_node_names.size(), nullptr); |
|||
std::transform(std::begin(input_node_names), std::end(input_node_names), std::begin(input_names_char), |
|||
[&](const std::string& str) { return str.c_str(); }); |
|||
|
|||
std::vector<const char*> output_names_char(output_node_names.size(), nullptr); |
|||
std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char), |
|||
[&](const std::string& str) { return str.c_str(); }); |
|||
|
|||
try { |
|||
auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(), |
|||
input_names_char.size(), output_names_char.data(), output_names_char.size()); |
|||
|
|||
if (timeStamp) { |
|||
auto end = std::chrono::high_resolution_clock::now(); |
|||
std::chrono::duration<double, std::milli> elapsed = end - start; |
|||
std::cout << "Update loop took " << elapsed.count() << " ms" << std::endl; |
|||
} |
|||
|
|||
return output_tensors; |
|||
|
|||
} catch (const Ort::Exception& exception) { |
|||
std::cout << "ERROR running model inference: " << exception.what() << std::endl; |
|||
return input_tensors; |
|||
|
|||
} |
|||
|
|||
} |
|||
|
|||
/*
|
|||
Runs a model, with a batch of images as input (emotion.onnx) |
|||
(1) Creates a 1-dim float array based on the batch * channels * width * height) |
|||
(2) Transforms each image into a cv::Mat, and appends it to the array |
|||
(3) Sends that information to the model to be processed. |
|||
*/ |
|||
std::vector<Ort::Value> Onnx::RunBatch(std::vector<ofImage>& images){ |
|||
|
|||
auto start = std::chrono::high_resolution_clock::now(); |
|||
|
|||
// Number of images in the batch
|
|||
size_t batchSize = images.size(); |
|||
|
|||
std::vector<int64_t> batch_node_dims = {static_cast<int64_t>(batchSize), 3, 260, 260}; |
|||
|
|||
std::vector<float> batch_image_array(static_cast<int64_t>(batchSize) * 3 * 260 * 260); |
|||
|
|||
for(size_t i = 0; i < batchSize; i++){ |
|||
TransformImage(images[i]); |
|||
// Copy the image data into the batch_image_array
|
|||
std::memcpy( |
|||
batch_image_array.data() + i * 3 * 260 * 260, // Destination: starting point for this image in the batch array
|
|||
image_array.data, // Source: pixel data in the cv::Mat
|
|||
3 * 260 * 260 * sizeof(float) // Size: number of bytes to copy (3 channels * 260 * 260 pixels)
|
|||
); |
|||
} |
|||
|
|||
size_t input_tensor_size = batch_image_array.size(); |
|||
std::vector<Ort::Value> input_tensors; |
|||
Ort::MemoryInfo mem_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); |
|||
input_tensors.emplace_back(Ort::Value::CreateTensor<float>(mem_info, batch_image_array.data(), |
|||
input_tensor_size, batch_node_dims.data(), batch_node_dims.size())); |
|||
|
|||
// double-check the dimensions of the input tensor
|
|||
assert(input_tensors[0].IsTensor() && input_tensors[0].GetTensorTypeAndShapeInfo().GetShape() == batch_node_dims); |
|||
|
|||
std::cout << "\ninput_tensor shape: " << PrintShape(input_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << std::endl; |
|||
|
|||
// pass data through model
|
|||
std::vector<const char*> input_names_char(input_node_names.size(), nullptr); |
|||
std::transform(std::begin(input_node_names), std::end(input_node_names), std::begin(input_names_char), |
|||
[&](const std::string& str) { return str.c_str(); }); |
|||
|
|||
std::vector<const char*> output_names_char(output_node_names.size(), nullptr); |
|||
std::transform(std::begin(output_node_names), std::end(output_node_names), std::begin(output_names_char), |
|||
[&](const std::string& str) { return str.c_str(); }); |
|||
|
|||
try { |
|||
auto output_tensors = ort_session->Run(Ort::RunOptions{nullptr}, input_names_char.data(), input_tensors.data(), |
|||
input_names_char.size(), output_names_char.data(), output_names_char.size()); |
|||
|
|||
if (timeStamp) { |
|||
auto end = std::chrono::high_resolution_clock::now(); |
|||
std::chrono::duration<double, std::milli> elapsed = end - start; |
|||
std::cout << "Update loop took " << elapsed.count() << " ms" << std::endl; |
|||
} |
|||
|
|||
return output_tensors; |
|||
|
|||
} catch (const Ort::Exception& exception) { |
|||
std::cout << "ERROR running model inference: " << exception.what() << std::endl; |
|||
return input_tensors; |
|||
|
|||
} |
|||
|
|||
} |
|||
|
|||
// Transforms an ofImage into a cv::ImageArray
|
|||
void Onnx::TransformImage(ofImage &img){ |
|||
// Convert ofImage to cv::Mat
|
|||
ofPixels &pixels = img.getPixels(); // Get pixels from ofImage
|
|||
int width = img.getWidth(); |
|||
int height = img.getHeight(); |
|||
int channels = img.getPixels().getNumChannels(); |
|||
|
|||
// Create a cv::Mat from the pixel data
|
|||
cv::Mat cvImg = cv::Mat(height, width, (channels == 3) ? CV_8UC3 : CV_8UC1, pixels.getData()); |
|||
|
|||
// Convert cv::Mat to cv::InputArray
|
|||
cv::InputArray inputArray(cvImg); |
|||
|
|||
image_array = cv::dnn::blobFromImage(inputArray, 1 / 255.0, cv::Size(input_node_dims[2] , input_node_dims[3] ), (0, 0, 0), false, false); |
|||
} |
|||
|
|||
/* Generates a random 1 dimensional float array, creating values between 0->255.
|
|||
Returns a tensor. */ |
|||
Ort::Value Onnx::GenerateTensor(){ |
|||
std::vector<float> random_input_tensor_values(CalculateProduct(input_node_dims)); |
|||
std::generate(random_input_tensor_values.begin(), random_input_tensor_values.end(), [&] { return rand() % 255; }); |
|||
return VectorToTensor(random_input_tensor_values, input_node_dims); |
|||
} |
|||
|
|||
// Calculates the product of the vector, how many values.
|
|||
int Onnx::CalculateProduct(const std::vector<std::int64_t>& v){ |
|||
int total = 1; |
|||
for (auto& i : v) total *= i; |
|||
return total; |
|||
} |
|||
|
|||
// Creates a tensor from a given vector input.
|
|||
Ort::Value Onnx::VectorToTensor(std::vector<float>& data, const std::vector<std::int64_t>& shape){ |
|||
Ort::MemoryInfo mem_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); |
|||
auto tensor = Ort::Value::CreateTensor<float>(mem_info, data.data(), data.size(), shape.data(), shape.size()); |
|||
return tensor; |
|||
} |
|||
|
|||
// Prints the shape of the given tensor (ex. input: (1, 1, 512, 512))
|
|||
std::string Onnx::PrintShape(const std::vector<std::int64_t>& v){ |
|||
std::stringstream ss(""); |
|||
for (std::size_t i = 0; i < v.size() - 1; i++) ss << v[i] << "x"; |
|||
ss << v[v.size() - 1]; |
|||
return ss.str(); |
|||
} |
|||
|
|||
// Function to calculate the minimum value in an array
|
|||
float Onnx::ReduceMin(const float* data, size_t size) { |
|||
return *std::min_element(data, data + size); |
|||
} |
|||
|
|||
// Function to calculate the maximum value in an array
|
|||
float Onnx::ReduceMax(const float* data, size_t size) { |
|||
return *std::max_element(data, data + size); |
|||
} |
|||
|
|||
// Function to normalize an array between 0 and 1
|
|||
void Onnx::Normalize(float* data, size_t size, float min_value, float max_value) { |
|||
for (size_t i = 0; i < size; ++i) { |
|||
data[i] = (data[i] - min_value) / (max_value - min_value); |
|||
} |
|||
} |
|||
|
|||
// Coverts the output tensor data to a texture of a given ofFbo.
|
|||
void Onnx::DataToFbo(float* data, size_t width, size_t height, ofFbo& fbo){ |
|||
// Convert data into opencv mat
|
|||
cv::Mat inputMat(height, width, CV_32FC1); |
|||
memcpy(inputMat.data, data, width * height * sizeof(float)); |
|||
// // Convert to 8-bit grayscale Mat
|
|||
cv::Mat inputMat8U; |
|||
inputMat.convertTo(inputMat8U, CV_8UC1, 255.0); // Convert float to 8-bit grayscale
|
|||
|
|||
// // Resize the image using OpenCV
|
|||
cv::Mat resizedMat; |
|||
cv::resize(inputMat8U, resizedMat, cv::Size(fbo.getWidth(), fbo.getHeight()), 0, 0, cv::INTER_LINEAR); |
|||
|
|||
// // Convert OpenCV Mat to ofPixels
|
|||
pixels.allocate(fbo.getWidth(), fbo.getHeight(), OF_PIXELS_GRAY); |
|||
|
|||
// // Copy data from resizedMat to ofPixels
|
|||
memcpy(pixels.getData(), resizedMat.data, fbo.getWidth() * fbo.getHeight()); |
|||
} |
|||
|
|||
void Onnx::SetPixels(ofFbo& fbo){ |
|||
fbo.begin(); |
|||
ofTexture& texture = fbo.getTexture(); |
|||
texture.loadData(pixels); |
|||
fbo.end(); |
|||
} |
|||
|
|||
void Onnx::Softmax(float* data, size_t size) { |
|||
std::vector<float> logits(data, data + size); |
|||
std::vector<float> expValues(size); |
|||
float maxLogit = *std::max_element(logits.begin(), logits.end()); |
|||
|
|||
// Calculate exp(logit - maxLogit) for numerical stability
|
|||
std::transform(logits.begin(), logits.end(), expValues.begin(), |
|||
[maxLogit](float logit) { return std::exp(logit - maxLogit); }); |
|||
|
|||
float sumExp = std::accumulate(expValues.begin(), expValues.end(), 0.0f); |
|||
|
|||
// Normalize to get probabilities
|
|||
std::transform(expValues.begin(), expValues.end(), data, |
|||
[sumExp](float expValue) { return expValue / sumExp; }); |
|||
} |
|||
|
|||
|
|||
@ -0,0 +1,56 @@ |
|||
#pragma once |
|||
|
|||
#include <onnxruntime_cxx_api.h> |
|||
#include <algorithm> // std::generate |
|||
#include <cassert> |
|||
#include <cstddef> |
|||
#include <cstdint> |
|||
#include <iostream> |
|||
#include <sstream> |
|||
#include <string> |
|||
#include <vector> |
|||
#include <cstdlib> // For std::rand and std::srand |
|||
#include "ofMain.h" |
|||
#include "ofxOpenCv.h" |
|||
#include <numeric> |
|||
#include <cmath> |
|||
|
|||
class Onnx { |
|||
|
|||
public: |
|||
Onnx() {} |
|||
void Setup(ORTCHAR_T* modelPath, bool isLog, bool useCuda); |
|||
std::vector<Ort::Value> Run(ofImage &img); |
|||
std::vector<Ort::Value> RunBatch(std::vector<ofImage> &images); |
|||
std::string PrintShape(const std::vector<std::int64_t>& v); |
|||
Ort::Value VectorToTensor(std::vector<float>& data, const std::vector<std::int64_t>& shape); |
|||
Ort::Value GenerateTensor(); |
|||
int CalculateProduct(const std::vector<std::int64_t>& v); |
|||
void TransformImage(ofImage &img); |
|||
float ReduceMin(const float* data, size_t size); |
|||
float ReduceMax(const float* data, size_t size); |
|||
void Normalize(float* data, size_t size, float min_value, float max_value); |
|||
void DataToFbo(float* data, size_t width, size_t height, ofFbo& fbo); |
|||
void Softmax(float* data, size_t size); |
|||
void SetPixels(ofFbo& fbo); |
|||
bool timeStamp = false; |
|||
bool log = false; |
|||
ofPixels pixels; |
|||
|
|||
protected: |
|||
Ort::Env ort_env; |
|||
Ort::SessionOptions session_options; |
|||
cv::Mat image_array; |
|||
std::shared_ptr<Ort::Session> ort_session; |
|||
std::vector<std::string> input_node_names; |
|||
std::vector<int64_t> input_node_dims; // 1 input only.
|
|||
std::size_t input_tensor_size = 1; |
|||
std::vector<float> input_values_handler; |
|||
Ort::MemoryInfo memory_info_handler = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); |
|||
std::vector<std::string> output_node_names; |
|||
std::vector<std::vector<int64_t>> output_node_dims; // >=1 outputs
|
|||
std::vector<Ort::Value> output_values; |
|||
Ort::Value dummy_tensor{ nullptr }; |
|||
std::vector<ofImage> imageBatch; |
|||
int num_outputs = 1; |
|||
}; |
|||
@ -0,0 +1,157 @@ |
|||
#include "Yolo.h" |
|||
|
|||
// Takes output tensor data, processes that data, and returns an array of BoxfWithLandmarks (detected_faces)
|
|||
void Yolo::ParseOutput(float* &output_tensors, std::vector<types::BoxfWithLandmarks> &sorted_faces, unsigned int num_anchors) { |
|||
|
|||
std::vector<types::BoxfWithLandmarks> detected_faces; |
|||
|
|||
for (unsigned int i = 0; i < num_anchors; ++i) { |
|||
const float *row_ptr = output_tensors + i * 16; // Each row contains 16 values
|
|||
float obj_conf = row_ptr[4]; // Objectness confidence
|
|||
|
|||
if (obj_conf < 0.5) continue; // Filter out low-confidence detections
|
|||
|
|||
// Extract bounding box, confidence, and landmarks
|
|||
float cx = row_ptr[0]; |
|||
float cy = row_ptr[1]; |
|||
float w = row_ptr[2]; |
|||
float h = row_ptr[3]; |
|||
float cls_conf = row_ptr[15]; // Face confidence score
|
|||
|
|||
if (cls_conf < 0.5) continue; // Filter by class confidence
|
|||
|
|||
types::BoxfWithLandmarks face; |
|||
face.box.x1 = cx - w / 2; |
|||
face.box.y1 = cy - h / 2; |
|||
face.box.x2 = cx + w / 2; |
|||
face.box.y2 = cy + h / 2; |
|||
face.box.score = cls_conf; |
|||
face.box.label_text = "face"; |
|||
|
|||
// Extract landmarks
|
|||
for (int j = 0; j < 10; j += 2) { |
|||
face.landmarks.points.push_back(cv::Point2f(row_ptr[5 + j], row_ptr[5 + j + 1])); |
|||
} |
|||
|
|||
detected_faces.push_back(face); // Store the detected face with landmarks
|
|||
} |
|||
|
|||
// Apply NMS to detected faces list to remove any overlapping bounding boxes.
|
|||
NonMaximumSuppression(detected_faces, sorted_faces, 0.5); |
|||
|
|||
// Sort faces based on confidence value
|
|||
SortDetectedFaces(sorted_faces); |
|||
} |
|||
|
|||
// Simple helper for drawing boxes given x1, y1, x2, y2 coordinates.
|
|||
void Yolo::DrawBox(std::vector<types::BoxfWithLandmarks> &detected_faces){ |
|||
for (const auto &face : detected_faces) { |
|||
ofNoFill(); |
|||
float w = ofGetWindowWidth() / 2; |
|||
ofDrawRectangle(face.box.x1 + w, face.box.y1, ((face.box.x2 + w) - (face.box.x1 + w)), face.box.y2 - face.box.y1); |
|||
} |
|||
} |
|||
|
|||
// Simple helper to draw boxes at the center of the detected face.
|
|||
void Yolo::DrawCenter(std::vector<types::BoxfWithLandmarks> &detected_faces){ |
|||
ofNoFill(); |
|||
glm::vec2 position = detected_faces[0].box.center; |
|||
ofDrawCircle(position.x + ofGetWindowWidth() / 2, position.y, 5); |
|||
|
|||
} |
|||
|
|||
// Applies NMS to an array of BoxfWithLandmarks.face.boxes, to remove any overlapping bounding boxes.
|
|||
void Yolo::NonMaximumSuppression(std::vector<types::BoxfWithLandmarks> &input_faces, std::vector<types::BoxfWithLandmarks> &output_faces, float iou_threshold) |
|||
{ |
|||
// Sort the boxes by their confidence scores (highest to lowest)
|
|||
std::sort(input_faces.begin(), input_faces.end(), |
|||
[](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) { |
|||
return a.box.score > b.box.score; |
|||
}); |
|||
|
|||
std::vector<int> suppressed(input_faces.size(), 0); // Suppression mask
|
|||
|
|||
// Iterate through the boxes
|
|||
for (size_t i = 0; i < input_faces.size(); ++i) { |
|||
if (suppressed[i]) continue; // Skip already suppressed boxes
|
|||
|
|||
// Add this box to the output
|
|||
output_faces.push_back(input_faces[i]); |
|||
|
|||
for (size_t j = i + 1; j < input_faces.size(); ++j) { |
|||
if (suppressed[j]) continue; |
|||
|
|||
// Calculate IoU between box i and box j
|
|||
float iou = input_faces[i].box.iou_of(input_faces[j].box); |
|||
|
|||
// Suppress box j if IoU is greater than the threshold
|
|||
if (iou > iou_threshold) { |
|||
suppressed[j] = 1; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
// Scales the coordinates of detected faces from the model output dimensions -> the original input image dimensions.
|
|||
void Yolo::ConvertBoxCoordsToOriginalSize(std::vector<types::BoxfWithLandmarks> &detected_faces, size_t original_width, size_t original_height){ |
|||
float width_scale = static_cast<float>(original_width) / modelSize; |
|||
float height_scale = static_cast<float>(original_height) / modelSize; |
|||
|
|||
for (auto &face : detected_faces) { |
|||
// Convert bounding box coordinates
|
|||
face.box.x1 *= width_scale; |
|||
face.box.y1 *= height_scale; |
|||
face.box.x2 *= width_scale; |
|||
face.box.y2 *= height_scale; |
|||
face.box.UpdateCenter(); |
|||
|
|||
// Convert landmarks
|
|||
for (size_t j = 0; j < face.landmarks.points.size(); ++j) { |
|||
face.landmarks.points[j].x *= width_scale; |
|||
face.landmarks.points[j].y *= height_scale; |
|||
} |
|||
} |
|||
} |
|||
|
|||
void Yolo::CropFaceToImage(ofImage &inputImage, types::BoxfWithLandmarks &face, ofxCvColorImage &colorImage){ |
|||
|
|||
colorImage.resetROI(); |
|||
|
|||
// Calculate the coordinates and dimensions of the face box
|
|||
float x1 = face.box.x1; |
|||
float y1 = face.box.y1; |
|||
float x2 = face.box.x2; |
|||
float y2 = face.box.y2; |
|||
|
|||
// Ensure coordinates are within the input image bounds
|
|||
x1 = ofClamp(x1, 0.0f, (float)inputImage.getWidth()); |
|||
y1 = ofClamp(y1, 0.0f, (float)inputImage.getHeight()); |
|||
x2 = ofClamp(x2, 0.0f, (float)inputImage.getWidth()); |
|||
y2 = ofClamp(y2, 0.0f, (float)inputImage.getHeight()); |
|||
|
|||
// Calculate width and height of the cropped area
|
|||
float cropWidth = x2 - x1; |
|||
float cropHeight = y2 - y1; |
|||
|
|||
// Create cropped section, defined by the box coords
|
|||
ofFbo tempFbo; |
|||
tempFbo.allocate(cropWidth, cropHeight, GL_RGB); |
|||
|
|||
tempFbo.begin(); |
|||
ofClear(0); |
|||
inputImage.getTexture().drawSubsection(0, 0, cropWidth, cropHeight, x1, y1); |
|||
tempFbo.end(); |
|||
|
|||
ofFloatPixels pix; |
|||
tempFbo.readToPixels(pix); |
|||
colorImage.setFromPixels(pix); |
|||
|
|||
colorImage.resize(260, 260); |
|||
} |
|||
|
|||
void Yolo::SortDetectedFaces(std::vector<types::BoxfWithLandmarks> &detectedFaces){ |
|||
std::sort(detectedFaces.begin(), detectedFaces.end(), |
|||
[](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) { |
|||
return a.box.score > b.box.score; // Sort in descending order
|
|||
}); |
|||
} |
|||
@ -0,0 +1,81 @@ |
|||
#ifndef YOLO |
|||
#define YOLO |
|||
|
|||
#include "ofMain.h" |
|||
#include "ofxOpenCv.h" |
|||
#include <onnxruntime_cxx_api.h> |
|||
#include <algorithm> |
|||
|
|||
struct Emotef{ |
|||
float emotions[7]; |
|||
}; |
|||
|
|||
namespace types { |
|||
|
|||
/*
|
|||
Struct for storing information about detetced faces. |
|||
*/ |
|||
struct Boxf { |
|||
float x1, y1, x2, y2; // Coordinates of the bounding box
|
|||
float score; // Confidence score
|
|||
glm::vec2 center; |
|||
int label; // Class label (e.g., "face")
|
|||
std::string label_text; |
|||
Emotef emotional_state; |
|||
|
|||
// Calculate Intersection over Union (IoU) with another box
|
|||
float iou_of(const Boxf &other) const { |
|||
float intersection_x1 = std::max(x1, other.x1); |
|||
float intersection_y1 = std::max(y1, other.y1); |
|||
float intersection_x2 = std::min(x2, other.x2); |
|||
float intersection_y2 = std::min(y2, other.y2); |
|||
|
|||
float intersection_area = std::max(0.0f, intersection_x2 - intersection_x1) * |
|||
std::max(0.0f, intersection_y2 - intersection_y1); |
|||
|
|||
float this_area = (x2 - x1) * (y2 - y1); |
|||
float other_area = (other.x2 - other.x1) * (other.y2 - other.y1); |
|||
|
|||
float union_area = this_area + other_area - intersection_area; |
|||
|
|||
return intersection_area / union_area; |
|||
} |
|||
|
|||
void UpdateCenter(){ |
|||
center.x = (x1 + x2) / 2; |
|||
center.y = (y1 + y2) / 2; |
|||
} |
|||
|
|||
void SetEmotionState(float* emotional_data){ |
|||
std::copy(emotional_data, emotional_data + 7, emotional_state.emotions); |
|||
} |
|||
}; |
|||
|
|||
struct Landmarks { |
|||
std::vector<cv::Point2f> points; // Facial landmarks points (e.g., eyes, nose, mouth)
|
|||
bool flag = false; // Indicator if landmarks are available
|
|||
}; |
|||
|
|||
struct BoxfWithLandmarks { |
|||
Boxf box; // Bounding box for the face
|
|||
Landmarks landmarks; // Landmark points for the face
|
|||
bool flag = false; // Indicator if this detection is valid
|
|||
}; |
|||
} |
|||
|
|||
class Yolo{ |
|||
public: |
|||
Yolo(){}; |
|||
void ParseOutput(float* &out_ptr, std::vector<types::BoxfWithLandmarks> &sorted_faces, unsigned int num_anchors); |
|||
void DrawBox(std::vector<types::BoxfWithLandmarks> &detected_faces); |
|||
void DrawCenter(std::vector<types::BoxfWithLandmarks> &detected_faces); |
|||
void NonMaximumSuppression(std::vector<types::BoxfWithLandmarks> &input_faces, std::vector<types::BoxfWithLandmarks> &output_faces, float iou_threshold); |
|||
void ConvertBoxCoordsToOriginalSize(std::vector<types::BoxfWithLandmarks> &detected_faces, size_t original_width, size_t original_height); |
|||
void CropFaceToImage(ofImage &inputImage, types::BoxfWithLandmarks &face, ofxCvColorImage &colorImage); |
|||
void SortDetectedFaces(std::vector<types::BoxfWithLandmarks> &detectedFaces); |
|||
private: |
|||
// Input dimenions of the model -- used for coordinate scaling.
|
|||
size_t modelSize = 640; |
|||
}; |
|||
|
|||
#endif |
|||
Loading…
Reference in new issue