using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Events;
namespace com.rfilkov.kinect
{
///
/// KinectEventManager provides sensor-frame events to the registered Unity event listeners.
///
public class KinectEventManager : MonoBehaviour
{
[Tooltip("Depth sensor index - 0 is the 1st one, 1 - the 2nd one, etc.")]
public int sensorIndex = 0;
[Tooltip("Consider the sensor as disconnected, if no frames are received within this time period, in seconds.")]
public float sensorTimeoutAfter = 2f;
[System.Serializable]
public class SensorStateEvent : UnityEvent { }
[System.Serializable]
public class ImageTextureEvent : UnityEvent { }
[System.Serializable]
public class UshortFrameEvent : UnityEvent { }
[System.Serializable]
public class ByteFrameEvent : UnityEvent { }
[System.Serializable]
public class BodyFrameEvent : UnityEvent { }
[Header("Sensor Events")]
///
/// Fired when the sensor is considered as disconnected.
///
public SensorStateEvent OnSensorDisconnect = new SensorStateEvent();
/////
///// Fired when the sensor is considered as reconnected.
/////
//public SensorStateEvent OnSensorReconnect = new SensorStateEvent();
[Header("Sensor Data")]
///
/// Fired when new color-camera frame is detected.
///
public ImageTextureEvent OnNewColorImage = new ImageTextureEvent();
///
/// Fired when new depth frame is detected.
///
public UshortFrameEvent OnNewDepthFrame = new UshortFrameEvent();
///
/// Fired when new infrared frame is detected.
///
public UshortFrameEvent OnNewInfraredFrame = new UshortFrameEvent();
[Header("Body Tracking")]
///
/// Fired when new body frame frame is detected.
///
public BodyFrameEvent OnNewBodyFrame = new BodyFrameEvent();
///
/// Fired when new body-index frame is detected.
///
public ByteFrameEvent OnNewBodyIndexFrame = new ByteFrameEvent();
[Header("Sensor Images")]
///
/// Fired when new depth image is detected.
///
public ImageTextureEvent OnNewDepthImage = new ImageTextureEvent();
///
/// Fired when new infrared image is detected.
///
public ImageTextureEvent OnNewInfraredImage = new ImageTextureEvent();
///
/// Fired when new user body image is detected.
///
public ImageTextureEvent OnNewBodyIndexImage = new ImageTextureEvent();
[Header("Transformed Frames")]
///
/// Fired when new depth-camera transformed color image is detected.
///
public ImageTextureEvent OnNewDepthCameraColorImage = new ImageTextureEvent();
///
/// Fired when new color-camera transformed depth frame is detected.
///
public UshortFrameEvent OnNewColorCameraDepthFrame = new UshortFrameEvent();
///
/// Fired when new color-camera transformed infrared image is detected.
///
public ImageTextureEvent OnNewColorCameraInfraredImage = new ImageTextureEvent();
///
/// Fired when new color-camera transformed body-index frame is detected.
///
public ByteFrameEvent OnNewColorCameraBodyIndexFrame = new ByteFrameEvent();
// reference to KinectManager
private KinectManager kinectManager = null;
// reference to the respective sensor-data
private KinectInterop.SensorData sensorData = null;
// last frame times
private ulong lastColorFrameTime = 0;
private ulong lastDepthFrameTime = 0;
private ulong lastInfraredFrameTime = 0;
private ulong lastBodyFrameTime = 0;
private ulong lastBodyIndexFrameTime = 0;
private ulong lastDepthImageTime = 0;
private ulong lastInfraredImageTime = 0;
private ulong lastBodyIndexImageTime = 0;
private ulong lastDepthCamColorFrameTime = 0;
private ulong lastColorCamDepthFrameTime = 0;
private ulong lastColorCamInfraredFrameTime = 0;
private ulong lastColorCamBodyIndexFrameTime = 0;
// transformed frames enabled-flags
private bool isDepthCamColorFramesEnabled = false;
private bool isColorCamDepthFramesEnabled = false;
private bool isColorCamInfraredFramesEnabled = false;
private bool isColorCamBodyIndexFramesEnabled = false;
// transformed depth cam color texture
private Texture2D depthCamColorTex2D = null;
private ushort[] colorCamDepthFrame = null;
private byte[] colorCamBodyIndexFrame = null;
private Texture2D colorCamInfraredTex2D = null;
// whether the sensor is currently disconnected
private long lastFrameTime = 0;
private bool isSensorDisconnected = false;
void Start()
{
kinectManager = KinectManager.Instance;
sensorData = kinectManager != null ? kinectManager.GetSensorData(sensorIndex) : null;
}
void OnDestroy()
{
if(isDepthCamColorFramesEnabled)
{
isDepthCamColorFramesEnabled = false;
sensorData.sensorInterface.EnableDepthCameraColorFrame(sensorData, false);
depthCamColorTex2D = null;
Debug.Log("Disabled DepthCameraColorFrames.");
}
if (isColorCamDepthFramesEnabled)
{
isColorCamDepthFramesEnabled = false;
sensorData.sensorInterface.EnableColorCameraDepthFrame(sensorData, false);
Debug.Log("Disabled ColorCameraDepthFrames.");
}
if (isColorCamBodyIndexFramesEnabled)
{
isColorCamBodyIndexFramesEnabled = false;
sensorData.sensorInterface.EnableColorCameraBodyIndexFrame(sensorData, false);
Debug.Log("Disabled ColorCameraBodyIndexFrames.");
}
}
void Update()
{
if (sensorData != null)
{
// color frame
if(lastColorFrameTime != sensorData.lastColorFrameTime)
{
lastFrameTime = System.DateTime.Now.Ticks;
lastColorFrameTime = sensorData.lastColorFrameTime;
OnNewColorImage?.Invoke(sensorData.colorImageTexture, sensorData.lastColorFrameTime);
//Debug.Log("OnNewColorFrame invoked. Time: " + lastColorFrameTime);
}
// depth frame
if (lastDepthFrameTime != sensorData.lastDepthFrameTime)
{
lastFrameTime = System.DateTime.Now.Ticks;
lastDepthFrameTime = sensorData.lastDepthFrameTime;
OnNewDepthFrame?.Invoke(sensorData.depthImage, sensorData.depthImageWidth, sensorData.depthImageHeight, sensorData.lastDepthFrameTime);
//Debug.Log("OnNewDepthFrame invoked. Time: " + lastDepthFrameTime);
}
// infrared frame
if (lastInfraredFrameTime != sensorData.lastInfraredFrameTime)
{
lastFrameTime = System.DateTime.Now.Ticks;
lastInfraredFrameTime = sensorData.lastInfraredFrameTime;
OnNewInfraredFrame?.Invoke(sensorData.infraredImage, sensorData.depthImageWidth, sensorData.depthImageHeight, sensorData.lastInfraredFrameTime);
//Debug.Log("OnNewInfraredFrame invoked. Time: " + lastInfraredFrameTime);
}
// body frame
if (lastBodyFrameTime != sensorData.lastBodyFrameTime)
{
lastFrameTime = System.DateTime.Now.Ticks;
lastBodyFrameTime = sensorData.lastBodyFrameTime;
OnNewBodyFrame?.Invoke(sensorData.alTrackedBodies, sensorData.trackedBodiesCount, sensorData.lastBodyFrameTime);
//Debug.Log("OnNewBodyFrame invoked. Time: " + lastBodyFrameTime);
}
// body-index frame
if (lastBodyIndexFrameTime != sensorData.lastBodyIndexFrameTime)
{
lastFrameTime = System.DateTime.Now.Ticks;
lastBodyIndexFrameTime = sensorData.lastBodyIndexFrameTime;
OnNewBodyIndexFrame?.Invoke(sensorData.bodyIndexImage, sensorData.depthImageWidth, sensorData.depthImageHeight, sensorData.lastBodyIndexFrameTime);
//Debug.Log("OnNewBodyIndexFrame invoked. Time: " + lastBodyIndexFrameTime);
}
// depth image
if (OnNewDepthImage != null && OnNewDepthImage.GetPersistentEventCount() > 0 && lastDepthImageTime != sensorData.lastDepthImageTime)
{
lastDepthImageTime = sensorData.lastDepthImageTime;
OnNewDepthImage?.Invoke(sensorData.depthImageTexture, sensorData.lastDepthImageTime);
//Debug.Log("OnNewDepthImage invoked. Time: " + lastDepthImageTime);
}
// infrared image
if (OnNewInfraredImage != null && OnNewInfraredImage.GetPersistentEventCount() > 0 && lastInfraredImageTime != sensorData.lastInfraredImageTime)
{
lastInfraredImageTime = sensorData.lastInfraredImageTime;
OnNewInfraredImage?.Invoke(sensorData.infraredImageTexture, sensorData.lastInfraredImageTime);
//Debug.Log("OnNewInfraredImage invoked. Time: " + lastInfraredImageTime);
}
// body-index image
if (OnNewBodyIndexImage != null && OnNewBodyIndexImage.GetPersistentEventCount() > 0 && lastBodyIndexImageTime != sensorData.lastBodyImageTime)
{
lastBodyIndexImageTime = sensorData.lastBodyImageTime;
OnNewBodyIndexImage?.Invoke(sensorData.bodyImageTexture, sensorData.lastBodyImageTime);
//Debug.Log("OnNewBodyIndexImage invoked. Time: " + lastBodyIndexImageTime);
}
// check for sensor events
long currentTime = System.DateTime.Now.Ticks;
long timeoutAfter = (long)(sensorTimeoutAfter * 10000000);
if(!isSensorDisconnected && lastFrameTime != 0 && (currentTime - lastFrameTime) > timeoutAfter)
{
isSensorDisconnected = true;
OnSensorDisconnect?.Invoke(GetLastFrameTime());
Debug.Log("Sensor disconnected. Time: " + GetLastFrameTime());
}
//else if(isSensorDisconnected && (currentTime - lastFrameTime) <= timeoutAfter)
//{
// isSensorDisconnected = false;
// OnSensorReconnect?.Invoke(GetLastFrameTime());
// Debug.Log("Sensor reconnected. Time: " + GetLastFrameTime());
//}
// depth-cam color frame
if (OnNewDepthCameraColorImage != null && OnNewDepthCameraColorImage.GetPersistentEventCount() > 0 && !isDepthCamColorFramesEnabled)
{
isDepthCamColorFramesEnabled = true;
sensorData.sensorInterface.EnableDepthCameraColorFrame(sensorData, true);
depthCamColorTex2D = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight, sensorData.colorImageFormat, false);
Debug.Log("Enabled DepthCameraColorFrames.");
}
else if ((OnNewDepthCameraColorImage == null || OnNewDepthCameraColorImage.GetPersistentEventCount() == 0) && isDepthCamColorFramesEnabled)
{
isDepthCamColorFramesEnabled = false;
sensorData.sensorInterface.EnableDepthCameraColorFrame(sensorData, false);
depthCamColorTex2D = null;
Debug.Log("Disabled DepthCameraColorFrames.");
}
if (isDepthCamColorFramesEnabled)
{
ulong oldFrameTime = lastDepthCamColorFrameTime;
sensorData.sensorInterface.GetDepthCameraColorFrameTexture(sensorData, ref depthCamColorTex2D, ref lastDepthCamColorFrameTime);
if(oldFrameTime != lastDepthCamColorFrameTime)
{
OnNewDepthCameraColorImage?.Invoke(depthCamColorTex2D, lastDepthCamColorFrameTime);
//Debug.Log("OnNewDepthCameraColorImage invoked. Time: " + lastDepthCamColorFrameTime);
}
}
// color-cam depth frame
if (OnNewColorCameraDepthFrame != null && OnNewColorCameraDepthFrame.GetPersistentEventCount() > 0 && !isColorCamDepthFramesEnabled)
{
isColorCamDepthFramesEnabled = true;
sensorData.sensorInterface.EnableColorCameraDepthFrame(sensorData, true);
colorCamDepthFrame = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
Debug.Log("Enabled ColorCameraDepthFrames.");
}
else if ((OnNewColorCameraDepthFrame == null || OnNewColorCameraDepthFrame.GetPersistentEventCount() == 0) && isColorCamDepthFramesEnabled)
{
isColorCamDepthFramesEnabled = false;
sensorData.sensorInterface.EnableColorCameraDepthFrame(sensorData, false);
colorCamDepthFrame = null;
Debug.Log("Disabled ColorCameraDepthFrames.");
}
if (isColorCamDepthFramesEnabled)
{
ulong oldFrameTime = lastColorCamDepthFrameTime;
sensorData.sensorInterface.GetColorCameraDepthFrame(sensorData, ref colorCamDepthFrame, ref lastColorCamDepthFrameTime);
if (colorCamDepthFrame != null && oldFrameTime != lastColorCamDepthFrameTime)
{
OnNewColorCameraDepthFrame?.Invoke(colorCamDepthFrame, sensorData.colorImageWidth, sensorData.colorImageHeight, lastColorCamDepthFrameTime);
//Debug.Log("OnNewColorCameraDepthFrame invoked. Time: " + lastColorCamDepthFrameTime);
}
}
// color-cam infrared frame
if (OnNewColorCameraInfraredImage != null && OnNewColorCameraInfraredImage.GetPersistentEventCount() > 0 && !isColorCamInfraredFramesEnabled)
{
isColorCamInfraredFramesEnabled = true;
sensorData.sensorInterface.EnableColorCameraInfraredFrame(sensorData, false, true);
Debug.Log("Enabled ColorCameraInfraredFrames.");
}
else if ((OnNewColorCameraInfraredImage == null || OnNewColorCameraInfraredImage.GetPersistentEventCount() == 0) && isColorCamInfraredFramesEnabled)
{
isColorCamInfraredFramesEnabled = false;
sensorData.sensorInterface.EnableColorCameraInfraredFrame(sensorData, false, false);
Debug.Log("Disabled ColorCameraInfraredFrames.");
}
if (isColorCamInfraredFramesEnabled)
{
//if (colorCamInfraredTex2D == null)
//{
// colorCamInfraredTex2D = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.ARGB32, false);
//}
ulong oldFrameTime = lastColorCamInfraredFrameTime;
Texture colorCamInfraredTex = sensorData.sensorInterface.GetColorCameraInfraredFrameTexture(sensorData, ref colorCamInfraredTex2D, ref lastColorCamInfraredFrameTime);
if (colorCamInfraredTex != null && oldFrameTime != lastColorCamInfraredFrameTime)
{
OnNewColorCameraInfraredImage?.Invoke(colorCamInfraredTex, lastColorCamInfraredFrameTime);
//Debug.Log("OnNewColorCameraInfraredImage invoked. Time: " + lastColorCamInfraredFrameTime);
}
}
// color-cam body-index frame
if (OnNewColorCameraBodyIndexFrame != null && OnNewColorCameraBodyIndexFrame.GetPersistentEventCount() > 0 && !isColorCamBodyIndexFramesEnabled)
{
isColorCamBodyIndexFramesEnabled = true;
sensorData.sensorInterface.EnableColorCameraBodyIndexFrame(sensorData, true);
colorCamBodyIndexFrame = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight];
Debug.Log("Enabled ColorCameraBodyIndexFrames.");
}
else if ((OnNewColorCameraBodyIndexFrame == null || OnNewColorCameraBodyIndexFrame.GetPersistentEventCount() == 0) && isColorCamBodyIndexFramesEnabled)
{
isColorCamBodyIndexFramesEnabled = false;
sensorData.sensorInterface.EnableColorCameraBodyIndexFrame(sensorData, false);
colorCamBodyIndexFrame = null;
Debug.Log("Disabled ColorCameraBodyIndexFrames.");
}
if (isColorCamBodyIndexFramesEnabled)
{
ulong oldFrameTime = lastColorCamBodyIndexFrameTime;
sensorData.sensorInterface.GetColorCameraBodyIndexFrame(sensorData, ref colorCamBodyIndexFrame, ref lastColorCamBodyIndexFrameTime);
if (colorCamBodyIndexFrame != null && oldFrameTime != lastColorCamBodyIndexFrameTime)
{
OnNewColorCameraBodyIndexFrame?.Invoke(colorCamBodyIndexFrame, sensorData.colorImageWidth, sensorData.colorImageHeight, lastColorCamBodyIndexFrameTime);
//Debug.Log("OnNewColorCameraBodyIndexFrame invoked. Time: " + lastColorCamBodyIndexFrameTime);
}
}
}
}
// returns the last frame time
private ulong GetLastFrameTime()
{
ulong maxFrameTime = 0;
if (lastColorFrameTime != 0 && maxFrameTime < lastColorFrameTime)
maxFrameTime = lastColorFrameTime;
if (lastDepthFrameTime != 0 && maxFrameTime < lastDepthFrameTime)
maxFrameTime = lastDepthFrameTime;
if (lastInfraredFrameTime != 0 && maxFrameTime < lastInfraredFrameTime)
maxFrameTime = lastInfraredFrameTime;
if (lastBodyFrameTime != 0 && maxFrameTime < lastBodyFrameTime)
maxFrameTime = lastBodyFrameTime;
if (lastBodyIndexFrameTime != 0 && maxFrameTime < lastBodyIndexFrameTime)
maxFrameTime = lastBodyIndexFrameTime;
return maxFrameTime;
}
}
}