using UnityEngine;
using UnityEngine.UI;
using System.Collections;
using System.Collections.Generic;
using com.rfilkov.kinect;


namespace com.rfilkov.components
{
    /// <summary>
    /// This interface has to be implemented by all interaction listeners.
    /// </summary>
    public interface InteractionListenerInterface
    {
        /// <summary>
        /// Invoked when hand grip is detected.
        /// </summary>
        /// <param name="userId">User ID</param>
        /// <param name="userIndex">User index</param>
        /// <param name="isRightHand">Whether it is the right hand or not</param>
        /// <param name="isHandInteracting">Whether this hand is the interacting one or not</param>
        /// <param name="handScreenPos">Hand screen position, including depth (Z)</param>
        void HandGripDetected(ulong userId, int userIndex, bool isRightHand, bool isHandInteracting, Vector3 handScreenPos);

        /// <summary>
        /// Invoked when hand release is detected.
        /// </summary>
        /// <param name="userId">User ID</param>
        /// <param name="userIndex">User index</param>
        /// <param name="isRightHand">Whether it is the right hand or not</param>
        /// <param name="isHandInteracting">Whether this hand is the interacting one or not</param>
        /// <param name="handScreenPos">Hand screen position, including depth (Z)</param>
        void HandReleaseDetected(ulong userId, int userIndex, bool isRightHand, bool isHandInteracting, Vector3 handScreenPos);

        /// <summary>
        /// Invoked when hand click is detected.
        /// </summary>
        /// <returns><c>true</c>, if the click detection must be restarted, <c>false</c> otherwise.</returns>
        /// <param name="userId">User ID</param>
        /// <param name="userIndex">User index</param>
        /// <param name="isRightHand">Whether it is the right hand or not</param>
        /// <param name="handScreenPos">Hand screen position, including depth (Z)</param>
        bool HandClickDetected(ulong userId, int userIndex, bool isRightHand, Vector3 handScreenPos);
    }


    /// <summary>
    /// InteractionManager is the component that controls the hand cursor and manages the hand interactions.
    /// </summary>
    public class InteractionManager : MonoBehaviour
    {
        /// <summary>
        /// The hand event types.
        /// </summary>
        public enum HandEventType : int
        {
            None = 0,
            Grip = 1,
            Release = 2
        }

        [Tooltip("Index of the player, tracked by this component. 0 means the 1st player, 1 - the 2nd one, 2 - the 3rd one, etc.")]
        public int playerIndex = 0;

        [Tooltip("Whether the left hand interaction is allowed.")]
        public bool leftHandInteraction = true;

        [Tooltip("Whether the right hand interaction is allowed.")]
        public bool rightHandInteraction = true;

        [Tooltip("The image that may be used to show the hand-moved cursor on the screen or not. The sprite textures below need to be set too.")]
        public Image guiHandCursor;

        [Tooltip("Hand-cursor sprite texture, for the hand-grip state.")]
        public Sprite gripHandTexture;
        [Tooltip("Hand-cursor sprite texture, for the hand-release state.")]
        public Sprite releaseHandTexture;
        [Tooltip("Hand-cursor sprite texture, for the non-tracked state.")]
        public Sprite normalHandTexture;

        [Tooltip("Whether the hand cursor should overlay user's hand on color camera image, or not.")]
        public bool handOverlayCursor = false;

        [Tooltip("Depth sensor index used for color camera overlay - 0 is the 1st one, 1 - the 2nd one, etc.")]
        public int sensorIndex = 0;

        [Tooltip("Smooth factor for cursor movement.")]
        public float smoothFactor = 10f;

        [Tooltip("Whether the hand clicks (i.e. hand staying in place for ~2 seconds) are enabled or not.")]
        public bool allowHandClicks = true;

        [Tooltip("Whether the hand pushes need to be considered as clicks or not.")]
        public bool allowPushToClick = true;

        [Tooltip("Whether the hand cursor and interactions control the mouse cursor or not.")]
        public bool controlMouseCursor = false;

        [Tooltip("Whether the hand grips and releases control mouse dragging or not.")]
        public bool controlMouseDrag = false;

        // Bool to specify whether to convert Unity screen coordinates to full screen mouse coordinates
        //public bool convertMouseToFullScreen = false;

        [Tooltip("List of the interaction listeners in the scene. If the list is empty, the available interaction listeners will be detected at scene start up.")]
        public List<MonoBehaviour> interactionListeners;

        [Tooltip("UI-Text to display the interaction-manager debug messages.")]
        public Text debugText;

        // tracked userId
        private ulong playerUserID = 0;
        private ulong lastUserID = 0;

        // hand press properties
        private bool isLeftHandPrimary = false;
        private bool isRightHandPrimary = false;

        private bool isLeftHandPress = false;
        private bool isRightHandPress = false;

        private float lastLeftHandPressTime = 0f;
        private float lastRightHandPressTime = 0f;

        private float leftHandPressProgress = 0f;
        private float rightHandPressProgress = 0f;

        // cursor properties
        private Vector3 cursorScreenPos = Vector3.zero;
        private bool dragInProgress = false;

        private Image cursorProgressBar;
        private float cursorClickProgress = 0f;

        // hand states
        private KinectInterop.HandState leftHandState = KinectInterop.HandState.Unknown;
        private KinectInterop.HandState rightHandState = KinectInterop.HandState.Unknown;

        // left hand properties
        private HandEventType leftHandEvent = HandEventType.None;
        private HandEventType lastLeftHandEvent = HandEventType.Release;

        private Vector3 leftHandPos = Vector3.zero;
        private Vector3 leftHandScreenPos = Vector3.zero;
        private Vector3 leftIboxLeftBotBack = Vector3.zero;
        private Vector3 leftIboxRightTopFront = Vector3.zero;
        private bool isleftIboxValid = false;
        private bool isLeftHandInteracting = false;
        private float leftHandInteractingSince = 0f;

        // left hand click properties
        private Vector3 lastLeftHandPos = Vector3.zero;
        private float lastLeftHandClickTime = 0f;
        private bool isLeftHandClick = false;
        private float leftHandClickProgress = 0f;

        // left hand properties
        private HandEventType rightHandEvent = HandEventType.None;
        private HandEventType lastRightHandEvent = HandEventType.Release;

        private Vector3 rightHandPos = Vector3.zero;
        private Vector3 rightHandScreenPos = Vector3.zero;
        private Vector3 rightIboxLeftBotBack = Vector3.zero;
        private Vector3 rightIboxRightTopFront = Vector3.zero;
        private bool isRightIboxValid = false;
        private bool isRightHandInteracting = false;
        private float rightHandInteractingSince = 0f;

        // right hand click properties
        private Vector3 lastRightHandPos = Vector3.zero;
        private float lastRightHandClickTime = 0f;
        private bool isRightHandClick = false;
        private float rightHandClickProgress = 0f;

        // Bool to keep track whether Kinect and Interaction library have been initialized
        private bool interactionInited = false;

        // The single instance of FacetrackingManager
        //private static InteractionManager instance;

        // hand states
        private float lHandState = 0f;
        private float rHandState = 0f;


        ///// <summary>
        ///// Gets the single InteractionManager instance.
        ///// </summary>
        ///// <value>The InteractionManager instance.</value>
        //public static InteractionManager Instance
        //{
        //    get
        //    {
        //        return instance;
        //    }
        //}

        /// <summary>
        /// Gets the proper instance of InteractionManager, or null if no proper IM is found.
        /// </summary>
        /// <param name="playerIndex">Player index, tracked by the IM.</param>
        /// <param name="leftHandInteraction">Whether the IM tracks left-hand interactions or not.</param>
        /// <param name="rightHandInteraction">Whether the IM tracks right-hand interactions or not.</param>
        /// <returns></returns>
        public static InteractionManager GetInstance(int playerIndex, bool leftHandInteraction, bool rightHandInteraction)
        {
            // find the proper interaction manager
            MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];

            foreach (MonoBehaviour monoScript in monoScripts)
            {
                if ((monoScript is InteractionManager) && monoScript.enabled)
                {
                    InteractionManager intManager = (InteractionManager)monoScript;

                    if (intManager.playerIndex == playerIndex && 
                        (!leftHandInteraction || intManager.leftHandInteraction) && 
                        (!rightHandInteraction || intManager.rightHandInteraction))
                    {
                        return intManager;
                    }
                }
            }

            // not found
            return null;
        }

        /// <summary>
        /// Determines whether the InteractionManager was successfully initialized.
        /// </summary>
        /// <returns><c>true</c> if InteractionManager was successfully initialized; otherwise, <c>false</c>.</returns>
        public bool IsInteractionInited()
        {
            return interactionInited;
        }

        /// <summary>
        /// Gets the current user ID, or 0 if no user is currently tracked.
        /// </summary>
        /// <returns>The user ID</returns>
        public ulong GetUserID()
        {
            return playerUserID;
        }

        /// <summary>
        /// Gets the current left hand event (none, grip or release).
        /// </summary>
        /// <returns>The current left hand event.</returns>
        public HandEventType GetLeftHandEvent()
        {
            return leftHandEvent;
        }

        /// <summary>
        /// Gets the last detected left hand event (grip or release).
        /// </summary>
        /// <returns>The last left hand event.</returns>
        public HandEventType GetLastLeftHandEvent()
        {
            return lastLeftHandEvent;
        }

        /// <summary>
        /// Gets the current normalized viewport position of the left hand, in range [0, 1].
        /// </summary>
        /// <returns>The left hand viewport position.</returns>
        public Vector3 GetLeftHandScreenPos()
        {
            return leftHandScreenPos;
        }

        /// <summary>
        /// Determines whether the left hand is primary for the user.
        /// </summary>
        /// <returns><c>true</c> if the left hand is primary for the user; otherwise, <c>false</c>.</returns>
        public bool IsLeftHandPrimary()
        {
            return isLeftHandPrimary;
        }

        /// <summary>
        /// Determines whether the left hand is pressing.
        /// </summary>
        /// <returns><c>true</c> if the left hand is pressing; otherwise, <c>false</c>.</returns>
        public bool IsLeftHandPress()
        {
            return isLeftHandPress;
        }

        /// <summary>
        /// Determines whether a left hand click is detected, false otherwise.
        /// </summary>
        /// <returns><c>true</c> if a left hand click is detected; otherwise, <c>false</c>.</returns>
        public bool IsLeftHandClickDetected()
        {
            if (isLeftHandClick)
            {
                isLeftHandClick = false;
                cursorClickProgress = leftHandClickProgress = 0f;
                lastLeftHandPos = Vector3.zero;

                lastLeftHandClickTime = Time.realtimeSinceStartup;
                lastLeftHandPressTime = Time.realtimeSinceStartup;

                return true;
            }

            return false;
        }

        /// <summary>
        /// Gets the left hand click progress, in range [0, 1].
        /// </summary>
        /// <returns>The left hand click progress.</returns>
        public float GetLeftHandClickProgress()
        {
            return leftHandClickProgress;
        }

        /// <summary>
        /// Gets the current right hand event (none, grip or release).
        /// </summary>
        /// <returns>The current right hand event.</returns>
        public HandEventType GetRightHandEvent()
        {
            return rightHandEvent;
        }

        /// <summary>
        /// Gets the last detected right hand event (grip or release).
        /// </summary>
        /// <returns>The last right hand event.</returns>
        public HandEventType GetLastRightHandEvent()
        {
            return lastRightHandEvent;
        }

        /// <summary>
        /// Gets the current normalized viewport position of the right hand, in range [0, 1].
        /// </summary>
        /// <returns>The right hand viewport position.</returns>
        public Vector3 GetRightHandScreenPos()
        {
            return rightHandScreenPos;
        }

        /// <summary>
        /// Determines whether the right hand is primary for the user.
        /// </summary>
        /// <returns><c>true</c> if the right hand is primary for the user; otherwise, <c>false</c>.</returns>
        public bool IsRightHandPrimary()
        {
            return isRightHandPrimary;
        }

        /// <summary>
        /// Determines whether the right hand is pressing.
        /// </summary>
        /// <returns><c>true</c> if the right hand is pressing; otherwise, <c>false</c>.</returns>
        public bool IsRightHandPress()
        {
            return isRightHandPress;
        }

        /// <summary>
        /// Determines whether a right hand click is detected, false otherwise.
        /// </summary>
        /// <returns><c>true</c> if a right hand click is detected; otherwise, <c>false</c>.</returns>
        public bool IsRightHandClickDetected()
        {
            if (isRightHandClick)
            {
                isRightHandClick = false;
                cursorClickProgress = rightHandClickProgress = 0f;
                lastRightHandPos = Vector3.zero;

                lastRightHandClickTime = Time.realtimeSinceStartup;
                lastRightHandPressTime = Time.realtimeSinceStartup;

                return true;
            }

            return false;
        }

        /// <summary>
        /// Gets the right hand click progress, in range [0, 1].
        /// </summary>
        /// <returns>The right hand click progress.</returns>
        public float GetRightHandClickProgress()
        {
            return rightHandClickProgress;
        }

        /// <summary>
        /// Gets the current cursor normalized viewport position.
        /// </summary>
        /// <returns>The cursor viewport position.</returns>
        public Vector3 GetCursorPosition()
        {
            return cursorScreenPos;
        }

        /// <summary>
        /// Gets the cursor click progress, in range [0, 1].
        /// </summary>
        /// <returns>The right hand click progress.</returns>
        public float GetCursorClickProgress()
        {
            return cursorClickProgress;
        }


        //----------------------------------- end of public functions --------------------------------------//

        void Awake()
        {
            //instance = this;
        }


        void Start()
        {
            // get the progress bar reference if any
            GameObject objProgressBar = guiHandCursor && guiHandCursor.gameObject.transform.childCount > 0 ? guiHandCursor.transform.GetChild(0).gameObject : null;
            cursorProgressBar = objProgressBar ? objProgressBar.GetComponent<Image>() : null;

            interactionInited = true;

            // try to automatically detect the available interaction listeners in the scene
            if (interactionListeners.Count == 0)
            {
                MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];

                foreach (MonoBehaviour monoScript in monoScripts)
                {
                    //if (typeof(InteractionListenerInterface).IsAssignableFrom(monoScript.GetType()) &&
                    //    monoScript.enabled)
                        if ((monoScript is InteractionListenerInterface) && monoScript.enabled)
                    {
                        interactionListeners.Add(monoScript);
                    }
                }
            }

        }

        void OnDestroy()
        {
            interactionInited = false;
            //instance = null;
        }

        void Update()
        {
            KinectManager kinectManager = KinectManager.Instance;

            // update Kinect interaction
            if (kinectManager && kinectManager.IsInitialized())
            {
                playerUserID = kinectManager.GetUserIdByIndex(playerIndex);

                if (playerUserID != 0)
                {
                    lastUserID = playerUserID;
                    HandEventType handEvent = HandEventType.None;
                    float fTimeSmooth = 10f * Time.deltaTime;

                    // get the left hand state
                    int handState = (int)kinectManager.GetLeftHandState(playerUserID);
                    lHandState = Mathf.Lerp(lHandState, handState, fTimeSmooth);
                    leftHandState = (KinectInterop.HandState)Mathf.RoundToInt(lHandState);  // (KinectInterop.HandState)handState;  // 

                    // check if the left hand is interacting
                    isleftIboxValid = kinectManager.GetLeftHandInteractionBox(playerUserID, ref leftIboxLeftBotBack, ref leftIboxRightTopFront, isleftIboxValid);
                    //bool bLeftHandPrimaryNow = false;

                    // was the left hand interacting till now
                    bool wasLeftHandInteracting = isLeftHandInteracting;

                    if (isleftIboxValid && leftHandInteraction && //bLeftHandPrimaryNow &&
                       kinectManager.GetJointTrackingState(playerUserID, (int)KinectInterop.JointType.HandLeft) != KinectInterop.TrackingState.NotTracked)
                    {
                        leftHandPos = kinectManager.GetJointPosition(playerUserID, (int)KinectInterop.JointType.HandLeft);
                        leftHandScreenPos.z = Mathf.Clamp01((leftIboxLeftBotBack.z - leftHandPos.z) / (leftIboxLeftBotBack.z - leftIboxRightTopFront.z));

                        if (!handOverlayCursor)
                        {
                            leftHandScreenPos.x = Mathf.Clamp01((leftHandPos.x - leftIboxLeftBotBack.x) / (leftIboxRightTopFront.x - leftIboxLeftBotBack.x));
                            leftHandScreenPos.y = Mathf.Clamp01((leftHandPos.y - leftIboxLeftBotBack.y) / (leftIboxRightTopFront.y - leftIboxLeftBotBack.y));

                            isLeftHandInteracting = (leftHandPos.x >= (leftIboxLeftBotBack.x - 1.0f)) && (leftHandPos.x <= (leftIboxRightTopFront.x + 0.5f)) &&
                                (leftHandPos.y >= (leftIboxLeftBotBack.y - 0.1f)) && (leftHandPos.y <= (leftIboxRightTopFront.y + 0.7f)) &&
                                (leftIboxLeftBotBack.z >= leftHandPos.z) && (leftIboxRightTopFront.z * 0.8f <= leftHandPos.z);
                        }
                        else
                        {
                            isLeftHandInteracting = GetHandOverlayScreenPos(kinectManager, (int)KinectInterop.JointType.HandLeft, ref leftHandScreenPos) &&
                                (leftHandPos.y >= (leftIboxLeftBotBack.y - 0.15f)) && (leftHandPos.y <= (leftIboxRightTopFront.y + 0.7f)) &&
                                (leftIboxLeftBotBack.z >= leftHandPos.z) && (leftIboxRightTopFront.z * 0.8f <= leftHandPos.z);
                        }

                        //bLeftHandPrimaryNow = isLeftHandInteracting;
                        // start interacting?
                        if (!wasLeftHandInteracting && isLeftHandInteracting)
                        {
                            leftHandInteractingSince = Time.realtimeSinceStartup;
                        }

                        // check for left press
                        isLeftHandPress = leftHandScreenPos.z > 0.99f; // ((leftIboxRightTopFront.z - 0.1f) >= leftHandPos.z);
                        leftHandPressProgress = (Time.realtimeSinceStartup - lastLeftHandPressTime) >= KinectInterop.Constants.ClickStayDuration &&
                                                    leftHandScreenPos.z >= 0.7f ? (leftHandScreenPos.z - 0.7f) / 0.3f : 0f;

                        // check for left hand click
                        if (!dragInProgress && isLeftHandInteracting &&
                            ((allowHandClicks && ((leftHandPos - lastLeftHandPos).magnitude < KinectInterop.Constants.ClickMaxDistance)) ||
                                (allowPushToClick && leftHandPressProgress > 0f)))
                        {
                            if ((allowHandClicks && (Time.realtimeSinceStartup - lastLeftHandClickTime) >= KinectInterop.Constants.ClickStayDuration) ||
                                (allowPushToClick && leftHandPressProgress > 0.99f && isLeftHandPress))
                            {
                                if (!isLeftHandClick)
                                {
                                    isLeftHandClick = true;
                                    cursorClickProgress = leftHandClickProgress = 1f;

                                    foreach (InteractionListenerInterface listener in interactionListeners)
                                    {
                                        if (listener.HandClickDetected(playerUserID, playerIndex, false, leftHandScreenPos))
                                        {
                                            isLeftHandClick = false;
                                            cursorClickProgress = leftHandClickProgress = 0f;
                                            lastLeftHandPos = Vector3.zero;

                                            lastLeftHandClickTime = Time.realtimeSinceStartup;
                                            lastLeftHandPressTime = Time.realtimeSinceStartup;
                                        }
                                    }

                                    if (controlMouseCursor)
                                    {
                                        MouseControl.MouseClick();

                                        isLeftHandClick = false;
                                        cursorClickProgress = leftHandClickProgress = 0f;
                                        lastLeftHandPos = Vector3.zero;

                                        lastLeftHandClickTime = Time.realtimeSinceStartup;
                                        lastLeftHandPressTime = Time.realtimeSinceStartup;
                                    }
                                }
                            }
                            else
                            {
                                // show progress after the 1st half of the needed duration
                                float leftHandTimeProgress = allowHandClicks && (Time.realtimeSinceStartup - lastLeftHandClickTime) >= (KinectInterop.Constants.ClickStayDuration / 2f) ?
                                    ((Time.realtimeSinceStartup - lastLeftHandClickTime - (KinectInterop.Constants.ClickStayDuration / 2f)) * 2f / KinectInterop.Constants.ClickStayDuration) : 0f;
                                cursorClickProgress = leftHandClickProgress = allowPushToClick && leftHandScreenPos.z >= 0.7f ? leftHandPressProgress : leftHandTimeProgress;
                            }
                        }
                        else
                        {
                            isLeftHandClick = false;
                            leftHandClickProgress = 0f;
                            lastLeftHandPos = leftHandPos;
                            lastLeftHandClickTime = Time.realtimeSinceStartup;
                        }
                    }
                    else
                    {
                        isLeftHandInteracting = false;
                        isLeftHandPress = false;
                        leftHandPressProgress = 0f;
                    }

                    // get the right hand state
                    handState = (int)kinectManager.GetRightHandState(playerUserID);
                    rHandState = Mathf.Lerp(rHandState, handState, fTimeSmooth);
                    rightHandState = (KinectInterop.HandState)Mathf.RoundToInt(rHandState);

                    // check if the right hand is interacting
                    isRightIboxValid = kinectManager.GetRightHandInteractionBox(playerUserID, ref rightIboxLeftBotBack, ref rightIboxRightTopFront, isRightIboxValid);
                    //bool bRightHandPrimaryNow = false;

                    // was the right hand interacting till now
                    bool wasRightHandInteracting = isRightHandInteracting;

                    if (isRightIboxValid && rightHandInteraction && //bRightHandPrimaryNow &&
                       kinectManager.GetJointTrackingState(playerUserID, (int)KinectInterop.JointType.HandRight) != KinectInterop.TrackingState.NotTracked)
                    {
                        rightHandPos = kinectManager.GetJointPosition(playerUserID, (int)KinectInterop.JointType.HandRight);
                        rightHandScreenPos.z = Mathf.Clamp01((rightIboxLeftBotBack.z - rightHandPos.z) / (rightIboxLeftBotBack.z - rightIboxRightTopFront.z));

                        if (!handOverlayCursor)
                        {
                            rightHandScreenPos.x = Mathf.Clamp01((rightHandPos.x - rightIboxLeftBotBack.x) / (rightIboxRightTopFront.x - rightIboxLeftBotBack.x));
                            rightHandScreenPos.y = Mathf.Clamp01((rightHandPos.y - rightIboxLeftBotBack.y) / (rightIboxRightTopFront.y - rightIboxLeftBotBack.y));

                            isRightHandInteracting = (rightHandPos.x >= (rightIboxLeftBotBack.x - 0.5f)) && (rightHandPos.x <= (rightIboxRightTopFront.x + 1.0f)) &&
                                (rightHandPos.y >= (rightIboxLeftBotBack.y - 0.1f)) && (rightHandPos.y <= (rightIboxRightTopFront.y + 0.7f)) &&
                                (rightIboxLeftBotBack.z >= rightHandPos.z) && (rightIboxRightTopFront.z * 0.8f <= rightHandPos.z);
                        }
                        else
                        {
                            isRightHandInteracting = GetHandOverlayScreenPos(kinectManager, (int)KinectInterop.JointType.HandRight, ref rightHandScreenPos) &&
                                (rightHandPos.y >= (rightIboxLeftBotBack.y - 0.15f)) && (rightHandPos.y <= (rightIboxRightTopFront.y + 0.7f)) &&
                                (rightIboxLeftBotBack.z >= rightHandPos.z) && (rightIboxRightTopFront.z * 0.8f <= rightHandPos.z);
                        }

                        //bRightHandPrimaryNow = isRightHandInteracting;
                        if (!wasRightHandInteracting && isRightHandInteracting)
                        {
                            rightHandInteractingSince = Time.realtimeSinceStartup;
                        }

                        // check for right press
                        isRightHandPress = rightHandScreenPos.z > 0.99f; // ((rightIboxRightTopFront.z - 0.1f) >= rightHandPos.z);
                        rightHandPressProgress = (Time.realtimeSinceStartup - lastRightHandPressTime) >= KinectInterop.Constants.ClickStayDuration &&
                                                    rightHandScreenPos.z >= 0.7f ? (rightHandScreenPos.z - 0.7f) / 0.3f : 0f;

                        // check for right hand click
                        if (!dragInProgress && isRightHandInteracting &&
                            ((allowHandClicks && ((rightHandPos - lastRightHandPos).magnitude < KinectInterop.Constants.ClickMaxDistance)) ||
                                (allowPushToClick && rightHandPressProgress > 0f)))
                        {
                            if ((allowHandClicks && (Time.realtimeSinceStartup - lastRightHandClickTime) >= KinectInterop.Constants.ClickStayDuration) ||
                                (allowPushToClick && rightHandPressProgress > 0.99f && isRightHandPress))
                            {
                                if (!isRightHandClick)
                                {
                                    isRightHandClick = true;
                                    cursorClickProgress = rightHandClickProgress = 1f;

                                    foreach (InteractionListenerInterface listener in interactionListeners)
                                    {
                                        if (listener.HandClickDetected(playerUserID, playerIndex, true, rightHandScreenPos))
                                        {
                                            isRightHandClick = false;
                                            cursorClickProgress = rightHandClickProgress = 0f;
                                            lastRightHandPos = Vector3.zero;

                                            lastRightHandClickTime = Time.realtimeSinceStartup;
                                            lastRightHandPressTime = Time.realtimeSinceStartup;
                                        }
                                    }

                                    if (controlMouseCursor)
                                    {
                                        MouseControl.MouseClick();

                                        isRightHandClick = false;
                                        cursorClickProgress = rightHandClickProgress = 0f;
                                        lastRightHandPos = Vector3.zero;

                                        lastRightHandClickTime = Time.realtimeSinceStartup;
                                        lastRightHandPressTime = Time.realtimeSinceStartup;
                                    }
                                }
                            }
                            else
                            {
                                // show progress after the 1st half of the needed duration
                                float rightHandTimeProgress = allowHandClicks && (Time.realtimeSinceStartup - lastRightHandClickTime) >= (KinectInterop.Constants.ClickStayDuration / 2f) ?
                                    ((Time.realtimeSinceStartup - lastRightHandClickTime - (KinectInterop.Constants.ClickStayDuration / 2f)) * 2f / KinectInterop.Constants.ClickStayDuration) : 0f;
                                cursorClickProgress = rightHandClickProgress = allowPushToClick && rightHandScreenPos.z >= 0.7f ? rightHandPressProgress : rightHandTimeProgress;
                            }
                        }
                        else
                        {
                            isRightHandClick = false;
                            rightHandClickProgress = 0f;
                            lastRightHandPos = rightHandPos;
                            lastRightHandClickTime = Time.realtimeSinceStartup;
                        }
                    }
                    else
                    {
                        isRightHandInteracting = false;
                        isRightHandPress = false;
                        rightHandPressProgress = 0f;
                    }

                    // stop the cursor click progress, if both left and right hand are not clicking
                    if (leftHandClickProgress == 0f && rightHandClickProgress == 0f && cursorClickProgress > 0f)
                    {
                        cursorClickProgress = 0f;
                    }

                    // if both hands are interacting, check which one interacts longer than the other
                    if (isLeftHandInteracting && isRightHandInteracting)
                    {
                        if (rightHandInteractingSince <= leftHandInteractingSince)
                            isLeftHandInteracting = false;
                        else
                            isRightHandInteracting = false;
                    }

                    // if left hand just stopped interacting, send extra non-interaction event
                    if (wasLeftHandInteracting && !isLeftHandInteracting)
                    {
                        foreach (InteractionListenerInterface listener in interactionListeners)
                        {
                            if (lastLeftHandEvent == HandEventType.Grip)
                                listener.HandReleaseDetected(playerUserID, playerIndex, false, true, leftHandScreenPos);
                        }

                        lastLeftHandEvent = HandEventType.Release;
                    }


                    // if right hand just stopped interacting, send extra non-interaction event
                    if (wasRightHandInteracting && !isRightHandInteracting)
                    {
                        foreach (InteractionListenerInterface listener in interactionListeners)
                        {
                            if (lastRightHandEvent == HandEventType.Grip)
                                listener.HandReleaseDetected(playerUserID, playerIndex, true, true, rightHandScreenPos);
                        }

                        lastRightHandEvent = HandEventType.Release;
                    }


                    // process left hand
                    handEvent = HandStateToEvent(leftHandState, lastLeftHandEvent);

                    if ((isLeftHandInteracting != isLeftHandPrimary) || (isRightHandInteracting != isRightHandPrimary))
                    {
                        if (controlMouseCursor && dragInProgress)
                        {
                            MouseControl.MouseRelease();
                            dragInProgress = false;
                        }

                        lastLeftHandEvent = HandEventType.Release;
                        lastRightHandEvent = HandEventType.Release;
                    }

                    if (controlMouseCursor && (handEvent != lastLeftHandEvent))
                    {
                        if (controlMouseDrag && !dragInProgress && (handEvent == HandEventType.Grip))
                        {
                            dragInProgress = true;
                            MouseControl.MouseDrag();
                        }
                        else if (dragInProgress && (handEvent == HandEventType.Release))
                        {
                            MouseControl.MouseRelease();
                            dragInProgress = false;
                        }
                    }

                    leftHandEvent = handEvent;
                    if (handEvent != HandEventType.None)
                    {
                        // no clicks, while hand grip is detected
                        if (leftHandEvent == HandEventType.Grip && leftHandClickProgress > 0f)
                        {
                            cursorClickProgress = leftHandClickProgress = 0f;
                            lastLeftHandClickTime = Time.realtimeSinceStartup;
                        }

                        if (leftHandEvent != lastLeftHandEvent)
                        {
                            // invoke interaction listeners
                            foreach (InteractionListenerInterface listener in interactionListeners)
                            {
                                if (leftHandEvent == HandEventType.Grip)
                                    listener.HandGripDetected(playerUserID, playerIndex, false, isLeftHandInteracting, leftHandScreenPos);
                                else if (leftHandEvent == HandEventType.Release)
                                    listener.HandReleaseDetected(playerUserID, playerIndex, false, isLeftHandInteracting, leftHandScreenPos);
                            }
                        }

                        lastLeftHandEvent = handEvent;
                    }

                    // if the hand is primary, set the cursor position
                    if (isLeftHandInteracting)
                    {
                        isLeftHandPrimary = true;

                        if (leftHandClickProgress < 0.8f)  // stop the cursor after 80% click progress
                        {
                            float smooth = smoothFactor * Time.deltaTime;
                            if (smooth == 0f) smooth = 1f;

                            cursorScreenPos = Vector3.Lerp(cursorScreenPos, leftHandScreenPos, smooth);
                        }

                        // move mouse-only if there is no cursor texture
                        if (controlMouseCursor &&
                            (!guiHandCursor || (!gripHandTexture && !releaseHandTexture && !normalHandTexture)))
                        {
                            MouseControl.MouseMove(cursorScreenPos, debugText);
                        }
                    }
                    else
                    {
                        isLeftHandPrimary = false;
                    }


                    // process right hand
                    handEvent = HandStateToEvent(rightHandState, lastRightHandEvent);

                    if (controlMouseCursor && (handEvent != lastRightHandEvent))
                    {
                        if (controlMouseDrag && !dragInProgress && (handEvent == HandEventType.Grip))
                        {
                            dragInProgress = true;
                            MouseControl.MouseDrag();
                        }
                        else if (dragInProgress && (handEvent == HandEventType.Release))
                        {
                            MouseControl.MouseRelease();
                            dragInProgress = false;
                        }
                    }

                    rightHandEvent = handEvent;
                    if (handEvent != HandEventType.None)
                    {
                        // no clicks, while hand grip is detected
                        if (rightHandEvent == HandEventType.Grip && rightHandClickProgress > 0f)
                        {
                            cursorClickProgress = rightHandClickProgress = 0f;
                            lastRightHandClickTime = Time.realtimeSinceStartup;
                        }

                        if (rightHandEvent != lastRightHandEvent)
                        {
                            // invoke interaction listeners
                            foreach (InteractionListenerInterface listener in interactionListeners)
                            {
                                if (rightHandEvent == HandEventType.Grip)
                                    listener.HandGripDetected(playerUserID, playerIndex, true, isRightHandInteracting, rightHandScreenPos);
                                else if (rightHandEvent == HandEventType.Release)
                                    listener.HandReleaseDetected(playerUserID, playerIndex, true, isRightHandInteracting, rightHandScreenPos);
                            }
                        }

                        lastRightHandEvent = handEvent;
                    }

                    // if the hand is primary, set the cursor position
                    if (isRightHandInteracting)
                    {
                        isRightHandPrimary = true;

                        if (rightHandClickProgress < 0.8f)  // stop the cursor after 80% click progress
                        {
                            float smooth = smoothFactor * Time.deltaTime;
                            if (smooth == 0f) smooth = 1f;

                            cursorScreenPos = Vector3.Lerp(cursorScreenPos, rightHandScreenPos, smooth);
                        }

                        // move mouse-only if there is no cursor texture
                        if (controlMouseCursor &&
                            (!guiHandCursor || (!gripHandTexture && !releaseHandTexture && !normalHandTexture)))
                        {
                            MouseControl.MouseMove(cursorScreenPos, debugText);
                        }
                    }
                    else
                    {
                        isRightHandPrimary = false;
                    }

                }
                else
                {
                    // send release events
                    if (lastLeftHandEvent == HandEventType.Grip || lastRightHandEvent == HandEventType.Grip)
                    {
                        foreach (InteractionListenerInterface listener in interactionListeners)
                        {
                            if (lastLeftHandEvent == HandEventType.Grip)
                                listener.HandReleaseDetected(lastUserID, playerIndex, false, true, leftHandScreenPos);
                            if (lastRightHandEvent == HandEventType.Grip)
                                listener.HandReleaseDetected(lastUserID, playerIndex, true, true, leftHandScreenPos);
                        }
                    }

                    leftHandState = KinectInterop.HandState.NotTracked;
                    rightHandState = KinectInterop.HandState.NotTracked;

                    isLeftHandPrimary = isRightHandPrimary = false;
                    isLeftHandInteracting = isRightHandInteracting = false;
                    leftHandInteractingSince = rightHandInteractingSince = 0f;

                    isLeftHandClick = isRightHandClick = false;
                    cursorClickProgress = leftHandClickProgress = rightHandClickProgress = 0f;
                    lastLeftHandClickTime = lastRightHandClickTime = Time.realtimeSinceStartup;
                    lastLeftHandPressTime = lastRightHandPressTime = Time.realtimeSinceStartup;

                    isLeftHandPress = false;
                    isRightHandPress = false;

                    leftHandPressProgress = 0f;
                    rightHandPressProgress = 0f;

                    leftHandEvent = HandEventType.None;
                    rightHandEvent = HandEventType.None;

                    lastLeftHandEvent = HandEventType.Release;
                    lastRightHandEvent = HandEventType.Release;

                    if (controlMouseCursor && dragInProgress)
                    {
                        MouseControl.MouseRelease();
                        dragInProgress = false;
                    }
                }

                // update cursor texture and position
                UpdateGUI();
            }

        }


        // updates cursor texture and position
        private void UpdateGUI()
        {
            if (!interactionInited)
                return;

            // display debug information
            if (debugText)
            {
                string sGuiText = string.Empty;

                //if(isLeftHandPrimary)
                {
                    sGuiText += "L.Hand" + (isLeftHandInteracting ? "*: " : " : ") + leftHandScreenPos.ToString();

                    if (lastLeftHandEvent == HandEventType.Grip)
                    {
                        sGuiText += "  LeftGrip";
                    }
                    else if (lastLeftHandEvent == HandEventType.Release)
                    {
                        sGuiText += "  LeftRelease";
                    }

                    if (isLeftHandClick)
                    {
                        sGuiText += "  LeftClick";
                    }
                    //else if (leftHandClickProgress > 0.5f)
                    //{
                    //    sGuiText += String.Format("  {0:F0}%", leftHandClickProgress * 100);
                    //}

                    if (isLeftHandPress)
                    {
                        sGuiText += "  LeftPress";
                    }

                    //sGuiText += " " + leftHandClickProgress;
                }

                //if(isRightHandPrimary)
                {
                    sGuiText += "\nR.Hand" + (isRightHandInteracting ? "*: " : " : ") + rightHandScreenPos.ToString();

                    if (lastRightHandEvent == HandEventType.Grip)
                    {
                        sGuiText += "  RightGrip";
                    }
                    else if (lastRightHandEvent == HandEventType.Release)
                    {
                        sGuiText += "  RightRelease";
                    }

                    if (isRightHandClick)
                    {
                        sGuiText += "  RightClick";
                    }
                    //else if (rightHandClickProgress > 0.5f)
                    //{
                    //    sGuiText += String.Format("  {0:F0}%", rightHandClickProgress * 100);
                    //}

                    if (isRightHandPress)
                    {
                        sGuiText += "  RightPress";
                    }

                    //sGuiText += " " + rightHandClickProgress;
                }

                debugText.text = sGuiText;
            }

            // display the cursor status and position
            if (guiHandCursor)
            {
                Sprite cursorTexture = null;

                if (isLeftHandPrimary)
                {
                    if (lastLeftHandEvent == HandEventType.Grip)
                        cursorTexture = gripHandTexture;
                    else if (lastLeftHandEvent == HandEventType.Release)
                        cursorTexture = releaseHandTexture;
                }
                else if (isRightHandPrimary)
                {
                    if (lastRightHandEvent == HandEventType.Grip)
                        cursorTexture = gripHandTexture;
                    else if (lastRightHandEvent == HandEventType.Release)
                        cursorTexture = releaseHandTexture;
                }

                if (cursorTexture == null)
                {
                    cursorTexture = normalHandTexture;
                }

                if ((cursorTexture != null) /**&& (isLeftHandPrimary || isRightHandPrimary)*/)
                {
                    Vector2 posSprite;

                    if (controlMouseCursor)
                    {
                        MouseControl.MouseMove(cursorScreenPos, debugText);
                        posSprite = new Vector2(Input.mousePosition.x, Input.mousePosition.y);
                    }
                    else
                    {
                        Rect rectCanvas = guiHandCursor.canvas.pixelRect;
                        float canvasScale = guiHandCursor.canvas.scaleFactor;

                        posSprite = new Vector2(cursorScreenPos.x * rectCanvas.width / canvasScale, cursorScreenPos.y * rectCanvas.height / canvasScale);
                        //Debug.Log("PosCursor: " + posSprite + ", Rect: " + rectCanvas + ", Scale: " + canvasScale);
                    }

                    guiHandCursor.sprite = cursorTexture;
                    guiHandCursor.rectTransform.anchoredPosition = posSprite;

                    if (cursorProgressBar)
                    {
                        cursorProgressBar.fillAmount = cursorClickProgress;
                    }
                }
            }
        }


        // estimates screen cursor overlay position for the given hand
        private bool GetHandOverlayScreenPos(KinectManager kinectManager, int iHandJointIndex, ref Vector3 handScreenPos)
        {
            Vector3 posJointRaw = kinectManager.GetJointKinectPosition(playerUserID, iHandJointIndex, false);

            if (posJointRaw != Vector3.zero)
            {
                Vector2 posDepth = kinectManager.MapSpacePointToDepthCoords(sensorIndex, posJointRaw);
                ushort depthValue = kinectManager.GetDepthForPixel(sensorIndex, posDepth);

                if (posDepth != Vector2.zero && depthValue > 0)
                {
                    // depth pos to color pos
                    Vector2 posColor = kinectManager.MapDepthPointToColorCoords(sensorIndex, posDepth, depthValue);

                    if (posColor.x != 0f && !float.IsInfinity(posColor.x))
                    {
                        float clrImageW = kinectManager.GetColorImageWidth(sensorIndex);
                        float clrImageH = kinectManager.GetColorImageHeight(sensorIndex);

                        // get the color image x-offset and width (use the portrait background, if available)
                        float colorWidth = clrImageW;
                        float colorOfsX = 0f;

                        PortraitBackground portraitBack = PortraitBackground.Instance;
                        if (portraitBack && portraitBack.enabled)
                        {
                            colorWidth = clrImageH * clrImageH / clrImageW;
                            colorOfsX = (clrImageW - colorWidth) / 2f;
                        }

                        float xScaled = (posColor.x - colorOfsX) / colorWidth;
                        float yScaled = posColor.y / clrImageH;

                        handScreenPos.x = xScaled;
                        handScreenPos.y = 1f - yScaled;

                        return true;
                    }
                }
            }

            return false;
        }


        // converts hand state to hand event type
        public static HandEventType HandStateToEvent(KinectInterop.HandState handState, HandEventType lastEventType)
        {
            switch (handState)
            {
                case KinectInterop.HandState.Open:
                    return HandEventType.Release;

                case KinectInterop.HandState.Closed:
                case KinectInterop.HandState.Lasso:
                    return HandEventType.Grip;

                case KinectInterop.HandState.Unknown:
                    return lastEventType;
            }

            return HandEventType.None;
        }


    }
}