dwelling act 4 (live motion cap w/ kinect azure)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

284 lines
12 KiB

1 year ago
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using com.rfilkov.kinect;
namespace com.rfilkov.components
{
/// <summary>
/// SceneBlendRenderer provides volumetric rendering and lighting of the real environment, as seen by the sensor's color camera.
/// </summary>
public class SceneBlendRenderer : MonoBehaviour
{
[Tooltip("Added depth distance between the real environment and the virtual environment, in meters.")]
[Range(-0.5f, 0.5f)]
public float depthDistance = 0.1f;
[Tooltip("Index of the depth sensor that generates the color camera background. 0 is the 1st one, 1 - the 2nd one, etc.")]
public int sensorIndex = 0;
[Tooltip("Depth value in meters, used for invalid depth points.")]
public float invalidDepthValue = 5f;
[Tooltip("Whether to maximize the rendered object on the screen, or not.")]
private bool maximizeOnScreen = true;
[Tooltip("Whether to apply per-pixel lighting on the foreground, or not.")]
public bool applyLighting = false;
[Tooltip("Camera used to scale the mesh, to fill the camera's background. If left empty, it will default to the main camera in the scene.")]
public Camera foregroundCamera;
[Tooltip("Background image (if any) that needs to be overlayed by this blend renderer.")]
public UnityEngine.UI.RawImage backgroundImage;
[Tooltip("Reference to a background removal manager, if one is available in the scene.")]
public BackgroundRemovalManager backgroundRemovalManager = null;
// references to KM and data
private KinectManager kinectManager = null;
private KinectInterop.SensorData sensorData = null;
private Material matRenderer = null;
// depth image buffer (in depth camera resolution)
private ComputeBuffer depthImageBuffer = null;
// textures
private Texture alphaTex = null;
private Texture colorTex = null;
// lighting
private FragmentLighting lighting = new FragmentLighting();
// saved screen width & height
private int lastScreenW = 0;
private int lastScreenH = 0;
private int lastColorW = 0;
private int lastColorH = 0;
private float lastAnchorPos = 0f;
private Vector3 initialScale = Vector3.one;
// distances
private float distToBackImage = 0f;
private float distToTransform = 0f;
void Start()
{
kinectManager = KinectManager.Instance;
initialScale = transform.localScale;
// get distance to back image
if (backgroundImage)
{
Canvas canvas = backgroundImage.canvas;
if (canvas.renderMode == RenderMode.ScreenSpaceCamera)
distToBackImage = canvas.planeDistance;
else
distToBackImage = 0f;
}
// get distance to transform
distToTransform = transform.localPosition.z;
// set renderer material
Renderer meshRenderer = GetComponent<Renderer>();
if (meshRenderer)
{
Shader blendShader = Shader.Find("Kinect/ForegroundBlendShader");
if (blendShader != null)
{
matRenderer = new Material(blendShader);
meshRenderer.material = matRenderer;
}
}
// get sensor data
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData(sensorIndex);
}
if(foregroundCamera == null)
{
foregroundCamera = Camera.main;
}
// find scene lights
Light[] sceneLights = GameObject.FindObjectsOfType<Light>();
lighting.SetLightsAndBounds(sceneLights, transform.position, new Vector3(20f, 20f, 20f));
}
void OnDestroy()
{
if (sensorData != null && sensorData.colorDepthBuffer != null)
{
sensorData.colorDepthBuffer.Release();
sensorData.colorDepthBuffer = null;
}
if (depthImageBuffer != null)
{
//depthImageCopy = null;
depthImageBuffer.Release();
depthImageBuffer = null;
}
// release lighting resources
lighting.ReleaseResources();
}
void Update()
{
if (matRenderer == null || sensorData == null)
return;
if(alphaTex == null || alphaTex.width != sensorData.colorImageWidth || alphaTex.height != sensorData.colorImageHeight)
{
// alpha texture
alphaTex = backgroundRemovalManager != null ? backgroundRemovalManager.GetAlphaTex() : null;
if (alphaTex != null)
{
matRenderer.SetTexture("_AlphaTex", alphaTex);
}
}
if(colorTex == null || colorTex.width != sensorData.colorImageWidth || colorTex.height != sensorData.colorImageHeight)
{
// color texture
colorTex = sensorData.colorImageTexture;
if(backgroundRemovalManager != null)
{
colorTex = !backgroundRemovalManager.computeAlphaMaskOnly ? backgroundRemovalManager.GetForegroundTex() : alphaTex;
}
if (colorTex != null)
{
matRenderer.SetInt("_TexResX", colorTex.width);
matRenderer.SetInt("_TexResY", colorTex.height);
matRenderer.SetTexture("_ColorTex", colorTex);
}
}
if (colorTex == null)
return;
int bufferLength = sensorData.colorImageWidth * sensorData.colorImageHeight / 2;
if (sensorData.colorDepthBuffer == null || sensorData.colorDepthBuffer.count != bufferLength)
{
sensorData.colorDepthBuffer = new ComputeBuffer(bufferLength, sizeof(uint));
matRenderer.SetBuffer("_DepthMap", sensorData.colorDepthBuffer);
//Debug.Log("Created colorDepthBuffer with len: " + bufferLength);
}
matRenderer.SetFloat("_DepthDistance", depthDistance);
matRenderer.SetFloat("_InvDepthVal", invalidDepthValue);
int curScreenW = foregroundCamera ? foregroundCamera.pixelWidth : Screen.width;
int curScreenH = foregroundCamera ? foregroundCamera.pixelHeight : Screen.height;
if (lastScreenW != curScreenW || lastScreenH != curScreenH || lastColorW != sensorData.colorImageWidth || lastColorH != sensorData.colorImageHeight)
{
ScaleRendererTransform(curScreenW, curScreenH);
}
Vector2 anchorPos = backgroundImage ? backgroundImage.rectTransform.anchoredPosition : Vector2.zero;
float curAnchorPos = anchorPos.x + anchorPos.y; // Mathf.Abs(anchorPos.x) + Mathf.Abs(anchorPos.y);
if (Mathf.Abs(curAnchorPos - lastAnchorPos) >= 20f)
{
//Debug.Log("anchorPos: " + anchorPos + ", curAnchorPos: " + curAnchorPos + ", lastAnchorPos: " + lastAnchorPos + ", diff: " + Mathf.Abs(curAnchorPos - lastAnchorPos));
CenterRendererTransform(anchorPos, curAnchorPos);
}
// update lighting parameters
lighting.UpdateLighting(matRenderer, applyLighting);
}
// scales the renderer's transform properly
private void ScaleRendererTransform(int curScreenW, int curScreenH)
{
lastScreenW = curScreenW;
lastScreenH = curScreenH;
lastColorW = sensorData.colorImageWidth;
lastColorH = sensorData.colorImageHeight;
Vector3 localScale = transform.localScale;
if (maximizeOnScreen && foregroundCamera)
{
float objectZ = distToTransform; // transform.localPosition.z; // the transform should be a child of the camera
float screenW = foregroundCamera.pixelWidth;
float screenH = foregroundCamera.pixelHeight;
if (backgroundImage)
{
PortraitBackground portraitBack = backgroundImage.gameObject.GetComponent<PortraitBackground>();
if (portraitBack != null)
{
Rect backRect = portraitBack.GetBackgroundRect();
screenW = backRect.width;
screenH = backRect.height;
}
}
Vector3 vLeft = foregroundCamera.ScreenToWorldPoint(new Vector3(0f, screenH / 2f, objectZ));
Vector3 vRight = foregroundCamera.ScreenToWorldPoint(new Vector3(screenW, screenH / 2f, objectZ));
float distLeftRight = (vRight - vLeft).magnitude;
Vector3 vBottom = foregroundCamera.ScreenToWorldPoint(new Vector3(screenW / 2f, 0f, objectZ));
Vector3 vTop = foregroundCamera.ScreenToWorldPoint(new Vector3(screenW / 2f, screenH, objectZ));
float distBottomTop = (vTop - vBottom).magnitude;
localScale.x = distLeftRight / initialScale.x;
localScale.y = distBottomTop / initialScale.y;
//Debug.Log("SceneRenderer scale: " + localScale + ", screenW: " + screenW + ", screenH: " + screenH + ", objZ: " + objectZ +
// "\nleft: " + vLeft + ", right: " + vRight + ", bottom: " + vBottom + ", vTop: " + vTop +
// "\ndH: " + distLeftRight + ", dV: " + distBottomTop + ", initialScale: " + initialScale);
}
// scale according to color-tex resolution
//localScale.y = localScale.x * colorTex.height / colorTex.width;
// apply color image scale
Vector3 colorImageScale = kinectManager.GetColorImageScale(sensorIndex);
if (colorImageScale.x < 0f)
localScale.x = -localScale.x;
if (colorImageScale.y < 0f)
localScale.y = -localScale.y;
transform.localScale = localScale;
}
// centers the renderer's transform, according to the background image
private void CenterRendererTransform(Vector2 anchorPos, float curAnchorPos)
{
lastAnchorPos = curAnchorPos;
if (foregroundCamera && distToBackImage > 0f)
{
float objectZ = distToTransform; // transform.localPosition.z; // the transform should be a child of the camera
float screenW = sensorData.colorImageWidth; // foregroundCamera.pixelWidth;
float screenH = sensorData.colorImageHeight; // foregroundCamera.pixelHeight;
Vector2 screenCenter = new Vector2(screenW / 2f, screenH / 2f);
Vector2 anchorScaled = new Vector2(anchorPos.x * distToTransform / distToBackImage, anchorPos.y * distToTransform / distToBackImage);
Vector3 vCenter = foregroundCamera.ScreenToWorldPoint(new Vector3(screenCenter.x + anchorScaled.x, screenCenter.y + anchorScaled.y, objectZ));
transform.position = vCenter;
//Vector3 vLocalPos = transform.localPosition;
//string sLocalPos = string.Format("({0:F3}, {1:F3}, {2:F3})", vLocalPos.x, vLocalPos.y, vLocalPos.z);
//Debug.Log("SceneRenderer anchor: " + anchorPos + ", screenW: " + screenW + ", screenH: " + screenH + ", objZ: " + objectZ + ", localPos: " + sLocalPos);
}
}
}
}