Ok, intel real sense has good opportunity to deal with facial animation. May be I’ll buy it when it goes on sale. I do not need extremely precise facial animation with 120fps, kinect for me is fine.
In the case of not supported source file that I have found the default .fbx model contains 8 pre-rigged facial expressions at once.
http://s3.postimg.org/kq9p10eyb/default_model_2.jpg
It is not a big deal to make a new 3d model whatever I want and place several expressions to one .fbx file.
The main problem for me is to add a new facial expressions in the code and control other stuff like head orientation, eyes traking and so on.
AnimUnits.cs
using System;
using UnityEngine;
public struct AnimationUnits
{
public const int MaxNbAnimUnits = 6;
public Vector3 Au012;
public Vector3 Au345;
public AnimationUnits(float au0, float au1, float au2, float au3, float au4, float au5)
{
Au012 = new Vector3(au0, au1, au2);
Au345 = new Vector3(au3, au4, au5);
}
public float LipRaiser
{
get { return Au012[0]; }
set { Au012[0] = value; }
}
public float JawLowerer
{
get { return Au012[1]; }
set { Au012[1] = value; }
}
public float LipStretcher
{
get { return Au012[2]; }
set { Au012[2] = value; }
}
public float BrowLowerer
{
get { return Au345[0]; }
set { Au345[0] = value; }
}
public float LipCornerDepressor
{
get { return Au345[1]; }
set { Au345[1] = value; }
}
public float OuterBrowRaiser
{
get { return Au345[2]; }
set { Au345[2] = value; }
}
public float this[int i]
{
get
{
if (i < 0 || i > MaxNbAnimUnits)
throw new ArgumentOutOfRangeException("There is only " + MaxNbAnimUnits + " animation units but you requested the nb: " + i);
if (i < 3)
{
return Au012*;
}
return Au345[i - 3];
}
set
{
if (i < 0 || i > MaxNbAnimUnits)
throw new ArgumentOutOfRangeException("There is only " + MaxNbAnimUnits + " animation units but you requested the nb: " + i);
if (i < 3)
{
Au012* = value;
return;
}
Au345[i - 3] = value;
}
}
public static AnimationUnits operator +(AnimationUnits first, AnimationUnits second)
{
var animUnits = new AnimationUnits();
animUnits.Au012 = first.Au012 + second.Au012;
animUnits.Au345 = first.Au345 + second.Au345;
return animUnits;
}
}
FaceTrackingExample.cs
using UnityEngine;
public class FaceTrackingExample : MonoBehaviour
{
public float FaceTrackingTimeout = 0.1f;
public float TimeToReturnToDefault = 0.5f;
public float SmoothTime = 0.1f;
public KinectBinder Kinect;
public Transform Model;
public PoseAnimator ModelAnimator;
public GameObject Flames;
private Vector3 _position;
private Vector3 _rotation;
private Vector3 _smoothedRotation;
private Vector3 _currentPosVelocity;
private Vector3 _currentRotVelocity;
private AnimationUnits _animUnits, _targetAnimUnits;
private AnimationUnits _currentAuVelocity;
private bool _isInitialized;
private bool _hasNewData;
private float _waitTimer;
private float _timeOfLastFrame;
private float _gravityStartTime;
private AnimationUnits _startGravityAu;
private Vector3 _startGravityPos;
private Vector3 _startGravityRot;
private AnimationUnits _accAu;
private float] _morphCoefs = new float[7];
private Vector3 _userInitialPosition;
public enum TrackingMode
{
UserFace,
Gravity,
ComputerControlled,
}
public TrackingMode CurrentMode { get; set; }
void Start ()
{
_position = _userInitialPosition;
_rotation = Model.transform.rotation.eulerAngles;
Kinect.FaceTrackingDataReceived += ProcessFaceTrackingData;
}
void ProcessFaceTrackingData(float au0, float au1, float au2, float au3, float au4, float au5, float posX, float posY, float posZ, float rotX, float rotY, float rotZ)
{
_hasNewData = true;
var animUnits = new AnimationUnits(au0, au1, au2, au3, au4, au5);
_position = new Vector3(posX, posY, posZ);
_rotation = new Vector3(rotX, rotY, rotZ);
// We amplify the position to exagerate the head movements.
_position *= 10;
SetCurrentAUs(animUnits);
}
private void SetCurrentAUs(AnimationUnits animUnits)
{
const float weight = 0.8f;
for (int i = 0; i < 6; i++)
{
_accAu* = animUnits* * weight + _accAu* * (1 - weight);
}
animUnits = _accAu;
_targetAnimUnits.LipRaiser = MapLipRaiserValue(animUnits.LipRaiser);
_targetAnimUnits.JawLowerer = MapJawLowererValue(animUnits.JawLowerer);
_targetAnimUnits.LipStretcher = MapLipStretcherValue(animUnits.LipStretcher);
_targetAnimUnits.BrowLowerer = MapBrowLowererValue(animUnits);
_targetAnimUnits.LipCornerDepressor = MapLipCornerDepressorValue(animUnits.LipCornerDepressor);
_targetAnimUnits.OuterBrowRaiser = MapOuterBrowRaiserValue(animUnits.OuterBrowRaiser);
}
#region Au Range Calibration
/**
* Notes on the Animation Units remapping:
* In this part we simply that the raw data from the kinect and filter/re-mapped it.
* In some cases we amplify it, we in others we use a bit of logic to manipulate the data.
*
* The magic numbers only reflect what we found worked well for the experience we wanted to setup.
*/
// AU0
private float MapLipRaiserValue(float coef)
{
return coef;
}
// AU1
private float MapJawLowererValue(float coef)
{
return coef;
}
// AU2
private float MapLipStretcherValue(float coef)
{
return coef;
}
// AU3
private float MapBrowLowererValue(AnimationUnits animUnits)
{
if (animUnits.OuterBrowRaiser > 0f)
return Mathf.Clamp(animUnits.BrowLowerer - animUnits.OuterBrowRaiser, -1f, 1.7f);
return Mathf.Clamp(animUnits.BrowLowerer - 3 * animUnits.OuterBrowRaiser, -1f, 1.7f);
}
// AU4
private float MapLipCornerDepressorValue(float coef)
{
return 2 * coef;
}
// AU5
private float MapOuterBrowRaiserValue(float coef)
{
return Mathf.Clamp(coef * 2, -1f, 1f);
}
#endregion
void Update()
{
if (_hasNewData)
{
_hasNewData = false;
_timeOfLastFrame = Time.time;
ProcessFaceData();
}
else
{
float timeSinceLastFrame = Time.time - _timeOfLastFrame;
if (timeSinceLastFrame > TimeToReturnToDefault + FaceTrackingTimeout)
{
ManipulateMaskAutomatically();
}
else if (timeSinceLastFrame > FaceTrackingTimeout)
{
ReturnMaskToNeutralState();
}
}
UpdateTransform();
UpdateAUs();
CheckForSpecialPoses();
}
private void ProcessFaceData()
{
if (!_isInitialized)
{
_isInitialized = true;
InitializeUserData();
}
CurrentMode = TrackingMode.UserFace;
}
private void InitializeUserData()
{
_userInitialPosition = _position;
}
// Perform random faces automatically to fill the gaps whenever we do not have any input data from kinect.
private void ManipulateMaskAutomatically()
{
CurrentMode = TrackingMode.ComputerControlled;
_waitTimer -= Time.deltaTime;
if (_waitTimer > 0)
return;
_waitTimer = 3f;
_rotation = new Vector3(Random.Range(-10f, 10f), Random.Range(-10f, 10f), Random.Range(-10f, 10f));
_targetAnimUnits = new AnimationUnits(Random.Range(-1f, 1f), Random.Range(-1f, 1f), Random.Range(-1f, 1f),
Random.Range(-1f, 1f), Random.Range(-1f, 1f), Random.Range(-1f, 1f));
}
private void ReturnMaskToNeutralState()
{
if (CurrentMode != TrackingMode.Gravity)
{
InitializeGravity();
CurrentMode = TrackingMode.Gravity;
}
ApplyGravityToParams();
}
// By gravity we mean the force that will pull the mask back to its neutral state.
private void InitializeGravity()
{
_isInitialized = false;
_gravityStartTime = Time.time;
_startGravityAu = _animUnits;
_startGravityPos = _position;
_startGravityRot = _rotation;
}
private void ApplyGravityToParams()
{
float time = Mathf.Clamp01((Time.time - _gravityStartTime) / TimeToReturnToDefault);
_position = Vector3.Lerp(_startGravityPos, _userInitialPosition, time);
_rotation = Vector3.Lerp(_startGravityRot, new Vector3(0, 0, 0), time);
var animUnits = new AnimationUnits();
animUnits.Au012 = Vector3.Lerp(_startGravityAu.Au012, Vector3.zero, time);
animUnits.Au345 = Vector3.Lerp(_startGravityAu.Au345, Vector3.zero, time);
SetCurrentAUs(animUnits);
}
private void UpdateTransform()
{
// Apply some smoothing to both the position and rotation. The raw input data is quite noisy.
_smoothedRotation = Vector3.SmoothDamp(_smoothedRotation, _rotation, ref _currentRotVelocity, SmoothTime);
Model.rotation = Quaternion.Euler(_smoothedRotation);
Model.position = Vector3.SmoothDamp(Model.position, _position - _userInitialPosition, ref _currentPosVelocity, SmoothTime);
}
private void UpdateAUs()
{
// Smooth the animation units as the data received directly by the kinect is noisy.
_animUnits.Au012 = Vector3.SmoothDamp(_animUnits.Au012, _targetAnimUnits.Au012, ref _currentAuVelocity.Au012, SmoothTime);
_animUnits.Au345 = Vector3.SmoothDamp(_animUnits.Au345, _targetAnimUnits.Au345, ref _currentAuVelocity.Au345, SmoothTime);
UpdateLipRaiser(_animUnits.LipRaiser);
UpdateJawLowerer(_animUnits.JawLowerer);
UpdateLipStretcher(_animUnits.LipStretcher);
UpdateBrowLowerer(_animUnits.BrowLowerer);
UpdateLipCornerDepressor(_animUnits.LipCornerDepressor);
UpdateOuterBrowRaiser(_animUnits.OuterBrowRaiser);
}
#region Specific AU Updates
/**
* Note on animating the mask:
* In this example we use a pose animation technology to animate the mask.
* You could just as easily use unity animation system and control the animation timeline yourself,
* or control the animations by direct manipulation of the bones. Whatever suits you best.
*
* In general, we have one or two specific pose (animation) per Animation Unit (AU).
* This setup was simply motived by its ease of use.
*
* Finally you will notice that sometimes we do not use the full range or that we use some magic numbers.
* These were hand tweaked so that we could better exagerate certain expressions. In that case we decided
* to go benefit the user experience instead of plain data accuracy.
*/
private void UpdateLipRaiser(float coef)
{
_morphCoefs[0] = coef;
ModelAnimator.SetWeight(0, coef);
}
private void UpdateJawLowerer(float coef)
{
// The jaw lowerer animation unit has no negative range.
_morphCoefs[1] = Mathf.Max(0, coef);
ModelAnimator.SetWeight(1, Mathf.Max(0, coef));
}
private void UpdateLipStretcher(float coef)
{
// The lip stretcher animation has 2 animations simply because it was easier to design that way.
// One represents the Animation Unit range -1, 0] and the other is for [0, 1].
_morphCoefs[2] = Mathf.Clamp(-1.5f * coef, 0, 1.5f);
_morphCoefs[3] = Mathf.Clamp(coef, -0.7f, 1);
ModelAnimator.SetWeight(2, Mathf.Clamp(-1.5f * coef, 0, 1.5f));
ModelAnimator.SetWeight(3, Mathf.Clamp(coef, -0.7f, 1));
}
private void UpdateBrowLowerer(float coef)
{
_morphCoefs[4] = coef;
ModelAnimator.SetWeight(4, coef);
}
private void UpdateLipCornerDepressor(float coef)
{
_morphCoefs[5] = Mathf.Clamp(coef, -0.15f, 1);
ModelAnimator.SetWeight(5, Mathf.Clamp(coef, -0.15f, 1));
}
private void UpdateOuterBrowRaiser(float coef)
{
_morphCoefs[6] = coef;
ModelAnimator.SetWeight(6, coef);
}
#endregion
private void CheckForSpecialPoses()
{
if (IsHappy())
{
if (!Flames.activeSelf)
Flames.SetActive(true);
}
else
{
if (Flames.activeSelf)
Flames.SetActive(false);
}
}
private bool IsHappy()
{
return _morphCoefs[1] > 0.6f && _morphCoefs[6] > 0.35f;
}
}
KinectBinder.cs
using System;
using System.Diagnostics;
using DataConverter;
using UnityEngine;
using Debug = UnityEngine.Debug;
/// <summary>
/// The kinect binder creates the necessary setup for you to receive data for the kinect face tracking system directly.
///
/// Simply subscribe to the FaceTrackingDataReceived event to receive face tracking data.
/// Check FaceTrackingExample.cs for an example.
///
/// VideoFrameDataReceived and DepthFrameDataReceived events will give you the raw rbg/depth data from kinect cameras.
/// Check ImageFeedback.cs for an example.
/// </summary>
public class KinectBinder : MonoBehaviour
{
public delegate void FaceTrackingDataDelegate(float au0, float au1, float au2, float au3, float au4, float au5, float posX, float posY, float posZ, float rotX, float rotY, float rotZ);
public event FaceTrackingDataDelegate FaceTrackingDataReceived;
public delegate void VideoFrameDataDelegate(Color32] pixels);
public event VideoFrameDataDelegate VideoFrameDataReceived;
public delegate void DepthFrameDataDelegate(short] pixels);
public event DepthFrameDataDelegate DepthFrameDataReceived;
public delegate void SkeletonDataDelegate(JointData] jointsData);
public event SkeletonDataDelegate SkeletonDataReceived;
private float _timeOfLastFrame;
private int _frameNumber = -1;
private int _processedFrame = -1;
private Process _otherProcess;
private int _kinectFps;
private int _kinectLastFps;
private float _kinectFpsTimer;
private bool _hasNewVideoContent;
private bool _hasNewDepthContent;
private string _faceTrackingData;
private string _skeletonData;
private short] _depthBuffer;
private Color32] _colorBuffer;
private JointData] _jointsData;
// Use this for initialization
void Start()
{
BootProcess();
}
private void BootProcess()
{
const string dataTransmitterFilename = "KinectDataTransmitter.exe";
string path = Application.dataPath + @"/../Kinect/";
_otherProcess = new Process();
_otherProcess.StartInfo.FileName = path + dataTransmitterFilename;
_otherProcess.StartInfo.UseShellExecute = false;
_otherProcess.StartInfo.CreateNoWindow = true;
_otherProcess.StartInfo.RedirectStandardInput = true;
_otherProcess.StartInfo.RedirectStandardOutput = true;
_otherProcess.StartInfo.RedirectStandardError = true;
_otherProcess.OutputDataReceived += (sender, args) => ParseReceivedData(args.Data);
_otherProcess.ErrorDataReceived += (sender, args) => Debug.LogError(args.Data);
try
{
_otherProcess.Start();
}
catch (Exception)
{
Debug.LogWarning(
"Could not find the kinect data transmitter. Please read the readme.txt for the setup instructions.");
_otherProcess = null;
enabled = false;
return;
}
_otherProcess.BeginOutputReadLine();
_otherProcess.StandardInput.WriteLine("1"); // gets rid of the Byte-order mark in the pipe.
}
void ParseReceivedData(string data)
{
if (Converter.IsFaceTrackingData(data))
{
_faceTrackingData = data;
}
else if (Converter.IsSkeletonData(data))
{
_skeletonData = data;
}
else if (Converter.IsVideoFrameData(data))
{
_hasNewVideoContent = true;
}
else if (Converter.IsDepthFrameData(data))
{
_hasNewDepthContent = true;
}
else if (Converter.IsPing(data))
{
if (_otherProcess != null && !_otherProcess.HasExited)
{
_otherProcess.StandardInput.WriteLine(Converter.EncodePingData());
}
}
else if (Converter.IsError(data))
{
Debug.LogError(Converter.GetDataContent(data));
}
else if (Converter.IsInformationMessage(data))
{
Debug.Log("Kinect (information message): " + Converter.GetDataContent(data));
}
else
{
Debug.LogWarning("Received this (unknown) message from kinect: " + data);
}
}
void Update()
{
if (_otherProcess == null || _otherProcess.HasExited)
{
Debug.LogWarning("KinectDataTransmitter has exited. Trying to reboot the process...");
BootProcess();
}
bool hasNewData = (_frameNumber > _processedFrame);
if (hasNewData)
{
_kinectFps += _frameNumber - _processedFrame;
_processedFrame = _frameNumber;
}
if (_hasNewVideoContent)
{
_hasNewVideoContent = false;
ProcessVideoFrame(Converter.GetVideoStreamData());
}
if (_hasNewDepthContent)
{
_hasNewDepthContent = false;
ProcessDepthFrame(Converter.GetDepthStreamData());
}
if (_faceTrackingData != null)
{
string data = _faceTrackingData;
_faceTrackingData = null;
ProcessFaceTrackingData(Converter.GetDataContent(data));
}
if (_skeletonData != null)
{
string data = _skeletonData;
_skeletonData = null;
ProcessSkeletonData(Converter.GetDataContent(data));
}
UpdateFrameCounter();
}
private void ProcessDepthFrame(byte] bytes)
{
if (DepthFrameDataReceived == null || bytes == null)
return;
if (_depthBuffer == null || _depthBuffer.Length != bytes.Length/2)
{
_depthBuffer = new short[bytes.Length / 2];
}
for (int i = 0; i < _depthBuffer.Length; i++)
{
int byteIndex = i * 2;
_depthBuffer* = BitConverter.ToInt16(bytes, byteIndex);
}
DepthFrameDataReceived(_depthBuffer);
}
private void ProcessVideoFrame(byte] bytes)
{
if (VideoFrameDataReceived == null || bytes == null)
return;
if (_colorBuffer == null || _colorBuffer.Length != bytes.Length / 4)
{
_colorBuffer = new Color32[bytes.Length / 4];
}
for (int i = 0; i < _colorBuffer.Length; i++)
{
int byteIndex = i*4;
_colorBuffer* = new Color32(bytes[byteIndex+2], bytes[byteIndex+1], bytes[byteIndex], byte.MaxValue);
}
VideoFrameDataReceived(_colorBuffer);
}
private void ProcessFaceTrackingData(string data)
{
if (FaceTrackingDataReceived == null)
return;
_frameNumber++;
float au0, au1, au2, au3, au4, au5, posX, posY, posZ, rotX, rotY, rotZ;
Converter.DecodeFaceTrackingData(data, out au0, out au1, out au2, out au3, out au4, out au5, out posX,
out posY, out posZ, out rotX, out rotY, out rotZ);
FaceTrackingDataReceived(au0, au1, au2, au3, au4, au5, posX, posY, posZ, rotX, rotY, rotZ);
}
private void ProcessSkeletonData(string data)
{
if (SkeletonDataReceived == null)
return;
_frameNumber++;
if (_jointsData == null)
{
_jointsData = new JointData(int)JointType.NumberOfJoints];
}
Converter.DecodeSkeletonData(data, _jointsData);
SkeletonDataReceived(_jointsData);
}
private void UpdateFrameCounter()
{
_kinectFpsTimer -= Time.deltaTime;
if (_kinectFpsTimer <= 0f)
{
_kinectLastFps = _kinectFps;
_kinectFps = 0;
_kinectFpsTimer = 1;
}
}
void OnGUI()
{
if (Event.current.type != EventType.Repaint)
return;
GUI.color = Color.white;
GUI.Label(new Rect(5, 5, 250, 30), "Kinect FPS: " + _kinectLastFps);
if (_kinectLastFps == 0)
{
GUI.Label(new Rect(5, 25, 400, 30), "(Kinect is not tracking... please get in range.)");
}
}
void OnApplicationQuit()
{
ShutdownKinect();
}
private void ShutdownKinect()
{
if (_otherProcess == null)
return;
try
{
Process.GetProcessById(_otherProcess.Id);
}
catch (ArgumentException)
{
// The other app might have been shut down externally already.
return;
}
try
{
_otherProcess.CloseMainWindow();
_otherProcess.Close();
}
catch (InvalidOperationException)
{
// The other app might have been shut down externally already.
}
}
}
I want to apply facial animation to the head of a whole 3d character.
In case of FaceShift I haven’t got any clue how to stream data to UE4.