Here is the code for the blendshape player. The blendshape player adjusts the blendshape values based on the keyframes exported from the plugin uploaded earlier.
The blendshape player consists of:
-
FaceAnimationPlayer.h / FaceAnimationPlayer.cpp (Handles playing the keyframe data)
-
FFaceAnimationData (struct containing keyframe table and audio references)
-
FKeyframeData (struct holding all preset control rig names that hold keyframe data)
How to use: Create an instance of the FaceAnimationPlayer class. Send a reference of the skeletal mesh containing the blendshapes into InitPlayer(). Play animation by sending a reference of the data table into PlayNewAnimation(). Your animation will play immediately.
keyframe structs to hold keyframes in a data table
//Blendshape Face Animation
USTRUCT(BlueprintType)
struct FFaceAnimationData : public FTableRowBase
{
GENERATED_USTRUCT_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Anim", meta = (ToolTip = "Animation data reference"))
UDataTable* keyframeData; //Reference to keyframe data struct holding the blendshape keyframes
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Anim", meta = (ToolTip = "Audio file to play"))
USoundCue* voiceAudio;
FFaceAnimationData()
{
}
};
//------------------------------------------------------------------
// Face animation keyframe data for blendshapes for single control
//------------------------------------------------------------------
USTRUCT(BlueprintType)
struct FKeyframeData : public FTableRowBase
{
GENERATED_USTRUCT_BODY()
//Name: frameNumber
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString chin;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString nose;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString brow;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString topBrow;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString middleMouth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString topMouth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftBottomMouth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftTopMouth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftBottomMouthTwo;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftTopMouthTwo;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftCornerMouth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftJaw;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftBottomCheek;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftMiddleCheek;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftTopCheek;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftBottomEyeRidge;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftEyeRidge;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString leftTopBrow;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightBottomMouth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightTopMouth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightBottomMouthTwo;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightTopMouthTwo;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightCornerMouth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightJaw;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightBottomCheek;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightMiddleCheek;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightTopCheek;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightBottomEyeRidge;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightEyeRidge;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rightTopBrow;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rEye;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString lEye;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rBottomLid;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rTopLid;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString lBottomLid;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString lTopLid;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rBrow1;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rBrow2;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString rBrow3;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString lBrow1;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString lBrow2;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString lBrow3;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString mShape;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString wShape;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Keyframe Data")
FString tShape;
FKeyframeData()
{
}
};
FaceAnimationPlayer.h
// Candax Productions
#pragma once
#include "GameFramework/Actor.h"
#include "MyStaticLibrary.h" //FaceAnimation Struct
#include "FaceAnimationPlayer.generated.h"
/**
* Blendshape Animation player
*/
UCLASS(Blueprintable)
class BFPONLINE_API AFaceAnimationPlayer : public AActor
{
GENERATED_UCLASS_BODY()
public:
virtual void BeginPlay() override;
virtual void Tick(float DeltaSeconds) override;
//---------------------------------------------------------
// PROPERTIES
//---------------------------------------------------------
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Anim")
FFaceAnimationData animData;
UPROPERTY(VisibleAnywhere, BlueprintReadWrite, Category = "Mesh")
USkeletalMeshComponent* sMeshComponent;
UPROPERTY(VisibleAnywhere, BlueprintReadWrite, Category = "Audio")
UAudioComponent* voiceAudioComponent;
UPROPERTY(VisibleAnywhere, BlueprintReadWrite, Category = "Audio")
USoundCue* voiceAudio;
UPROPERTY(VisibleAnywhere, BlueprintReadWrite, Category = "Animation")
float fps;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
float deltaTime;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 currentFrame;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 maxKeyframes;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
float lastTime;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
float currentTime;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
float totalTime;
//Loops
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 loops;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 maxLoops;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 lastLoop;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
bool finishedCycle;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 framesPassed;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 useableFrame;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
bool bHasStarted;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
bool bUseJointRotation;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
float animationScale;
//Blendshape Controls
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 curControl;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 maxControls;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
FName controlName;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
int32 additionalBlendshapeIndex;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
TArray<int32> AdditionalBlendshapeControls;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Animation")
TArray<FName> AdditionalBlendshapeNames;
//---------------------------------------------------------
// FUNCTIONS
//---------------------------------------------------------
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
void InitPlayer(USkeletalMeshComponent* meshC);
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
void ReformatMinMax(FString keyframeData);
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
void GetMaxKeyframe();
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
bool LimitFPSRate();
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
bool PlayNewAnimation(FFaceAnimationData animation, int32 newMaxLoops);
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
bool ResetAnim();
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
void AnimationCycle();
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
void AddAdditionalBlendshape(FName blendshapeName, int32 controlID);
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
FString GetFaceAnimKeyframe(UDataTable* dataTable, FName frameNumber, int32 controlNum);
UFUNCTION(BlueprintCallable, Category = "Blendshape Anim Player")
FName GetFaceAnimControlName(int32 controlNum);
private:
float minXValue;
float maxXValue;
float minYValue;
float maxYValue;
};
FaceAnimationPlayer.cpp
// Candax Productions
#include "BFPOnline.h"
#include "FaceAnimationPlayer.h"
AFaceAnimationPlayer::AFaceAnimationPlayer(const class FObjectInitializer& PCIP)
: Super(PCIP)
{
//Enable Ticking
PrimaryActorTick.bCanEverTick = true;
PrimaryActorTick.bStartWithTickEnabled = true;
PrimaryActorTick.bAllowTickOnDedicatedServer = true;
sMeshComponent = PCIP.CreateDefaultSubobject<USkeletalMeshComponent>(this, TEXT("SkeletalMesh"));
sMeshComponent->AttachTo(RootComponent);
}
void AFaceAnimationPlayer::BeginPlay()
{
Super::BeginPlay();
fps = 30.0f;
lastLoop = -1;
maxControls = 44;
finishedCycle = true;
animationScale = 1.0f;
}
void AFaceAnimationPlayer::InitPlayer(USkeletalMeshComponent* meshC)
{
sMeshComponent = meshC;
//Add additional blendshapes
AddAdditionalBlendshape("eyesMesh_rEye", 30);
AddAdditionalBlendshape("eyesMesh_lEye", 31);
AddAdditionalBlendshape("teethMesh_chin", 0);
}
void AFaceAnimationPlayer::AddAdditionalBlendshape(FName blendshapeName, int32 controlID){
AdditionalBlendshapeNames.Add(blendshapeName);
AdditionalBlendshapeControls.Add(controlID);
}
bool AFaceAnimationPlayer::PlayNewAnimation(FFaceAnimationData animation, int32 newMaxLoops)
{
ResetAnim();
maxLoops = newMaxLoops;
animData = animation;
GetMaxKeyframe();
//Set voice audio
voiceAudio = animData.voiceAudio;
bHasStarted = true;
return true;
}
void AFaceAnimationPlayer::Tick(float DeltaSeconds)
{
Super::Tick(DeltaSeconds);
//----------------------------------
//Play Animation
//----------------------------------
deltaTime = DeltaSeconds;
//Wait until an animation has started
if (bHasStarted){
//Accumulate delta time
totalTime += deltaTime;
AnimationCycle();
}
}
void AFaceAnimationPlayer::AnimationCycle(){
//Wait until frame has finished processing
if (finishedCycle){
finishedCycle = false;
//Set keyframe based on fps
LimitFPSRate();
//Keyframes table
UDataTable* tableRef = animData.keyframeData;
//-------------------------------------------------
//Loop through all predefined blendshape controls
//-------------------------------------------------
for (int32 i = 0; i < maxControls; i++){
//Get the value for the current keyframe
FName useableFrameName = FName(*FString::FromInt(useableFrame));
FString curKey = GetFaceAnimKeyframe(tableRef, useableFrameName, i);
//Reformat min max keyframe values
ReformatMinMax(curKey);
//Get the controlname from the id
FName curControlName = GetFaceAnimControlName(i);
//Set Morph Targets
sMeshComponent->SetMorphTarget(FName(*(curControlName.ToString() + "MinX")), minXValue);
sMeshComponent->SetMorphTarget(FName(*(curControlName.ToString() + "MaxX")), maxXValue);
sMeshComponent->SetMorphTarget(FName(*(curControlName.ToString() + "MinY")), minYValue);
sMeshComponent->SetMorphTarget(FName(*(curControlName.ToString() + "MaxY")), maxYValue);
}
//-------------------------------------------------
//Loop through additional blendshape controls
//-------------------------------------------------
for (int32 j = 0; j < AdditionalBlendshapeControls.Num(); j++){
//Get the value for the current keyframe
FName useableFrameName = FName(*FString::FromInt(j));
FString curKey = GetFaceAnimKeyframe(tableRef, useableFrameName, AdditionalBlendshapeControls[j]);
//Reformat min max keyframe values
ReformatMinMax(curKey);
//Get the controlname from the id
FName curControlName = AdditionalBlendshapeNames[j];
//Set Morph Targets
sMeshComponent->SetMorphTarget(FName(*(curControlName.ToString() + "MinX")), minXValue);
sMeshComponent->SetMorphTarget(FName(*(curControlName.ToString() + "MaxX")), maxXValue);
sMeshComponent->SetMorphTarget(FName(*(curControlName.ToString() + "MinY")), minYValue);
sMeshComponent->SetMorphTarget(FName(*(curControlName.ToString() + "MaxY")), maxYValue);
}
//Complete Cycle
curControl = 0;
finishedCycle = true;
}
}
void AFaceAnimationPlayer::ReformatMinMax(FString keyframeData)
{
//Handle null keyframes
if (keyframeData == "")
{
//Null - Return;
}
//Split keyframe data
FString xLitVal = "";
FString yLitVal = "";
keyframeData.Split(TEXT("x"), &xLitVal, &yLitVal, ESearchCase::IgnoreCase, ESearchDir::FromStart);
//Convert keyframe into float
float xVal = FCString::Atof(*xLitVal);
float yVal = FCString::Atof(*yLitVal);
//Scale animation
yVal = animationScale * yVal;
xVal = animationScale * xVal;
//Greater or less than X
if (xVal > 0){
minXValue = 0;
maxXValue = xVal;
}
else{
minXValue = xVal * -1;
maxXValue = 0;
}
//Greater or less than Y
if (yVal > 0){
minYValue = 0;
maxYValue = yVal;
}
else{
minYValue = yVal * -1;
maxYValue = 0;
}
}
void AFaceAnimationPlayer::GetMaxKeyframe()
{
UDataTable* tableRef = animData.keyframeData;
FString fMaxKeyframes = GetFaceAnimKeyframe(tableRef, "maxFrames", 0);
maxKeyframes = FCString::Atoi(*fMaxKeyframes);
maxKeyframes--; //Reduce frame by 1 to use index 0 as 1
//------------------------------------------------
// Can get and store all keyframe data locally *
//------------------------------------------------
}
bool AFaceAnimationPlayer::LimitFPSRate()
{
//Convert fps to milliseconds
float fpsMil = 1000 / fps;
//Add delta time to current time
currentTime += deltaTime * 1000;
//Check if elapsed time is greater than the last time + a new frame time
if (currentTime > (lastTime + fpsMil)){
//Determine if we have to skip keyframes based on time elapsed
framesPassed = FMath::FloorToInt(fps * totalTime);
//Go to next frame
lastTime = currentTime;
//Increment frame counter
currentFrame++;
if (currentFrame >= FMath::FloorToInt(fps)){
currentFrame = 0;
}
//Set the amount of times the animation has looped
loops = framesPassed / maxKeyframes;
//Check if we have started a new loop
if (lastLoop < loops){
//Only play set number of loops
if (maxLoops > 0){
//has gone over defined maxloops
if (loops >= maxLoops)
{
bHasStarted = false; //Stop Play
return false;
}
}
//Infinite Loop
lastLoop = loops;
//Replay Audio
UGameplayStatics::PlaySoundAtLocation(this, voiceAudio, sMeshComponent->GetComponentLocation());
}
//Get the keyframe to use
useableFrame = framesPassed - (loops * maxKeyframes);
}
else{
//Have to wait
}
return true;
}
bool AFaceAnimationPlayer::ResetAnim()
{
bHasStarted = false;
currentFrame = 0;
loops = 0;
lastLoop = -1;
finishedCycle = true;
curControl = 0;
framesPassed = 0;
totalTime = 0.0f;
return true;
}
FString AFaceAnimationPlayer::GetFaceAnimKeyframe(UDataTable* dataTable, FName frameNumber, int32 controlNum)
{
if (frameNumber != NAME_None){
FKeyframeData newData = *dataTable->FindRow<FKeyframeData>(frameNumber, "", false);
//Get the keyframe data from the control
//The id of the blendshape name
switch (controlNum){
case 0:
return newData.chin;
case 1:
return newData.nose;
case 2:
return newData.brow;
case 3:
return newData.topBrow;
case 4:
return newData.middleMouth;
case 5:
return newData.topMouth;
case 6:
return newData.leftBottomMouth;
case 7:
return newData.leftTopMouth;
case 8:
return newData.leftBottomMouthTwo;
case 9:
return newData.leftTopMouthTwo;
case 10:
return newData.leftCornerMouth;
case 11:
return newData.leftJaw;
case 12:
return newData.leftBottomCheek;
case 13:
return newData.leftMiddleCheek;
case 14:
return newData.leftTopCheek;
case 15:
return newData.leftBottomEyeRidge;
case 16:
return newData.leftEyeRidge;
case 17:
return newData.leftTopBrow;
case 18:
return newData.rightBottomMouth;
case 19:
return newData.rightTopMouth;
case 20:
return newData.rightBottomMouthTwo;
case 21:
return newData.rightTopMouthTwo;
case 22:
return newData.rightCornerMouth;
case 23:
return newData.rightJaw;
case 24:
return newData.rightBottomCheek;
case 25:
return newData.rightMiddleCheek;
case 26:
return newData.rightTopCheek;
case 27:
return newData.rightBottomEyeRidge;
case 28:
return newData.rightEyeRidge;
case 29:
return newData.rightTopBrow;
case 30:
return newData.rEye;
case 31:
return newData.lEye;
case 32:
return newData.rBottomLid;
case 33:
return newData.rTopLid;
case 34:
return newData.lBottomLid;
case 35:
return newData.lTopLid;
case 36:
return newData.rBrow1;
case 37:
return newData.rBrow2;
case 38:
return newData.rBrow3;
case 39:
return newData.lBrow1;
case 40:
return newData.lBrow2;
case 41:
return newData.lBrow3;
case 42:
return newData.mShape;
case 43:
return newData.wShape;
case 44:
return newData.tShape;
}
return "";
}
else
return "";
}
FName AFaceAnimationPlayer::GetFaceAnimControlName(int32 controlNum)
{
//Get the keyframe data from the control
switch (controlNum){
case 0:
return "chin";
case 1:
return "nose";
case 2:
return "brow";
case 3:
return "topBrow";
case 4:
return "middleMouth";
case 5:
return "topMouth";
case 6:
return "leftBottomMouth";
case 7:
return "leftTopMouth";
case 8:
return "leftBottomMouthTwo";
case 9:
return "leftTopMouthTwo";
case 10:
return "leftCornerMouth";
case 11:
return "leftJaw";
case 12:
return "leftBottomCheek";
case 13:
return "leftMiddleCheek";
case 14:
return "leftTopCheek";
case 15:
return "leftBottomEyeRidge";
case 16:
return "leftEyeRidge";
case 17:
return "leftTopBrow";
case 18:
return "rightBottomMouth";
case 19:
return "rightTopMouth";
case 20:
return "rightBottomMouthTwo";
case 21:
return "rightTopMouthTwo";
case 22:
return "rightCornerMouth";
case 23:
return "rightJaw";
case 24:
return "rightBottomCheek";
case 25:
return "rightMiddleCheek";
case 26:
return "rightTopCheek";
case 27:
return "rightBottomEyeRidge";
case 28:
return "rightEyeRidge";
case 29:
return "rightTopBrow";
case 30:
return "rEye";
case 31:
return "lEye";
case 32:
return "rBottomLid";
case 33:
return "rTopLid";
case 34:
return "lBottomLid";
case 35:
return "lTopLid";
case 36:
return "rBrow1";
case 37:
return "rBrow2";
case 38:
return "rBrow3";
case 39:
return "lBrow1";
case 40:
return "lBrow2";
case 41:
return "lBrow3";
case 42:
return "mShape";
case 43:
return "wShape";
case 44:
return "tShape";
}
//Default
return "chin";
}
