Hi everyone,
I am trying to play a live video capture of my webcam from raw data (uint8 array of pixels data) from another computer (image data received via TCP). The data coming in is an RGB array (no alpha, so 640x480x3 values). I got something working using a base actor class, a Texture2D, a billboard component and its attached sprite, but the video feed is delayed by around 0.5s-1s. I am new to Unreal Engine, so I assume I am doing something wrong, or in an un-optimized way. I know that you can get videos playing with minimal delays. In other terms, I believe what I am trying to achieve is updating a 2D texture at runtime, at each frame, that is why I call *UpdateTextureRegions *in the *Tick *method, followed by readjusting the Sprite texture pointer.
There is a lot of unnecessary code around, but I will post it anyway so you have the full picture of what I am trying to do. I am getting the image data from my callback function, which follows a publish/subscribe design pattern.
Please let me know if you know how I can improve this to have minimal delay in displaying the video feed.
**screen.h file: **
#pragma once
// Unreal
#include "CoreMinimal.h"
#include "GameFramework/Actor.h"
#include "Engine/Texture2DDynamic.h"
// ROS plugin headers
#include "ROSIntegration/Classes/RI/Topic.h"
#include "ROSIntegration/Classes/ROSIntegrationGameInstance.h"
#include "Screen.generated.h"
UCLASS()
class VIDEOSTREAMING_API AScreen : public AActor
{
GENERATED_BODY()
public:
void imageSubscribeCallback(TSharedPtr<FROSBaseMsg> msg);
private:
UPROPERTY() // this is required to keep the garbage collector from cleaning up the memory
TMap<FString, UTopic *> topics_;
UPROPERTY()
UROSIntegrationGameInstance* rosinst_;
uint32 texture_width = 640;
uint32 texture_height = 480;
// Texture regions
FUpdateTextureRegion2D* TextureRegions = nullptr;
void subscribe(const FString topic_name, const FString topic_type, std::function<void(TSharedPtr<FROSBaseMsg>)> callback, const int32 queue_size);
public:
// Sets default values for this actor's properties
AScreen();
//Our texture data (result of vertical blur pass)
UPROPERTY()
TArray<FColor> TextureData;
protected:
// Called when the game starts or when spawned
virtual void BeginPlay() override;
public:
// Called every frame
virtual void Tick(float DeltaTime) override;
UPROPERTY(VisibleAnywhere)
class UBillboardComponent* Billboard = nullptr;
//Our dynamically updated texture
UPROPERTY(VisibleAnywhere)
UTexture2D* MyTexture;
// from https://wiki.unrealengine.com/Dynamic_Textures
void UpdateTextureRegions(
UTexture2D* Texture,
int32 MipIndex,
uint32 NumRegions,
FUpdateTextureRegion2D* Regions,
uint32 SrcPitch,
uint32 SrcBpp,
uint8* SrcData,
bool bFreeData);
};
screen.cpp file:
// Fill out your copyright notice in the Description page of Project Settings.
#include "Screen.h"
#include <unordered_map>
#include <functional>
#include "Components/BillboardComponent.h"
#include "RHICommandList.h"
#include "RenderingThread.h"
#include "ROSIntegration/Public/sensor_msgs/Image.h"
// This is the callback function upon reception of the message. Create a std::function callback object
void AScreen::imageSubscribeCallback(TSharedPtr<FROSBaseMsg> msg)
{
auto Concrete = StaticCastSharedPtr<ROSMessages::sensor_msgs::Image>(msg);
if (Concrete.IsValid())
{
for (std::size_t i = 0; i < Concrete->width * Concrete->height; i++)
{
TextureData* = FColor(Concrete->data[3 * i], Concrete->data[3 * i + 1], Concrete->data[3 * i + 2], 255);
}
// UE_LOG(LogTemp, Log, TEXT("Received image message")); // print to Unreal console
}
return;
}
// Sets default values
AScreen::AScreen():
rosinst_(nullptr)
{
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
PrimaryActorTick.bCanEverTick = true;
Billboard = CreateDefaultSubobject<UBillboardComponent>(TEXT("My Billboard"));
Billboard->SetHiddenInGame(false, true);
RootComponent = Billboard;
TextureRegions = new FUpdateTextureRegion2D(0, 0, 0, 0, texture_width, texture_height);
// testing with full red
TextureData.Init(FColor(255, 0, 0, 255), texture_width * texture_height);
}
// Called when the game starts or when spawned
void AScreen::BeginPlay()
{
Super::BeginPlay();
// initialize unordered_map
std::unordered_map<std::string, FString> map_topic_types;
map_topic_types.insert({ "sensor_msgs/Image", TEXT("sensor_msgs/Image") });
rosinst_ = Cast<UROSIntegrationGameInstance>(GetGameInstance());
std::function<void(TSharedPtr<FROSBaseMsg>)> my_function = std::bind(&AScreen::imageSubscribeCallback, this, std::placeholders::_1);
subscribe(TEXT("/image_raw"), map_topic_types"sensor_msgs/Image"], my_function, 1);
for (const auto &pair : topics_)
UE_LOG(LogTemp, Log, TEXT("Topics registered are: %s"), *(pair.Key)); // print to Unreal console
MyTexture = UTexture2D::CreateTransient(texture_width, texture_height); // Allocate the texture HRI
MyTexture->UpdateResource(); // Use this function to update the texture rects you want to change:
}
// Called every frame
void AScreen::Tick(float DeltaTime)
{
Super::Tick(DeltaTime);
if (MyTexture != nullptr)
{
UpdateTextureRegions(MyTexture, (int32)0, (uint32)1, TextureRegions, (uint32)(4 * texture_width), (uint32)4, (uint8*)TextureData.GetData(), false);
Billboard->SetSprite(MyTexture);
}
}
// NOTE: There is a method called UpdateTextureRegions in UTexture2D but it is compiled WITH_EDITOR and is not marked as ENGINE_API so it cannot be linked // from plugins.
void AScreen::UpdateTextureRegions(UTexture2D* Texture, int32 MipIndex, uint32 NumRegions, FUpdateTextureRegion2D* Regions, uint32 SrcPitch, uint32 SrcBpp, uint8* SrcData, bool bFreeData)
{
if (Texture->Resource)
{
struct FUpdateTextureRegionsData {
FTexture2DResource* Texture2DResource;
int32 MipIndex;
uint32 NumRegions;
FUpdateTextureRegion2D* Regions;
uint32 SrcPitch;
uint32 SrcBpp;
uint8* SrcData;
};
FUpdateTextureRegionsData* RegionData = new FUpdateTextureRegionsData;
RegionData->Texture2DResource = (FTexture2DResource*)Texture->Resource;
RegionData->MipIndex = MipIndex;
RegionData->NumRegions = NumRegions;
RegionData->Regions = Regions;
RegionData->SrcPitch = SrcPitch;
RegionData->SrcBpp = SrcBpp;
RegionData->SrcData = SrcData;
ENQUEUE_UNIQUE_RENDER_COMMAND_TWOPARAMETER( UpdateTextureRegionsData, FUpdateTextureRegionsData*, RegionData, RegionData, bool, bFreeData, bFreeData,
{
for (uint32 RegionIndex = 0; RegionIndex < RegionData->NumRegions; ++RegionIndex)
{
int32 CurrentFirstMip = RegionData->Texture2DResource->GetCurrentFirstMip();
if (RegionData->MipIndex >= CurrentFirstMip)
{
RHIUpdateTexture2D( RegionData->Texture2DResource->GetTexture2DRHI(),
RegionData->MipIndex - CurrentFirstMip,
RegionData->Regions[RegionIndex],
RegionData->SrcPitch,
RegionData->SrcData + RegionData->Regions[RegionIndex].SrcY * RegionData->SrcPitch + RegionData->Regions[RegionIndex].SrcX * RegionData->SrcBpp );
}
}
if (bFreeData)
{
FMemory::Free(RegionData->Regions);
FMemory::Free(RegionData->SrcData);
}
delete RegionData;
});
}
}
void AScreen::subscribe(const FString topic_name, const FString topic_type, std::function<void(TSharedPtr<FROSBaseMsg>)> callback, const int32 queue_size)
{
UTopic * topic = nullptr;
// check if the game instance was acquired before trying to use it
if (rosinst_ != nullptr)
{
topic = NewObject<UTopic>(UTopic::StaticClass());
topic->Init(rosinst_->ROSIntegrationCore, topic_name, topic_type, queue_size);
topic->Subscribe(callback);
// store it into map
topics_.Add(topic_name, topic);
}
}