Announcement

Collapse
No announcement yet.

Playing live video capture -- C++ only / no blueprints

Collapse
X
 
  • Filter
  • Time
  • Show
Clear All
new posts

    Playing live video capture -- C++ only / no blueprints

    Hi everyone,

    I am trying to play a live video capture of my webcam from raw data (uint8 array of pixels data) from another computer (image data received via TCP). The data coming in is an RGB array (no alpha, so 640x480x3 values). I got something working using a base actor class, a Texture2D, a billboard component and its attached sprite, but the video feed is delayed by around 0.5s-1s. I am new to Unreal Engine, so I assume I am doing something wrong, or in an un-optimized way. I know that you can get videos playing with minimal delays. In other terms, I believe what I am trying to achieve is updating a 2D texture at runtime, at each frame, that is why I call UpdateTextureRegions in the Tick method, followed by readjusting the Sprite texture pointer.

    There is a lot of unnecessary code around, but I will post it anyway so you have the full picture of what I am trying to do. I am getting the image data from my callback function, which follows a publish/subscribe design pattern.

    Please let me know if you know how I can improve this to have minimal delay in displaying the video feed.

    screen.h file:
    Code:
    #pragma once
    
    // Unreal
    #include "CoreMinimal.h"
    #include "GameFramework/Actor.h"
    #include "Engine/Texture2DDynamic.h"
    
    // ROS plugin headers
    #include "ROSIntegration/Classes/RI/Topic.h"
    #include "ROSIntegration/Classes/ROSIntegrationGameInstance.h"
    
    #include "Screen.generated.h"
    
    UCLASS()
    class VIDEOSTREAMING_API AScreen : public AActor
    {
        GENERATED_BODY()
    
    public:
        void imageSubscribeCallback(TSharedPtr<FROSBaseMsg> msg);
    
    private:
    
        UPROPERTY() // this is required to keep the garbage collector from cleaning up the memory
        TMap<FString, UTopic *> topics_;
    
        UPROPERTY()
        UROSIntegrationGameInstance* rosinst_;
    
        uint32 texture_width = 640;
        uint32 texture_height = 480;
    
        // Texture regions    
        FUpdateTextureRegion2D* TextureRegions = nullptr;
    
        void subscribe(const FString topic_name, const FString topic_type, std::function<void(TSharedPtr<FROSBaseMsg>)> callback, const int32 queue_size);
    
    public:    
        // Sets default values for this actor's properties
        AScreen();
    
        //Our texture data (result of vertical blur pass)
        UPROPERTY()
        TArray<FColor> TextureData;
    
    protected:
        // Called when the game starts or when spawned
        virtual void BeginPlay() override;
    
    public:    
        // Called every frame
        virtual void Tick(float DeltaTime) override;
    
        UPROPERTY(VisibleAnywhere)
        class UBillboardComponent* Billboard = nullptr;
    
        //Our dynamically updated texture
        UPROPERTY(VisibleAnywhere)
        UTexture2D* MyTexture;
    
        // from https://wiki.unrealengine.com/Dynamic_Textures
        void UpdateTextureRegions(
            UTexture2D* Texture,
            int32 MipIndex,
            uint32 NumRegions,
            FUpdateTextureRegion2D* Regions,
            uint32 SrcPitch,
            uint32 SrcBpp,
            uint8* SrcData,
            bool bFreeData);
    
    };
    screen.cpp file:
    Code:
    // Fill out your copyright notice in the Description page of Project Settings.
    
    #include "Screen.h"
    
    #include <unordered_map>
    #include <functional>
    
    #include "Components/BillboardComponent.h"
    #include "RHICommandList.h"
    #include "RenderingThread.h"
    #include "ROSIntegration/Public/sensor_msgs/Image.h"
    
    // This is the callback function upon reception of the message. Create a std::function callback object
    
    void AScreen::imageSubscribeCallback(TSharedPtr<FROSBaseMsg> msg)
    {
        auto Concrete = StaticCastSharedPtr<ROSMessages::sensor_msgs::Image>(msg);
        if (Concrete.IsValid())
        {
            for (std::size_t i = 0; i < Concrete->width * Concrete->height; i++)
            {
                TextureData[i] = FColor(Concrete->data[3 * i], Concrete->data[3 * i + 1], Concrete->data[3 * i + 2], 255);
            }
    
            // UE_LOG(LogTemp, Log, TEXT("Received image message")); // print to Unreal console
    
        }
        return;
    }
    
    // Sets default values
    AScreen::AScreen():
        rosinst_(nullptr)
    {
         // Set this actor to call Tick() every frame.  You can turn this off to improve performance if you don't need it.
        PrimaryActorTick.bCanEverTick = true;
    
        Billboard = CreateDefaultSubobject<UBillboardComponent>(TEXT("My Billboard"));
        Billboard->SetHiddenInGame(false, true);
        RootComponent = Billboard;
    
        TextureRegions = new FUpdateTextureRegion2D(0, 0, 0, 0, texture_width, texture_height);
    
        // testing with full red
        TextureData.Init(FColor(255, 0, 0, 255), texture_width * texture_height);
    }
    
    // Called when the game starts or when spawned
    void AScreen::BeginPlay()
    {
        Super::BeginPlay();
    
        // initialize unordered_map
        std::unordered_map<std::string, FString> map_topic_types;
        map_topic_types.insert({ "sensor_msgs/Image", TEXT("sensor_msgs/Image") });
    
        rosinst_ = Cast<UROSIntegrationGameInstance>(GetGameInstance());
        std::function<void(TSharedPtr<FROSBaseMsg>)> my_function = std::bind(&AScreen::imageSubscribeCallback, this, std::placeholders::_1);
    
        subscribe(TEXT("/image_raw"), map_topic_types["sensor_msgs/Image"], my_function, 1);
        for (const auto &pair : topics_)
            UE_LOG(LogTemp, Log, TEXT("Topics registered are: %s"), *(pair.Key)); // print to Unreal console
    
        MyTexture = UTexture2D::CreateTransient(texture_width, texture_height);   // Allocate the texture HRI
        MyTexture->UpdateResource();   // Use this function to update the texture rects you want to change:
    }
    
    // Called every frame
    void AScreen::Tick(float DeltaTime)
    {
        Super::Tick(DeltaTime);
        if (MyTexture != nullptr)
        {
            UpdateTextureRegions(MyTexture, (int32)0, (uint32)1, TextureRegions, (uint32)(4 * texture_width), (uint32)4, (uint8*)TextureData.GetData(), false);
            Billboard->SetSprite(MyTexture);
        }
    }
    
    // NOTE: There is a method called UpdateTextureRegions in UTexture2D but it is compiled WITH_EDITOR and is not marked as ENGINE_API so it cannot be linked // from plugins.  
    void AScreen::UpdateTextureRegions(UTexture2D* Texture, int32 MipIndex, uint32 NumRegions, FUpdateTextureRegion2D* Regions, uint32 SrcPitch, uint32 SrcBpp, uint8* SrcData, bool bFreeData)
    {
        if (Texture->Resource)
        {
            struct FUpdateTextureRegionsData {
                FTexture2DResource* Texture2DResource;
                int32 MipIndex;
                uint32 NumRegions;
                FUpdateTextureRegion2D* Regions;
                uint32 SrcPitch;
                uint32 SrcBpp;
                uint8* SrcData;
            };  
        FUpdateTextureRegionsData* RegionData = new FUpdateTextureRegionsData;  
        RegionData->Texture2DResource = (FTexture2DResource*)Texture->Resource;
        RegionData->MipIndex = MipIndex;
        RegionData->NumRegions = NumRegions;
        RegionData->Regions = Regions;
        RegionData->SrcPitch = SrcPitch;
        RegionData->SrcBpp = SrcBpp;
        RegionData->SrcData = SrcData;  
    
        ENQUEUE_UNIQUE_RENDER_COMMAND_TWOPARAMETER( UpdateTextureRegionsData, FUpdateTextureRegionsData*, RegionData, RegionData, bool, bFreeData, bFreeData,
            {
                for (uint32 RegionIndex = 0; RegionIndex < RegionData->NumRegions; ++RegionIndex)
                {
                    int32 CurrentFirstMip = RegionData->Texture2DResource->GetCurrentFirstMip();
                    if (RegionData->MipIndex >= CurrentFirstMip)
                    {
                        RHIUpdateTexture2D( RegionData->Texture2DResource->GetTexture2DRHI(),
                            RegionData->MipIndex - CurrentFirstMip,
                            RegionData->Regions[RegionIndex],
                            RegionData->SrcPitch,
                            RegionData->SrcData + RegionData->Regions[RegionIndex].SrcY * RegionData->SrcPitch + RegionData->Regions[RegionIndex].SrcX * RegionData->SrcBpp );
                    }
                }
            if (bFreeData)
            {
                FMemory::Free(RegionData->Regions);
                FMemory::Free(RegionData->SrcData);
            }
            delete RegionData;
            });
        }
    }
    
    void AScreen::subscribe(const FString topic_name, const FString topic_type, std::function<void(TSharedPtr<FROSBaseMsg>)> callback, const int32 queue_size)
    {
        UTopic * topic = nullptr;
        // check if the game instance was acquired before trying to use it
        if (rosinst_ != nullptr)
        {
            topic = NewObject<UTopic>(UTopic::StaticClass());
            topic->Init(rosinst_->ROSIntegrationCore, topic_name, topic_type, queue_size);
            topic->Subscribe(callback);
            // store it into map
            topics_.Add(topic_name, topic);
        }
    }
    Last edited by Pinknoise; 04-08-2020, 01:52 PM.
Working...
X