Non-destructive TArray<AActor*>?

You can do anything with pointers. It’s just a number.
For example, you can convert and store them as Guids:

TArray<FGuid> PtrArray{};
FGuid IntPtr = PtrToGuid(MyActor);

PtrArray.Add(IntPtr);

AActor* ActorPtr = GuidToPtr(PtrArray[0]);

PtrArray.Empty();
FGuid ::PtrToGuid(AActor* Ptr)
{
	FGuid Guid = FGuid::NewGuid();

	if (Ptr==nullptr)
	{
		Guid.Invalidate();
	} else {

		UPTRINT IntPtr = reinterpret_cast<UPTRINT>(Ptr);

		if (sizeof(UPTRINT) > 4)
		{
			Guid[0] ^= (static_cast<uint64>(IntPtr) >> 32);
		}

		Guid[1] ^= IntPtr & 0xFFFFFFFF;
	}

	return Guid;
}

AActor* ::GuidToPtr(const FGuid &Guid)
{
	UPTRINT IntPtr = 0;

	if (sizeof(UPTRINT) > 4)
	{
		IntPtr = static_cast<UPTRINT>(static_cast<uint64>(Guid[0]) << 32);
	}

	IntPtr |= Guid[1] & 0xFFFFFFFF;

	return reinterpret_cast<AActor*>(IntPtr);
}

PtrArray.Empty() in that case will not invoke object destructors.