I just had to add a line when setting up the input mode:
FInputModeGameAndUI InputModeData;
InputModeData.SetLockMouseToViewportBehavior(EMouseLockMode::LockInFullscreen);
InputModeData.SetHideCursorDuringCapture(false); // Added this line
InputModeData.SetWidgetToFocus(TakeWidget());