Check the AbilitySystemComponent::OnRemoveAbility, It only trigger the OnRemoveAbility on primaryinstance if there is one.
// Notify the ability that it has been removed. It follows the same pattern as OnGiveAbility() and is only called on the primary instance of the ability or the CDO.
UGameplayAbility* PrimaryInstance = Spec.GetPrimaryInstance();
if (PrimaryInstance)
{
PrimaryInstance->OnRemoveAbility(AbilityActorInfo.Get(), Spec);
}
else
{
Spec.Ability->OnRemoveAbility(AbilityActorInfo.Get(), Spec);
}