I found a an additional AssetDeleted like this worked better:
void FAssetRegistry::AssetDeleted(const FString &sObjectPath)
{
FAssetData **AssetData = CachedAssetsByObjectPath.Find(FName(*sObjectPath));
if (AssetData && *AssetData)
{
RemoveAssetData(*AssetData);
}
}