I finally figured this out, sharing for future people searching around:
auto AxisMappings = UInputSettings::GetInputSettings()->GetAxisMappings();
for(auto mapping : AxisMappings) {
UInputSettings::GetInputSettings()->RemoveAxisMapping(mapping);
}
auto ActionMappings = UInputSettings::GetInputSettings()->GetActionMappings();
for(auto mapping : ActionMappings) {
UInputSettings::GetInputSettings()->RemoveActionMapping(mapping);
}
UInputSettings::GetInputSettings()->SaveKeyMappings();
FConfigCacheIni::LoadGlobalIniFile(GInputIni, TEXT("Input"), 0, true, true);
auto InputSettings = const_cast<UInputSettings*>(GetDefault<UInputSettings>());
InputSettings->ReloadConfig(InputSettings->GetClass(), *GInputIni, UE4::LCPF_PropagateToInstances);
InputSettings->SaveKeyMappings();