Disable client side rate limiting in Azure.

Client side rate limiting is being problematic for fresh installs and scaling operations [1]

Azure ARM throttling is applied at subscription level, so client side rate limiting helps to prevent cluster sharing the same subscription from disrupting each other.
However there's lower limits which apply at the SP/tenant and resource level e.g ARM limits the number of write calls per service principal to 1200/hour [2]. Since we ensure particular SPs per cluster via Cloud Credential Operator it should be relatively safe to disable the client rate limiting

Orthogonally to this some improvements on the rate limiting and back off mechanisms are being added to the cloud provider.

[1] https://bugzilla.redhat.com/show_bug.cgi?id=1782516.
[2] https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling
[3] kubernetes-sigs/cloud-provider-azure#247

pkg/asset/manifests/azure/cloudproviderconfig.go

@@ -34,30 +34,19 @@ func (params CloudProviderConfig) JSON() (string, error) {
 			// ref: https://github.com/kubernetes/kubernetes/blob/4b7c607ba47928a7be77fadef1550d6498397a4c/staging/src/k8s.io/legacy-cloud-providers/azure/auth/azure_auth.go#L69
 			UserAssignedIdentityID: "",
 		},
-		ResourceGroup:          resourceGroupName,
-		Location:               params.GroupLocation,
-		SubnetName:             params.SubnetName,
-		SecurityGroupName:      params.NetworkSecurityGroupName,
-		VnetName:               params.VirtualNetworkName,
-		VnetResourceGroup:      params.NetworkResourceGroupName,
-		RouteTableName:         params.ResourcePrefix + "-node-routetable",
-		CloudProviderBackoff:   true,
-		CloudProviderRateLimit: true,
-
-		// The default rate limits for Azure cloud provider are https://github.com/kubernetes/kubernetes/blob/f8d2b6b982bb06fc64979ac53ae668284d9c003c/staging/src/k8s.io/legacy-cloud-providers/azure/azure.go#L51-L56
-		// While the AKS recommends following rate limits for large clusters https://github.com/Azure/aks-engine/blob/0f6aa91fa1870d5be657c62374d11f7d6009121d/examples/largeclusters/kubernetes.json#L9-L15
-		// 									default		AKS (large)	Change
-		// cloudProviderBackoffRetries		6			6					NO
-		// cloudProviderBackoffJitter		1.0			1					NO
-		// cloudProviderBackoffExponent		1.5			1.5					NO
-		// cloudProviderBackoffDuration		5			6					YES to 6
-		// cloudProviderRateLimitQPS		3			3					YES to 6
-		// cloudProviderRateLimitBucket		5			10					YES to 10
-		CloudProviderBackoffDuration:      6,
-		CloudProviderRateLimitQPS:         6,
-		CloudProviderRateLimitQPSWrite:    6,
-		CloudProviderRateLimitBucket:      10,
-		CloudProviderRateLimitBucketWrite: 10,
+		ResourceGroup:     resourceGroupName,
+		Location:          params.GroupLocation,
+		SubnetName:        params.SubnetName,
+		SecurityGroupName: params.NetworkSecurityGroupName,
+		VnetName:          params.VirtualNetworkName,
+		VnetResourceGroup: params.NetworkResourceGroupName,
+		RouteTableName:    params.ResourcePrefix + "-node-routetable",
+		// client side rate limiting is problematic for scaling operations. We disable it by default.
+		// https://github.com/kubernetes-sigs/cloud-provider-azure/issues/247
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1782516#c7
+		CloudProviderRateLimit:       false,
+		CloudProviderBackoff:         true,
+		CloudProviderBackoffDuration: 6,
 
 		UseInstanceMetadata: true,
 		//default to standard load balancer, supports tcp resets on idle

pkg/asset/manifests/azure/cloudproviderconfig_test.go

@@ -43,11 +43,11 @@ func TestCloudProviderConfig(t *testing.T) {
 	"cloudProviderBackoffExponent": 0,
 	"cloudProviderBackoffDuration": 6,
 	"cloudProviderBackoffJitter": 0,
-	"cloudProviderRateLimit": true,
-	"cloudProviderRateLimitQPS": 6,
-	"cloudProviderRateLimitBucket": 10,
-	"cloudProviderRateLimitQPSWrite": 6,
-	"cloudProviderRateLimitBucketWrite": 10,
+	"cloudProviderRateLimit": false,
+	"cloudProviderRateLimitQPS": 0,
+	"cloudProviderRateLimitBucket": 0,
+	"cloudProviderRateLimitQPSWrite": 0,
+	"cloudProviderRateLimitBucketWrite": 0,
 	"useInstanceMetadata": true,
 	"loadBalancerSku": "standard",
 	"excludeMasterFromStandardLB": null,