From 526106d2690bfadee6125a9e5c9b36ac1381d485 Mon Sep 17 00:00:00 2001 From: bala16 Date: Tue, 17 Oct 2017 19:31:47 -0700 Subject: [PATCH] Use skiptoken to retrieve complete list of resources Currently all calls to ARM APIs ignore skiptoken link. This meant only partial list of resources would be retrieved in some cases. This change adds support to walk the skiptoken url when retrieving GET /subscriptions/{subscriptionId}/resources. Note: Other API calls still have this limitation. Resource Search uses the same API but has not been updated to make use of this new functionality yet (only looks at top 100 resources in a subscription) --- ARMExplorer.csproj | 2 + Controllers/ArmRepository.cs | 80 +++++++++++++++++----- Model/ArmResource.cs | 31 +++++++++ Model/ArmResourceListResult.cs | 13 ++++ Tests/WebApiTests/MockHttpClientWrapper.cs | 15 +++- 5 files changed, 123 insertions(+), 18 deletions(-) create mode 100644 Model/ArmResource.cs create mode 100644 Model/ArmResourceListResult.cs diff --git a/ARMExplorer.csproj b/ARMExplorer.csproj index 19aacc7..372b378 100644 --- a/ARMExplorer.csproj +++ b/ARMExplorer.csproj @@ -376,6 +376,8 @@ + + diff --git a/Controllers/ArmRepository.cs b/Controllers/ArmRepository.cs index 55c1b54..a7c763e 100644 --- a/Controllers/ArmRepository.cs +++ b/Controllers/ArmRepository.cs @@ -6,6 +6,7 @@ using System.Net.Http.Headers; using System.Text.RegularExpressions; using System.Threading.Tasks; +using ARMExplorer.Model; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -14,6 +15,7 @@ namespace ARMExplorer.Controllers public class ArmRepository : IArmRepository { private readonly IHttpClientWrapper _clientWrapper; + private readonly int _maxNextLinkDepth = 5; public ArmRepository(IHttpClientWrapper clientWrapper) { @@ -40,17 +42,63 @@ public async Task> GetSubscriptionIdsAsync(HttpRequestMessage requ return subscriptionIds; } + private static bool AddResourceToList(IEnumerable resources, ISet allResources) + { + var initalCount = allResources.Count; + + foreach (var resource in resources) + { + allResources.Add(resource); + } + + var updatedCount = allResources.Count; + + return updatedCount > initalCount; + } + + private async Task> GetResources(HttpRequestMessage requestMessage, string getResourcesUrl) + { + var allResources = new HashSet(); + var currentNextLinkDepth = 0; + + while (!string.IsNullOrEmpty(getResourcesUrl)) + { + var response = await GetAsync(requestMessage, getResourcesUrl); + response.EnsureSuccessStatusCode(); + var armResourceListResult = await response.Content.ReadAsAsync(); + + var newResourceFound = AddResourceToList(armResourceListResult.Value, allResources); + + // ARM API returns the same skiptoken and resources repeatedly when there are no more resources. To avoid infinite cycle break when + // 1. No new resource was found in the current response or + // 2. Limit the max number of links to follow to _maxNextLinkDepth or + // 3. When nextLink is empty + + if (!newResourceFound) + { + break; + } + + if (currentNextLinkDepth++ > _maxNextLinkDepth) + { + break; + } + + getResourcesUrl = armResourceListResult.NextLink; + } + + return allResources; + } + public async Task> GetProviderNamesFor(HttpRequestMessage requestMessage, string subscriptionId) { - var response = await GetResourcesForAsync(requestMessage, subscriptionId); - response.EnsureSuccessStatusCode(); - dynamic resources = await response.Content.ReadAsAsync(); - JArray values = resources.value; + var initialGetResourcesUrl = string.Format(Utils.ResourcesTemplate, HyakUtils.CSMUrl, subscriptionId, Utils.CSMApiVersion); + var resources = await GetResources(requestMessage, initialGetResourcesUrl); var uniqueProviders = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (dynamic value in values) + + foreach (var resource in resources) { - string id = value.id; - var match = Regex.Match(id, "/subscriptions/.*?/resourceGroups/(.*?)/providers/(.*?)/(.*?)/"); + var match = Regex.Match(resource.Id, "/subscriptions/.*?/resourceGroups/(.*?)/providers/(.*?)/(.*?)/"); if (match.Success) { var provider = match.Groups[2].Value.ToUpperInvariant(); @@ -63,15 +111,13 @@ public async Task> GetProviderNamesFor(HttpRequestMessage reques public async Task>>> GetProvidersFor(HttpRequestMessage requestMessage, string subscriptionId) { - var response = await GetResourcesForAsync(requestMessage, subscriptionId); - response.EnsureSuccessStatusCode(); - - dynamic resources = await response.Content.ReadAsAsync(); - JArray values = resources.value; + var initialGetResourcesUrl = string.Format(Utils.ResourcesTemplate, HyakUtils.CSMUrl, subscriptionId, Utils.CSMApiVersion); + var resources = await GetResources(requestMessage, initialGetResourcesUrl); var result = new Dictionary>>(); - foreach (dynamic value in values) + + foreach (var resource in resources) { - string id = value.id; + string id = resource.Id; var match = Regex.Match(id, "/subscriptions/.*?/resourceGroups/(.*?)/providers/(.*?)/(.*?)/"); if (match.Success) { @@ -116,10 +162,10 @@ private async Task GetSubscriptionsAsync(HttpRequestMessage return await _clientWrapper.SendAsync(requestMessage, sendRequest); } - private async Task GetResourcesForAsync(HttpRequestMessage requestMessage, string subscriptionId) + private async Task GetAsync(HttpRequestMessage requestMessage, string url) { - var sendRequest = new HttpRequestMessage(HttpMethod.Get, string.Format(Utils.ResourcesTemplate, HyakUtils.CSMUrl, subscriptionId, Utils.CSMApiVersion)); - return await _clientWrapper.SendAsync(requestMessage, sendRequest); + var sendRequest = new HttpRequestMessage(HttpMethod.Get, url); + return await _clientWrapper.ExecuteAsync(requestMessage, sendRequest); } } } \ No newline at end of file diff --git a/Model/ArmResource.cs b/Model/ArmResource.cs new file mode 100644 index 0000000..78f65c1 --- /dev/null +++ b/Model/ArmResource.cs @@ -0,0 +1,31 @@ +using System; + +namespace ARMExplorer.Model +{ + public class ArmResource : IEquatable + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string Id { get; set; } + // other fields ignored + + public bool Equals(ArmResource other) + { + if (ReferenceEquals(null, other)) return false; + if (ReferenceEquals(this, other)) return true; + return string.Equals(Id, other.Id, StringComparison.OrdinalIgnoreCase); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((ArmResource) obj); + } + + public override int GetHashCode() + { + return Id != null ? Id.GetHashCode() : 0; + } + } +} \ No newline at end of file diff --git a/Model/ArmResourceListResult.cs b/Model/ArmResourceListResult.cs new file mode 100644 index 0000000..61b52f1 --- /dev/null +++ b/Model/ArmResourceListResult.cs @@ -0,0 +1,13 @@ +using System.Collections.ObjectModel; + +namespace ARMExplorer.Model +{ + public class ArmResourceListResult + { + [Newtonsoft.Json.JsonProperty("value", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public Collection Value { get; set; } + + [Newtonsoft.Json.JsonProperty("nextLink", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string NextLink { get; set; } + } +} \ No newline at end of file diff --git a/Tests/WebApiTests/MockHttpClientWrapper.cs b/Tests/WebApiTests/MockHttpClientWrapper.cs index 6ddada8..3062eca 100644 --- a/Tests/WebApiTests/MockHttpClientWrapper.cs +++ b/Tests/WebApiTests/MockHttpClientWrapper.cs @@ -30,7 +30,20 @@ public Task SendAsync(HttpRequestMessage requestMessage, Ht public Task ExecuteAsync(HttpRequestMessage requestMessage, HttpRequestMessage executeRequest) { - throw new NotImplementedException(); + var responseMessage = new HttpResponseMessage(HttpStatusCode.OK); + string filePath; + if (executeRequest.RequestUri.ToString().Contains("resources")) + { + filePath = Path.Combine(new DirectoryInfo(Directory.GetCurrentDirectory()).FullName, Path.Combine("WebApiTests", "data", "resourcesForsubscription.json")); + } + else + { + filePath = Path.Combine(new DirectoryInfo(Directory.GetCurrentDirectory()).FullName, Path.Combine("WebApiTests", "data", "subscriptions.json")); + } + responseMessage.Content = new StringContent(File.ReadAllText(filePath), Encoding.UTF8, "application/json"); + var response = new TaskCompletionSource(); + response.SetResult(responseMessage); + return response.Task; } } } \ No newline at end of file