[Redesign] Merge dev into feature-redesign (#4078)

* Fixed Report Abuse Page's Accessibility (#4001)

Fixes #4002. Relevant to [VSTS #395879](https://devdiv.visualstudio.com/DefaultCollection/DevDiv/_workitems?id=395879&_a=edit). This help-text bug was for the sign-in page, but this also affects the Report Abuse page.

* Fixed accessibility of email field in report abuse page
* Fixed signature accessibility

* Use ServerCommon's Response Code Processor (#3999)

* Added Nuget.Services.Logging; moved to TelemetryResponseCodeProcessor
* Added binding redirect for AI

* Removing the "WITH(ONLINE=ON)" from the index creation as per #3952 (#4004)

* Removing the "WITH(ONLINE=ON)" from the index creation as per #3952 (#4004) (#4030)

* Remove validation rules that block uploading valid semver2 versions #3645 (#3757)

* Add support for semVerLevel query parameter to V2 endpoints (#3714)

* adding new optional semVerLevel query parameter to v2 odata endpoints

* adding new optional semVerLevel query parameter to v2 autocomplete endpoints

* Applying semVerLevel filter on v2 OData endpoints

* Use [FromUri] attribute on semVerLevel
(avoids having single quotes in the parameter value)

* Ensure navigation links on v2 feeds use normalized version

* Clarifying comment on Get(Id=,Version=) v2 API

* Properly default to semver2 inclusion on Get(Id=,Version=)

* Compare NormalizedVersion to be able to retrieve matching SemVer2 package versions for a given normalized version string.

* Code review feedback

* Update and fix broken test data

* Keep legacy version compliance checks in place for non-SemVer2 versions (#3761)

* Keep legacy version compatibility checks in place for non-semver2 versions

* Added comment to clarify the reasons behind the legacy version check.

* Fix typo

* Rename test for clarity

* code review feedback

* Set SemVerLevelKey after setting Dependencies

* #3861 V2 NuGetEntityTypeSerializer Id link patcher must retain curated feed name (#3864)

* Support Is(Absolute)Latest for SemVer2 + semVerLevel for SearchService (#3842)

* LuceneIndexingService in Gallery should take into account IsLatest(Stable)SemVer2 (#3863)

* POST VerifyPackage version validation should use ToFullString comparison

* ODataV2CuratedFeedController should support semver2 by default when requesting a specific version, and compare on NormalizedVersion

* Refactor NuGetEntityTypeSerializer + unit test coverage (#3879)

* Use NormalizedVersion in URLs contained in PackageAddedNotice (#3886)

* Add nullcheck and use TryParse. (#3890)

* Show full version on package details page (#3887)

* Highlight semver2 packages on package details view (#3893)

* Fix bug in IsLatest(Stable)SemVer2 (#3895)

* VerifyPackage on ApiController should treat version as optional parameter (#3903)

* UpdateIsLatest not resetting IsLatest(Stable)SemVer2 on previous latest versions (#3909)

* Fix malformed URL in redirect after package upload (#3915)

* Minor fix for search results package URLs (when to use version or not in the URL)

* UrlHelper extension for Package should use NormalizedVersion (#3925)

* Default to latest stable semver2 on package details page (#3930)

* User profile page does not show SemVer 2.0.0 packages #3911 (#3933)

* Fix functional test failure SearchMicrosoftDotNetCuratedFeed #3941 (#3942)

* Fix System.NotSupportedException on User profile page (#3943)

* Fix Functional Test failure for ODataFeeds.V2FeedExtendedTests.FindPackagesByIdTest #3947 (#3948)

* Fix load test failure due to incorrect test setup (#3957)

* Hijack IsLatest(Stable)Version OData filter when semVerLevel=2.0.0 (#3966)

* Detect if package only differ by metadata and show optimal user-facing error message (#3970)

* Update Semver2 package details message with final nuget client version #3897 (#3988)

* On package validation failure an actionable error message should be displayed. #3916 (#4031)

* Make downloads link on home page a proper link (#4052)

* Fix the date format on stats page (#4057)

* Update telemetry processors (#4059)

* Reorder SemVer2Latest migration to match deployment history (#4062)

* Average download shown incorrectly when its 1.x #4039 (#4040)

* Average download shown incorrectly when its 1.x #4039

* Moved logic to viewmodel and added UTs

* Port latest changes made to package details page

* SemVer2 - Missing db index on Packages table #498 (#4073)

* SemVer2 - Missing db index on Packages table #498

* SemVer2 - Missing db index on Packages table for partial search #499 (#4074)

* Package-Versions autocomplete endpoint does not properly handle semVerLevel when using the db #4086 (#4087)

* v2 package-versions auto-complete endpoint should exclude deleted versions #4092 (#4093)

* Remove auto-refresh AJAX call for total stats on home page #4090 (#4091)

* Fix merge conflicts

src/NuGet.Services.Search.Client/Client/SearchClient.cs

@@ -83,14 +83,20 @@ public async Task<ServiceResponse<SearchResults>> Search(
             bool countOnly = false,
             bool explain = false,
             bool getAllVersions = false,
-            string supportedFramework = null)
+            string supportedFramework = null,
+            string semVerLevel = null)
         {
             IDictionary<string, string> nameValue = new Dictionary<string, string>();
             nameValue.Add("q", query);
             nameValue.Add("skip", skip.ToString());
             nameValue.Add("take", take.ToString());
             nameValue.Add("sortBy", SortNames[sortBy]);
 
+            if (!String.IsNullOrEmpty(semVerLevel))
+            {
+                nameValue.Add("semVerLevel", semVerLevel);
+            }
+
             if (!String.IsNullOrEmpty(supportedFramework))
             {
                 nameValue.Add("supportedFramework", supportedFramework);

src/NuGetGallery.Core/Auditing/PackageAuditRecord.cs

@@ -49,7 +49,7 @@ public PackageAuditRecord(Package package, AuditedPackageAction action)
 
         public override string GetPath()
         {
-            return $"{Id}/{NuGetVersionNormalizer.Normalize(Version)}"
+            return $"{Id}/{NuGetVersionFormatter.Normalize(Version)}"
                 .ToLowerInvariant();
         }
     }

src/NuGetGallery.Core/CoreStrings.resx

@@ -168,9 +168,6 @@
   <data name="Manifest_TargetFrameworkNotSupported" xml:space="preserve">
     <value>The target framework {0} is not supported.</value>
   </data>
-  <data name="Manifest_InvalidVersionSemVer200" xml:space="preserve">
-    <value>The version '{0}' is not supported. The NuGet Gallery currently does not currently support Semantic Version 2.0 as it would break older NuGet clients.</value>
-  </data>
   <data name="Http404NotFound" xml:space="preserve">
     <value>(404) Error - Not Found</value>
   </data>
@@ -180,4 +177,7 @@
   <data name="NegativeIndexesAreInvalid" xml:space="preserve">
     <value>Negative indexes are invalid.</value>
   </data>
+  <data name="Manifest_InvalidDependencyVersion" xml:space="preserve">
+    <value>The package manifest contains an invalid Dependency Version: '{0}'</value>
+  </data>
 </root>

src/NuGetGallery.Core/Entities/Package.cs

@@ -80,6 +80,9 @@ public Package()
         public bool IsLatest { get; set; }
         public bool IsLatestStable { get; set; }
 
+        public bool IsLatestSemVer2 { get; set; }
+        public bool IsLatestStableSemVer2 { get; set; }
+
         /// <summary>
         /// This is when the Package Entity was last touched (so caches can notice changes). In UTC.
         /// </summary>

src/NuGetGallery.Core/NuGetGallery.Core.csproj

@@ -85,19 +85,19 @@
       <HintPath>..\..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
     <Reference Include="NuGet.Common, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\NuGet.Common.4.3.0-preview1-2507\lib\net45\NuGet.Common.dll</HintPath>
+      <HintPath>..\..\packages\NuGet.Common.4.3.0-preview1-2524\lib\net45\NuGet.Common.dll</HintPath>
     </Reference>
     <Reference Include="NuGet.Frameworks, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\NuGet.Frameworks.4.3.0-preview1-2507\lib\net45\NuGet.Frameworks.dll</HintPath>
+      <HintPath>..\..\packages\NuGet.Frameworks.4.3.0-preview1-2524\lib\net45\NuGet.Frameworks.dll</HintPath>
     </Reference>
     <Reference Include="NuGet.Packaging, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\NuGet.Packaging.4.3.0-preview1-2507\lib\net45\NuGet.Packaging.dll</HintPath>
+      <HintPath>..\..\packages\NuGet.Packaging.4.3.0-preview1-2524\lib\net45\NuGet.Packaging.dll</HintPath>
     </Reference>
     <Reference Include="NuGet.Packaging.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\NuGet.Packaging.Core.4.3.0-preview1-2507\lib\net45\NuGet.Packaging.Core.dll</HintPath>
+      <HintPath>..\..\packages\NuGet.Packaging.Core.4.3.0-preview1-2524\lib\net45\NuGet.Packaging.Core.dll</HintPath>
     </Reference>
     <Reference Include="NuGet.Versioning, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\NuGet.Versioning.4.3.0-preview1-2507\lib\net45\NuGet.Versioning.dll</HintPath>
+      <HintPath>..\..\packages\NuGet.Versioning.4.3.0-preview1-2524\lib\net45\NuGet.Versioning.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.ComponentModel.DataAnnotations" />

src/NuGetGallery.Core/NuGetVersionExtensions.cs

@@ -7,7 +7,7 @@
 
 namespace NuGetGallery
 {
-    public static class NuGetVersionNormalizer
+    public static class NuGetVersionFormatter
     {
         public static string Normalize(string version)
         {
@@ -19,16 +19,34 @@ public static string Normalize(string version)
 
             return parsed.ToNormalizedString();
         }
+
+        public static string ToFullStringOrFallback(string version, string fallback = "")
+        {
+            NuGetVersion nugetVersion;
+            if (NuGetVersion.TryParse(version, out nugetVersion))
+            {
+                return nugetVersion.ToFullString();
+            }
+            else
+            {
+                return fallback;
+            }
+        }
     }
 
     public static class NuGetVersionExtensions
     {
-        private const RegexOptions Flags = RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture;
-        private static readonly Regex SemanticVersionRegex = new Regex(@"^(?<Version>\d+(\s*\.\s*\d+){0,3})(?<Release>-[a-z][0-9a-z-]*)?$", Flags);
+        private const RegexOptions SemanticVersionRegexFlags = RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture;
+        private static readonly Regex SemanticVersionRegex = new Regex(@"^(?<Version>\d+(\s*\.\s*\d+){0,3})(?<Release>-[a-z][0-9a-z-]*)?$", SemanticVersionRegexFlags);
 
         public static string ToNormalizedStringSafe(this NuGetVersion self)
         {
-            return self != null ? self.ToNormalizedString() : String.Empty;
+            return self != null ? self.ToNormalizedString() : string.Empty;
+        }
+
+        public static string ToFullStringSafe(this NuGetVersion self)
+        {
+            return self != null ? self.ToFullString() : string.Empty;
         }
 
         public static bool IsValidVersionForLegacyClients(this NuGetVersion self)

src/NuGetGallery.Core/Packaging/ManifestValidator.cs

@@ -28,7 +28,7 @@ public static IEnumerable<ValidationResult> Validate(Stream nuspecStream, out Nu
             catch (Exception ex)
             {
                 nuspecReader = null;
-                return new [] { new ValidationResult(ex.Message) };
+                return new[] { new ValidationResult(ex.Message) };
             }
 
             return Enumerable.Empty<ValidationResult>();
@@ -59,7 +59,7 @@ private static IEnumerable<ValidationResult> ValidateCore(PackageMetadata packag
             // Check and validate URL properties
             foreach (var result in CheckUrls(
                 packageMetadata.GetValueFromMetadata("IconUrl"),
-                packageMetadata.GetValueFromMetadata("ProjectUrl"), 
+                packageMetadata.GetValueFromMetadata("ProjectUrl"),
                 packageMetadata.GetValueFromMetadata("LicenseUrl")))
             {
                 yield return result;
@@ -76,7 +76,7 @@ private static IEnumerable<ValidationResult> ValidateCore(PackageMetadata packag
                     version));
             }
 
-            var versionValidationResult = ValidateVersion(packageMetadata.Version);
+            var versionValidationResult = ValidateVersionForLegacyClients(packageMetadata.Version);
             if (versionValidationResult != null)
             {
                 yield return versionValidationResult;
@@ -143,17 +143,19 @@ private static IEnumerable<ValidationResult> ValidateCore(PackageMetadata packag
                         // Versions
                         if (dependency.VersionRange.MinVersion != null)
                         {
-                            var versionRangeValidationResult = ValidateVersion(dependency.VersionRange.MinVersion);
+                            var versionRangeValidationResult =
+                                ValidateDependencyVersion(dependency.VersionRange.MinVersion);
                             if (versionRangeValidationResult != null)
                             {
                                 yield return versionRangeValidationResult;
                             }
                         }
 
-                        if (dependency.VersionRange.MaxVersion != null 
+                        if (dependency.VersionRange.MaxVersion != null
                             && dependency.VersionRange.MaxVersion != dependency.VersionRange.MinVersion)
                         {
-                            var versionRangeValidationResult = ValidateVersion(dependency.VersionRange.MaxVersion);
+                            var versionRangeValidationResult =
+                                ValidateDependencyVersion(dependency.VersionRange.MaxVersion);
                             if (versionRangeValidationResult != null)
                             {
                                 yield return versionRangeValidationResult;
@@ -164,24 +166,37 @@ private static IEnumerable<ValidationResult> ValidateCore(PackageMetadata packag
             }
         }
 
-        private static ValidationResult ValidateVersion(NuGetVersion version)
+        /// <summary>
+        /// Checks whether the provided version is consumable by legacy 2.x clients,
+        /// which do not support a `.` in release labels, or release labels starting with numeric characters.
+        /// See also https://github.com/NuGet/NuGetGallery/issues/3226.
+        /// </summary>
+        /// <param name="version">The <see cref="NuGetVersion"/> to check for 2.x client compatibility.</param>
+        /// <returns>Returns a <see cref="ValidationResult"/> when non-compliant; otherwise <c>null</c>.</returns>
+        private static ValidationResult ValidateVersionForLegacyClients(NuGetVersion version)
         {
-            if (version.IsSemVer2)
+            if (!version.IsSemVer2 && !version.IsValidVersionForLegacyClients())
             {
                 return new ValidationResult(string.Format(
                     CultureInfo.CurrentCulture,
-                    CoreStrings.Manifest_InvalidVersionSemVer200,
-                    version.ToFullString()));
+                    CoreStrings.Manifest_InvalidVersion,
+                    version));
             }
-            else if (!version.IsValidVersionForLegacyClients())
+
+            return null;
+        }
+
+        private static ValidationResult ValidateDependencyVersion(NuGetVersion version)
+        {
+            if (version.HasMetadata)
             {
                 return new ValidationResult(string.Format(
                     CultureInfo.CurrentCulture,
-                    CoreStrings.Manifest_InvalidVersion,
-                    version));
+                    CoreStrings.Manifest_InvalidDependencyVersion,
+                    version.ToFullString()));
             }
 
-            return null;
+            return ValidateVersionForLegacyClients(version);
         }
 
         private static IEnumerable<ValidationResult> CheckUrls(params string[] urls)

src/NuGetGallery.Core/SemVerLevelKey.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq.Expressions;
 using NuGet.Versioning;
 
 namespace NuGetGallery
@@ -15,6 +16,9 @@ namespace NuGetGallery
     /// </summary>
     public static class SemVerLevelKey
     {
+        public static readonly string SemVerLevel2 = "2.0.0";
+        private static readonly NuGetVersion _semVer2Version = NuGetVersion.Parse(SemVerLevel2);
+
         /// <summary>
         /// This could either indicate being SemVer1-compliant, or non-SemVer-compliant at all (e.g. System.Versioning pattern).
         /// </summary>
@@ -31,7 +35,7 @@ public static class SemVerLevelKey
         /// </summary>
         /// <param name="originalVersion">The package's non-normalized, original version string.</param>
         /// <param name="dependencies">The package's direct dependencies as defined in the package's manifest.</param>
-        /// <returns>Returns <c>null</c> when unknown; otherwise the identified SemVer-level.</returns>
+        /// <returns>Returns <c>null</c> when unknown; otherwise the identified SemVer-level key.</returns>
         public static int? ForPackage(NuGetVersion originalVersion, IEnumerable<PackageDependency> dependencies)
         {
             if (originalVersion == null)
@@ -53,17 +57,66 @@ public static class SemVerLevelKey
                     // Check the package dependencies for SemVer-compliance.
                     // As soon as a SemVer2-compliant dependency version is found that is not SemVer1-compliant,
                     // this package in itself is to be identified as to have SemVerLevelKey.SemVer2.
-                    var dependencyVersionRange = VersionRange.Parse(dependency.VersionSpec);
-
-                    if ((dependencyVersionRange.MinVersion != null && dependencyVersionRange.MinVersion.IsSemVer2)
-                        || (dependencyVersionRange.MaxVersion != null && dependencyVersionRange.MaxVersion.IsSemVer2))
+                    VersionRange dependencyVersionRange;
+                    if (dependency.VersionSpec != null && VersionRange.TryParse(dependency.VersionSpec, out dependencyVersionRange))
                     {
-                        return SemVer2;
+                        if ((dependencyVersionRange.MinVersion != null && dependencyVersionRange.MinVersion.IsSemVer2)
+                            || (dependencyVersionRange.MaxVersion != null && dependencyVersionRange.MaxVersion.IsSemVer2))
+                        {
+                            return SemVer2;
+                        }
                     }
                 }
             }
 
             return Unknown;
         }
+
+        /// <summary>
+        /// Identifies the SemVer-level for a given semVerLevel version string.
+        /// </summary>
+        /// <param name="semVerLevel">The version string indicating the supported SemVer-level.</param>
+        /// <returns>
+        /// Returns <c>null</c> when unknown; otherwise the identified SemVer-level key.
+        /// </returns>
+        /// <remarks>
+        /// Older clients don't send the semVerLevel query parameter at all, 
+        /// so we default to Unknown for backwards-compatibility.
+        /// </remarks>
+        public static int? ForSemVerLevel(string semVerLevel)
+        {
+            if (semVerLevel == null)
+            {
+                return Unknown;
+            }
+
+            NuGetVersion parsedVersion;
+            if (NuGetVersion.TryParse(semVerLevel, out parsedVersion))
+            {
+                return _semVer2Version <= parsedVersion ? SemVer2 : Unknown;
+            }
+            else
+            {
+                return Unknown;
+            }
+        }
+
+        /// <summary>
+        /// Indicates whether the provided SemVer-level key is compliant with the provided SemVer-level version string.
+        /// </summary>
+        /// <param name="semVerLevel">The SemVer-level string indicating the SemVer-level to comply with.</param>
+        /// <returns><c>True</c> if compliant; otherwise <c>false</c>.</returns>
+        public static Expression<Func<Package, bool>> IsPackageCompliantWithSemVerLevel(string semVerLevel)
+        {
+            // Note: we must return an expression that Linq to Entities is able to translate to SQL
+            var parsedSemVerLevelKey = ForSemVerLevel(semVerLevel);
+
+            if (parsedSemVerLevelKey == SemVer2)
+            {
+                return p => p.SemVerLevelKey == Unknown || p.SemVerLevelKey == parsedSemVerLevelKey;
+            }
+
+            return p => p.SemVerLevelKey == Unknown;
+        }
     }
 }

src/NuGetGallery.Core/packages.config

@@ -9,11 +9,11 @@
   <package id="Microsoft.Web.Xdt" version="2.1.1" targetFramework="net452" />
   <package id="Microsoft.WindowsAzure.ConfigurationManager" version="3.1.0" targetFramework="net452" />
   <package id="Newtonsoft.Json" version="9.0.1" targetFramework="net452" />
-  <package id="NuGet.Common" version="4.3.0-preview1-2507" targetFramework="net452" />
-  <package id="NuGet.Frameworks" version="4.3.0-preview1-2507" targetFramework="net452" />
-  <package id="NuGet.Packaging" version="4.3.0-preview1-2507" targetFramework="net452" />
-  <package id="NuGet.Packaging.Core" version="4.3.0-preview1-2507" targetFramework="net452" />
-  <package id="NuGet.Versioning" version="4.3.0-preview1-2507" targetFramework="net452" />
+  <package id="NuGet.Common" version="4.3.0-preview1-2524" targetFramework="net452" />
+  <package id="NuGet.Frameworks" version="4.3.0-preview1-2524" targetFramework="net452" />
+  <package id="NuGet.Packaging" version="4.3.0-preview1-2524" targetFramework="net452" />
+  <package id="NuGet.Packaging.Core" version="4.3.0-preview1-2524" targetFramework="net452" />
+  <package id="NuGet.Versioning" version="4.3.0-preview1-2524" targetFramework="net452" />
   <package id="System.Spatial" version="5.6.5-beta" targetFramework="net452" />
   <package id="WindowsAzure.Storage" version="7.0.0" targetFramework="net452" />
 </packages>

src/NuGetGallery/App_Start/NuGetODataV2FeedConfig.cs

@@ -60,6 +60,7 @@ public static IEdmModel GetEdmModel()
             searchAction.Parameter<string>("searchTerm");
             searchAction.Parameter<string>("targetFramework");
             searchAction.Parameter<bool>("includePrerelease");
+            searchAction.Parameter<string>("semVerLevel");
             searchAction.ReturnsCollectionFromEntitySet<V2FeedPackage>("Packages");
 
             var findPackagesAction = builder.Action("FindPackagesById");
@@ -73,6 +74,7 @@ public static IEdmModel GetEdmModel()
             getUpdatesAction.Parameter<bool>("includeAllVersions");
             getUpdatesAction.Parameter<string>("targetFrameworks");
             getUpdatesAction.Parameter<string>("versionConstraints");
+            getUpdatesAction.Parameter<string>("semVerLevel");
             getUpdatesAction.ReturnsCollectionFromEntitySet<V2FeedPackage>("Packages");
 
             var model = builder.GetEdmModel();