Skip to content

Commit

Permalink
Add storage connection strings for each distinct purpose in the galle…
Browse files Browse the repository at this point in the history 
…ry (#4568)

Fix #4554
  • Loading branch information
@joelverhagen
joelverhagen committed Aug 18, 2017
1 parent 5a71165 commit bb6201d
Show file tree
Hide file tree
Showing 14 changed files with 409 additions and 93 deletions.
8 changes: 6 additions & 2 deletions src/NuGetGallery.Core/Auditing/CloudAuditingService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace NuGetGallery.Auditing
/// <summary>
/// Writes audit records to a specific container in the Cloud Storage Account provided
/// </summary>
public class CloudAuditingService : AuditingService
public class CloudAuditingService : AuditingService, ICloudStorageStatusDependency
{
public static readonly string DefaultContainerName = "auditing";

Expand All @@ -26,7 +26,6 @@ public class CloudAuditingService : AuditingService
public CloudAuditingService(string instanceId, string localIP, string storageConnectionString, Func<Task<AuditActor>> getOnBehalfOf)
: this(instanceId, localIP, GetContainer(storageConnectionString), getOnBehalfOf)
{

}

public CloudAuditingService(string instanceId, string localIP, CloudBlobContainer auditContainer, Func<Task<AuditActor>> getOnBehalfOf)
Expand Down Expand Up @@ -121,5 +120,10 @@ private static async Task WriteBlob(string auditData, string fullPath, CloudBloc
throw;
}
}

public Task<bool> IsAvailableAsync()
{
return _auditContainer.ExistsAsync();
}
}
}
16 changes: 16 additions & 0 deletions src/NuGetGallery.Core/ICloudStorageStatusDependency.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System.Threading.Tasks;

namespace NuGetGallery
{
/// <summary>
/// Determines whether a cloud storage object (e.g. a blob container) is available. The purpose of this interface
/// is to assess a gallery instance's connection to Azure Storage.
/// </summary>
public interface ICloudStorageStatusDependency
{
Task<bool> IsAvailableAsync();
}
}
1 change: 1 addition & 0 deletions src/NuGetGallery.Core/NuGetGallery.Core.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@
<Compile Include="Entities\Scope.cs" />
<Compile Include="Entities\UserSecurityPolicy.cs" />
<Compile Include="Entities\User.cs" />
<Compile Include="ICloudStorageStatusDependency.cs" />
<Compile Include="Infrastructure\AzureEntityList.cs" />
<Compile Include="Infrastructure\TableErrorLog.cs" />
<Compile Include="PackageReaderCoreExtensions.cs" />
Expand Down
145 changes: 81 additions & 64 deletions src/NuGetGallery/App_Start/DefaultDependenciesModule.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
using System.Net;
using System.Net.Mail;
using System.Security.Principal;
using System.Threading;
using System.Web;
using System.Web.Hosting;
using System.Web.Mvc;
using AnglicanGeek.MarkdownMailer;
using Autofac;
using Autofac.Core;
using Elmah;
using Microsoft.WindowsAzure.ServiceRuntime;
using NuGetGallery.Areas.Admin;
Expand Down Expand Up @@ -71,31 +71,13 @@ protected override void Load(ContainerBuilder builder)

ConfigureSearch(builder, configuration);

if (!string.IsNullOrEmpty(configuration.Current.AzureStorageConnectionString))
{
builder.RegisterInstance(new TableErrorLog(configuration.Current.AzureStorageConnectionString))
.As<ErrorLog>()
.SingleInstance();
}
else
{
builder.RegisterInstance(new SqlErrorLog(configuration.Current.SqlConnectionString))
.As<ErrorLog>()
.SingleInstance();
}

builder.RegisterType<DateTimeProvider>().AsSelf().As<IDateTimeProvider>().SingleInstance();

builder.RegisterType<HttpContextCacheService>()
.AsSelf()
.As<ICacheService>()
.InstancePerLifetimeScope();

builder.RegisterType<ContentService>()
.AsSelf()
.As<IContentService>()
.SingleInstance();

builder.Register(c => new EntitiesContext(configuration.Current.SqlConnectionString, readOnly: configuration.Current.ReadOnlyMode))
.AsSelf()
.As<IEntitiesContext>()
Expand Down Expand Up @@ -194,11 +176,6 @@ protected override void Load(ContainerBuilder builder)
.As<ITempDataProvider>()
.InstancePerLifetimeScope();

builder.RegisterType<NuGetExeDownloaderService>()
.AsSelf()
.As<INuGetExeDownloaderService>()
.InstancePerLifetimeScope();

builder.RegisterType<StatusService>()
.AsSelf()
.As<IStatusService>()
Expand Down Expand Up @@ -279,29 +256,14 @@ protected override void Load(ContainerBuilder builder)
break;
case StorageType.AzureStorage:
ConfigureForAzureStorage(builder, configuration);
defaultAuditingService = GetAuditingServiceForAzureStorage(configuration);
defaultAuditingService = GetAuditingServiceForAzureStorage(builder, configuration);
break;
}

RegisterAuditingServices(builder, defaultAuditingService);

RegisterCookieComplianceService(builder, configuration, diagnosticsService);

builder.RegisterType<FileSystemService>()
.AsSelf()
.As<IFileSystemService>()
.SingleInstance();

builder.RegisterType<PackageFileService>()
.AsSelf()
.As<IPackageFileService>()
.InstancePerLifetimeScope();

builder.RegisterType<UploadFileService>()
.AsSelf()
.As<IUploadFileService>()
.InstancePerLifetimeScope();

// todo: bind all package curators by convention
builder.RegisterType<WebMatrixPackageCurator>()
.AsSelf()
Expand Down Expand Up @@ -389,11 +351,24 @@ private static void ConfigureAutocomplete(ContainerBuilder builder, IGalleryConf

private static void ConfigureForLocalFileSystem(ContainerBuilder builder, IGalleryConfigurationService configuration)
{
builder.RegisterType<FileSystemService>()
.AsSelf()
.As<IFileSystemService>()
.SingleInstance();

builder.RegisterType<FileSystemFileStorageService>()
.AsSelf()
.As<IFileStorageService>()
.SingleInstance();

foreach (var dependent in StorageDependent.GetAll(configuration.Current))
{
builder.RegisterType(dependent.ImplementationType)
.AsSelf()
.As(dependent.InterfaceType)
.InstancePerLifetimeScope();
}

builder.RegisterInstance(NullReportService.Instance)
.AsSelf()
.As<IReportService>()
Expand All @@ -409,34 +384,75 @@ private static void ConfigureForLocalFileSystem(ContainerBuilder builder, IGalle
.AsSelf()
.As<IAggregateStatsService>()
.InstancePerLifetimeScope();

builder.RegisterInstance(new SqlErrorLog(configuration.Current.SqlConnectionString))
.As<ErrorLog>()
.SingleInstance();
}

private static void ConfigureForAzureStorage(ContainerBuilder builder, IGalleryConfigurationService configuration)
private static IAuditingService GetAuditingServiceForLocalFileSystem(IGalleryConfigurationService configuration)
{
builder.RegisterInstance(new CloudBlobClientWrapper(configuration.Current.AzureStorageConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
.AsSelf()
.As<ICloudBlobClient>()
.SingleInstance();
var auditingPath = Path.Combine(
FileSystemFileStorageService.ResolvePath(configuration.Current.FileStorageDirectory),
FileSystemAuditingService.DefaultContainerName);

builder.RegisterType<CloudBlobFileStorageService>()
.AsSelf()
.As<IFileStorageService>()
.SingleInstance();
return new FileSystemAuditingService(auditingPath, AuditActor.GetAspNetOnBehalfOfAsync);
}

private static void ConfigureForAzureStorage(ContainerBuilder builder, IGalleryConfigurationService configuration)
{
/// The goal here is to initialize a <see cref="ICloudBlobClient"/> and <see cref="IFileStorageService"/>
/// instance for each unique connection string. Each dependent of <see cref="IFileStorageService"/> (that
/// is, each service that has a <see cref="IFileStorageService"/> constructor parameter) is registered in
/// <see cref="StorageDependent.GetAll(IAppConfiguration)"/> and is grouped by the respective storage
/// connection string. Each group is given a binding key which refers to the appropriate instance of the
/// <see cref="IFileStorageService"/>.
var completedBindingKeys = new HashSet<string>();
foreach (var dependent in StorageDependent.GetAll(configuration.Current))
{
if (completedBindingKeys.Add(dependent.BindingKey))
{
builder.RegisterInstance(new CloudBlobClientWrapper(dependent.AzureStorageConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
.AsSelf()
.As<ICloudBlobClient>()
.SingleInstance()
.Keyed<ICloudBlobClient>(dependent.BindingKey);

builder.RegisterType<CloudBlobFileStorageService>()
.WithParameter(new ResolvedParameter(
(pi, ctx) => pi.ParameterType == typeof(ICloudBlobClient),
(pi, ctx) => ctx.ResolveKeyed<ICloudBlobClient>(dependent.BindingKey)))
.AsSelf()
.As<IFileStorageService>()
.As<ICloudStorageStatusDependency>()
.SingleInstance()
.Keyed<IFileStorageService>(dependent.BindingKey);
}

builder.RegisterType(dependent.ImplementationType)
.WithParameter(new ResolvedParameter(
(pi, ctx) => pi.ParameterType == typeof(IFileStorageService),
(pi, ctx) => ctx.ResolveKeyed<IFileStorageService>(dependent.BindingKey)))
.AsSelf()
.As(dependent.InterfaceType)
.InstancePerLifetimeScope();
}

// when running on Windows Azure, we use a back-end job to calculate stats totals and store in the blobs
builder.RegisterInstance(new JsonAggregateStatsService(configuration.Current.AzureStorageConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
builder.RegisterInstance(new JsonAggregateStatsService(configuration.Current.AzureStorage_Statistics_ConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
.AsSelf()
.As<IAggregateStatsService>()
.SingleInstance();

// when running on Windows Azure, pull the statistics from the warehouse via storage
builder.RegisterInstance(new CloudReportService(configuration.Current.AzureStorageConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
builder.RegisterInstance(new CloudReportService(configuration.Current.AzureStorage_Statistics_ConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
.AsSelf()
.As<IReportService>()
.As<ICloudStorageStatusDependency>()
.SingleInstance();

// when running on Windows Azure, download counts come from the downloads.v1.json blob
var downloadCountService = new CloudDownloadCountService(configuration.Current.AzureStorageConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant);
var downloadCountService = new CloudDownloadCountService(configuration.Current.AzureStorage_Statistics_ConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant);
builder.RegisterInstance(downloadCountService)
.AsSelf()
.As<IDownloadCountService>()
Expand All @@ -447,18 +463,13 @@ private static void ConfigureForAzureStorage(ContainerBuilder builder, IGalleryC
.AsSelf()
.As<IStatisticsService>()
.SingleInstance();
}

private static IAuditingService GetAuditingServiceForLocalFileSystem(IGalleryConfigurationService configuration)
{
var auditingPath = Path.Combine(
FileSystemFileStorageService.ResolvePath(configuration.Current.FileStorageDirectory),
FileSystemAuditingService.DefaultContainerName);

return new FileSystemAuditingService(auditingPath, AuditActor.GetAspNetOnBehalfOfAsync);
builder.RegisterInstance(new TableErrorLog(configuration.Current.AzureStorage_Errors_ConnectionString))
.As<ErrorLog>()
.SingleInstance();
}

private static IAuditingService GetAuditingServiceForAzureStorage(IGalleryConfigurationService configuration)
private static IAuditingService GetAuditingServiceForAzureStorage(ContainerBuilder builder, IGalleryConfigurationService configuration)
{
string instanceId;
try
Expand All @@ -472,10 +483,16 @@ private static IAuditingService GetAuditingServiceForAzureStorage(IGalleryConfig

var localIp = AuditActor.GetLocalIpAddressAsync().Result;

return new CloudAuditingService(instanceId, localIp, configuration.Current.AzureStorageConnectionString, AuditActor.GetAspNetOnBehalfOfAsync);
var service = new CloudAuditingService(instanceId, localIp, configuration.Current.AzureStorage_Auditing_ConnectionString, AuditActor.GetAspNetOnBehalfOfAsync);

builder.RegisterInstance(service)
.As<ICloudStorageStatusDependency>()
.SingleInstance();

return service;
}

private static IAuditingService CombineServices(IEnumerable<IAuditingService> services)
private static IAuditingService CombineAuditingServices(IEnumerable<IAuditingService> services)
{
if (!services.Any())
{
Expand Down Expand Up @@ -510,7 +527,7 @@ private static void RegisterAuditingServices(ContainerBuilder builder, IAuditing
services.Add(defaultAuditingService);
}

var service = CombineServices(services);
var service = CombineAuditingServices(services);

builder.RegisterInstance(service)
.AsSelf()
Expand Down
Loading

0 comments on commit bb6201d

Please sign in to comment.