-
Notifications
You must be signed in to change notification settings - Fork 644
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add CopyFileAsync to ICoreFileStorageService #5581
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,7 @@ | |
using System; | ||
using System.Collections.Concurrent; | ||
using System.Collections.Generic; | ||
using System.Diagnostics; | ||
using System.Globalization; | ||
using System.IO; | ||
using System.Net; | ||
|
@@ -16,6 +17,15 @@ namespace NuGetGallery | |
{ | ||
public class CloudBlobCoreFileStorageService : ICoreFileStorageService | ||
{ | ||
/// <summary> | ||
/// This is the maximum duration for <see cref="CopyFileAsync(string, string, string, string)"/> to poll, | ||
/// waiting for a package copy to complete. The value picked today is based off of the maximum duration we wait | ||
/// when uploading files to Azure China blob storage. Note that in cases when the copy source and destination | ||
/// are in the same container, the copy completed immediately and no polling is necessary. | ||
/// </summary> | ||
private static readonly TimeSpan MaxCopyDuration = TimeSpan.FromMinutes(10); | ||
private static readonly TimeSpan CopyPollFrequency = TimeSpan.FromMilliseconds(500); | ||
|
||
private static readonly HashSet<string> KnownPublicFolders = new HashSet<string> { | ||
CoreConstants.PackagesFolderName, | ||
CoreConstants.PackageBackupsFolderName, | ||
|
@@ -100,6 +110,93 @@ public async Task<IFileReference> GetFileReferenceAsync(string folderName, strin | |
} | ||
} | ||
|
||
public async Task CopyFileAsync(string srcFolderName, string srcFileName, string destFolderName, string destFileName) | ||
{ | ||
if (srcFolderName == null) | ||
{ | ||
throw new ArgumentNullException(nameof(srcFolderName)); | ||
} | ||
|
||
if (srcFileName == null) | ||
{ | ||
throw new ArgumentNullException(nameof(srcFileName)); | ||
} | ||
|
||
if (destFolderName == null) | ||
{ | ||
throw new ArgumentNullException(nameof(destFolderName)); | ||
} | ||
|
||
if (destFileName == null) | ||
{ | ||
throw new ArgumentNullException(nameof(destFileName)); | ||
} | ||
|
||
var srcContainer = await GetContainerAsync(srcFolderName); | ||
var srcBlob = srcContainer.GetBlobReference(srcFileName); | ||
|
||
var destContainer = await GetContainerAsync(destFolderName); | ||
var destBlob = destContainer.GetBlobReference(destFileName); | ||
|
||
// Determine the source blob etag. | ||
await srcBlob.FetchAttributesAsync(); | ||
var srcAccessCondition = AccessCondition.GenerateIfMatchCondition(srcBlob.ETag); | ||
|
||
// Check if the destination blob already exists and fetch attributes. | ||
var destAccessCondition = AccessCondition.GenerateIfNotExistsCondition(); | ||
if (await destBlob.ExistsAsync()) | ||
{ | ||
if (destBlob.CopyState?.Status == CopyStatus.Failed) | ||
{ | ||
// If the last copy failed, allow this copy to occur. | ||
destAccessCondition = AccessCondition.GenerateIfMatchCondition(destBlob.ETag); | ||
} | ||
else if ((srcBlob.Properties.ContentMD5 != null | ||
&& srcBlob.Properties.ContentMD5 == destBlob.Properties.ContentMD5 | ||
&& srcBlob.Properties.Length == destBlob.Properties.Length)) | ||
{ | ||
// If the blob hash is the same and the length is the same, no-op the copy. | ||
return; | ||
} | ||
} | ||
|
||
// Start the server-side copy and wait for it to complete. | ||
try | ||
{ | ||
await destBlob.StartCopyAsync( | ||
srcBlob, | ||
srcAccessCondition, | ||
destAccessCondition); | ||
} | ||
catch (StorageException ex) when (ex.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict) | ||
{ | ||
throw new InvalidOperationException( | ||
String.Format( | ||
CultureInfo.CurrentCulture, | ||
"There is already a blob with name {0} in container {1}.", | ||
destFileName, | ||
destFolderName), | ||
ex); | ||
} | ||
|
||
var stopwatch = Stopwatch.StartNew(); | ||
while (destBlob.CopyState.Status == CopyStatus.Pending | ||
&& stopwatch.Elapsed < MaxCopyDuration) | ||
{ | ||
await destBlob.FetchAttributesAsync(); | ||
await Task.Delay(CopyPollFrequency); | ||
} | ||
|
||
if (destBlob.CopyState.Status == CopyStatus.Pending) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should we kill the copy operation in those cases? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I decided against this because we can still recover if the caller calls this I would rather take this approach than aborting and trying again over and over. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. (since would rather trust blob storage server side to get the job done eventually than us trying to start it every N minutes) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what if we trigger a copy operation and the previous copy is still in progress? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We won't even start. We have a precondition |
||
{ | ||
throw new TimeoutException($"Waiting for the blob copy operation to complete timed out after {MaxCopyDuration.TotalSeconds} seconds."); | ||
} | ||
else if (destBlob.CopyState.Status != CopyStatus.Success) | ||
{ | ||
throw new StorageException($"The blob copy operation had copy status {destBlob.CopyState.Status} ({destBlob.CopyState.StatusDescription})."); | ||
} | ||
} | ||
|
||
public async Task SaveFileAsync(string folderName, string fileName, Stream packageFile, bool overwrite = true) | ||
{ | ||
ICloudBlobContainer container = await GetContainerAsync(folderName); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,7 @@ | |
using System; | ||
using System.Globalization; | ||
using System.IO; | ||
using System.Security.Cryptography; | ||
using System.Threading.Tasks; | ||
using System.Web.Hosting; | ||
using System.Web.Mvc; | ||
|
@@ -168,6 +169,45 @@ public Task SaveFileAsync(string folderName, string fileName, Stream packageFile | |
return Task.FromResult(0); | ||
} | ||
|
||
public Task CopyFileAsync(string srcFolderName, string srcFileName, string destFolderName, string destFileName) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. check input There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed. |
||
{ | ||
if (srcFolderName == null) | ||
{ | ||
throw new ArgumentNullException(nameof(srcFolderName)); | ||
} | ||
|
||
if (srcFileName == null) | ||
{ | ||
throw new ArgumentNullException(nameof(srcFileName)); | ||
} | ||
|
||
if (destFolderName == null) | ||
{ | ||
throw new ArgumentNullException(nameof(destFolderName)); | ||
} | ||
|
||
if (destFileName == null) | ||
{ | ||
throw new ArgumentNullException(nameof(destFileName)); | ||
} | ||
|
||
var srcFilePath = BuildPath(_configuration.FileStorageDirectory, srcFolderName, srcFileName); | ||
var destFilePath = BuildPath(_configuration.FileStorageDirectory, destFolderName, destFileName); | ||
|
||
_fileSystemService.CreateDirectory(Path.GetDirectoryName(destFilePath)); | ||
|
||
try | ||
{ | ||
_fileSystemService.Copy(srcFilePath, destFilePath, overwrite: false); | ||
} | ||
catch (IOException e) | ||
{ | ||
throw new InvalidOperationException("Could not copy because destination file already exists", e); | ||
} | ||
|
||
return Task.CompletedTask; | ||
} | ||
|
||
public Task<bool> IsAvailableAsync() | ||
{ | ||
return Task.FromResult(Directory.Exists(_configuration.FileStorageDirectory)); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -51,5 +51,10 @@ public IFileReference GetFileReference(string path) | |
var info = new FileInfo(path); | ||
return info.Exists ? new LocalFileReference(info) : null; | ||
} | ||
|
||
public virtual void Copy(string sourceFileName, string destFileName, bool overwrite) | ||
{ | ||
File.Copy(sourceFileName, destFileName, overwrite); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can we make this async? https://stackoverflow.com/questions/882686/asynchronous-file-copy-move-in-c-sharp There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd rather use the high level file system API on There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Did some tests. Performance is worse. |
||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
check input
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed.