From 15915b0ae60e8967eb4f8fccfb286a3bff04bbe7 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 4 Jun 2025 17:07:43 +0300 Subject: [PATCH 01/21] remove caching for ascendants --- osf/models/node.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/osf/models/node.py b/osf/models/node.py index 83d646cc717..71d960670dd 100644 --- a/osf/models/node.py +++ b/osf/models/node.py @@ -57,7 +57,7 @@ ) from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField from osf.utils.fields import NonNaiveDateTimeField -from osf.utils.requests import get_request_and_user_id, string_type_request_headers, get_current_request +from osf.utils.requests import get_request_and_user_id, string_type_request_headers from osf.utils.workflows import CollectionSubmissionStates from osf.utils import sanitize from website import language, settings @@ -2467,21 +2467,15 @@ def _remove_from_associated_collections(self, auth=None, force=False): ) def _get_addon_from_gv(self, gv_pk, requesting_user_id, auth=None): - request = get_current_request() - # This is to avoid making multiple requests to GV - # within the lifespan of one request on the OSF side - try: - gv_addons = request.gv_addons - except AttributeError: - requesting_user = OSFUser.load(requesting_user_id) - services = gv_translations.get_external_services(requesting_user) - for service in services: - if service.short_name == gv_pk: - break - else: - return None - gv_addons = request.gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) + requesting_user = OSFUser.load(requesting_user_id) + services = gv_translations.get_external_services(requesting_user) + for service in services: + if service.short_name == gv_pk: + break + else: + return None + gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) for item in gv_addons: if item.short_name == gv_pk: return item From f42f67dc4d61efcb6540a007b5af3833a926f070 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 10 Jun 2025 13:17:04 +0300 Subject: [PATCH 02/21] Revert "remove caching for ascendants" This reverts commit 15915b0ae60e8967eb4f8fccfb286a3bff04bbe7. --- osf/models/node.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/osf/models/node.py b/osf/models/node.py index 71d960670dd..83d646cc717 100644 --- a/osf/models/node.py +++ b/osf/models/node.py @@ -57,7 +57,7 @@ ) from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField from osf.utils.fields import NonNaiveDateTimeField -from osf.utils.requests import get_request_and_user_id, string_type_request_headers +from osf.utils.requests import get_request_and_user_id, string_type_request_headers, get_current_request from osf.utils.workflows import CollectionSubmissionStates from osf.utils import sanitize from website import language, settings @@ -2467,15 +2467,21 @@ def _remove_from_associated_collections(self, auth=None, force=False): ) def _get_addon_from_gv(self, gv_pk, requesting_user_id, auth=None): - requesting_user = OSFUser.load(requesting_user_id) - services = gv_translations.get_external_services(requesting_user) - for service in services: - if service.short_name == gv_pk: - break - else: - return None + request = get_current_request() + # This is to avoid making multiple requests to GV + # within the lifespan of one request on the OSF side + try: + gv_addons = request.gv_addons + except AttributeError: + requesting_user = OSFUser.load(requesting_user_id) + services = gv_translations.get_external_services(requesting_user) + for service in services: + if service.short_name == gv_pk: + break + else: + return None + gv_addons = request.gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) - gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) for item in gv_addons: if item.short_name == gv_pk: return item From ec2c7c0aac844e1e9140b09e6fed80d0df157871 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 10 Jun 2025 17:01:21 +0300 Subject: [PATCH 03/21] use GV addons caching when it's needed --- osf/models/archive.py | 2 +- osf/models/mixins.py | 4 ++-- osf/models/node.py | 31 +++++++++++++++++++------------ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/osf/models/archive.py b/osf/models/archive.py index 052d918a99c..9e622764ca7 100644 --- a/osf/models/archive.py +++ b/osf/models/archive.py @@ -146,7 +146,7 @@ def _set_target(self, addon_short_name): def set_targets(self): addons = [] - for addon in [self.src_node.get_addon(name) + for addon in [self.src_node.get_addon(name, cached=False) for name in settings.ADDONS_ARCHIVABLE if settings.ADDONS_ARCHIVABLE[name] != 'none']: if not addon or not isinstance(addon, BaseStorageAddon) or not addon.complete: diff --git a/osf/models/mixins.py b/osf/models/mixins.py index b7fe97b7ece..62e93ccc991 100644 --- a/osf/models/mixins.py +++ b/osf/models/mixins.py @@ -554,12 +554,12 @@ def get_or_add_addon(self, name, *args, **kwargs): return addon return self.add_addon(name, *args, **kwargs) - def get_addon(self, name, is_deleted=False, auth=None): + def get_addon(self, name, is_deleted=False, auth=None, cached=True): # Avoid test-breakages by avoiding early access to the request context if name not in self.OSF_HOSTED_ADDONS: request, user_id = get_request_and_user_id() if flag_is_active(request, features.ENABLE_GV): - return self._get_addon_from_gv(gv_pk=name, requesting_user_id=user_id, auth=auth) + return self._get_addon_from_gv(gv_pk=name, requesting_user_id=user_id, auth=auth, cached=cached) try: settings_model = self._settings_model(name) diff --git a/osf/models/node.py b/osf/models/node.py index 83d646cc717..abb4cda878f 100644 --- a/osf/models/node.py +++ b/osf/models/node.py @@ -2466,21 +2466,28 @@ def _remove_from_associated_collections(self, auth=None, force=False): force=True ) - def _get_addon_from_gv(self, gv_pk, requesting_user_id, auth=None): + def _get_addons_from_gv_without_caching(self, gv_pk, requesting_user_id, auth=None): + requesting_user = OSFUser.load(requesting_user_id) + services = gv_translations.get_external_services(requesting_user) + for service in services: + if service.short_name == gv_pk: + break + else: + return None + + return self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) + + def _get_addon_from_gv(self, gv_pk, requesting_user_id, auth=None, cached=True): request = get_current_request() # This is to avoid making multiple requests to GV # within the lifespan of one request on the OSF side - try: - gv_addons = request.gv_addons - except AttributeError: - requesting_user = OSFUser.load(requesting_user_id) - services = gv_translations.get_external_services(requesting_user) - for service in services: - if service.short_name == gv_pk: - break - else: - return None - gv_addons = request.gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) + if cached: + try: + gv_addons = request.gv_addons + except AttributeError: + gv_addons = request.gv_addons = self._get_addons_from_gv_without_caching(gv_pk, requesting_user_id, auth=auth) + else: + gv_addons = self._get_addons_from_gv_without_caching(gv_pk, requesting_user_id, auth=auth) for item in gv_addons: if item.short_name == gv_pk: From f0555f90931d4a2a280f119c7f2a468c0a24ff3e Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 17 Jun 2025 17:18:33 +0300 Subject: [PATCH 04/21] added storage allocation for draft registrations --- admin/draft_registrations/urls.py | 2 + admin/draft_registrations/views.py | 14 +++- admin/nodes/queries.py | 29 +++++++++ admin/nodes/views.py | 65 +++++++------------ .../templates/draft_registrations/detail.html | 7 +- .../draft_registrations/storage_usage.html | 35 ++++++++++ ...om_storage_usage_limit_private_and_more.py | 23 +++++++ osf/models/registrations.py | 3 + website/archiver/tasks.py | 11 +++- 9 files changed, 140 insertions(+), 49 deletions(-) create mode 100644 admin/nodes/queries.py create mode 100644 admin/templates/draft_registrations/storage_usage.html create mode 100644 osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py diff --git a/admin/draft_registrations/urls.py b/admin/draft_registrations/urls.py index 84d8ef6525c..abc5b18091f 100644 --- a/admin/draft_registrations/urls.py +++ b/admin/draft_registrations/urls.py @@ -9,4 +9,6 @@ urlpatterns = [ re_path(r'^$', views.UserDraftRegistrationSearchView.as_view(), name='search'), re_path(r'^(?P\w+)/$', views.DraftRegistrationView.as_view(), name='detail'), + re_path(r'^(?P\w+)/modify_storage_usage/$', views.DraftRegisrationModifyStorageUsage.as_view(), + name='adjust-draft-registration-storage-usage'), ] diff --git a/admin/draft_registrations/views.py b/admin/draft_registrations/views.py index 165407e3cc3..14512f476cf 100644 --- a/admin/draft_registrations/views.py +++ b/admin/draft_registrations/views.py @@ -6,13 +6,20 @@ from django.views.generic import DetailView from admin.base.forms import GuidForm +from admin.nodes.queries import STORAGE_USAGE_QUERY +from admin.nodes.views import StorageMixin + from osf.models.registrations import DraftRegistration class DraftRegistrationMixin(PermissionRequiredMixin): def get_object(self): - draft_registration = DraftRegistration.load(self.kwargs['draft_registration_id']) + draft_registration = DraftRegistration.objects.filter( + _id=self.kwargs['draft_registration_id'] + ).annotate( + **STORAGE_USAGE_QUERY + ).first() draft_registration.guid = draft_registration._id return draft_registration @@ -52,3 +59,8 @@ def get_context_data(self, **kwargs): return super().get_context_data(**{ 'draft_registration': draft_registration }, **kwargs) + + +class DraftRegisrationModifyStorageUsage(DraftRegistrationMixin, StorageMixin): + template_name = 'draft_registrations/detail.html' + permission_required = 'osf.change_draftregistration' diff --git a/admin/nodes/queries.py b/admin/nodes/queries.py new file mode 100644 index 00000000000..9cb0b09ff2c --- /dev/null +++ b/admin/nodes/queries.py @@ -0,0 +1,29 @@ +from django.db.models import F, Case, When, IntegerField + +from website import settings + + +STORAGE_USAGE_QUERY = { + 'public_cap': Case( + When( + custom_storage_usage_limit_public=None, + then=settings.STORAGE_LIMIT_PUBLIC, + ), + When( + custom_storage_usage_limit_public__gt=0, + then=F('custom_storage_usage_limit_public'), + ), + output_field=IntegerField() + ), + 'private_cap': Case( + When( + custom_storage_usage_limit_private=None, + then=settings.STORAGE_LIMIT_PRIVATE, + ), + When( + custom_storage_usage_limit_private__gt=0, + then=F('custom_storage_usage_limit_private'), + ), + output_field=IntegerField() + ) +} diff --git a/admin/nodes/views.py b/admin/nodes/views.py index 2d4f0c1194f..2e0aa243e76 100644 --- a/admin/nodes/views.py +++ b/admin/nodes/views.py @@ -6,7 +6,7 @@ from django.utils import timezone from django.core.exceptions import PermissionDenied, ValidationError from django.urls import NoReverseMatch -from django.db.models import F, Case, When, IntegerField +from django.db.models import F from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin from django.http import HttpResponse @@ -23,6 +23,7 @@ from admin.base.views import GuidView from admin.base.forms import GuidForm from admin.notifications.views import detect_duplicate_notifications, delete_selected_notifications +from admin.nodes.queries import STORAGE_USAGE_QUERY from api.share.utils import update_share from api.caching.tasks import update_storage_usage_cache @@ -61,34 +62,33 @@ def get_object(self): guids___id=self.kwargs['guid'] ).annotate( guid=F('guids___id'), - public_cap=Case( - When( - custom_storage_usage_limit_public=None, - then=settings.STORAGE_LIMIT_PUBLIC, - ), - When( - custom_storage_usage_limit_public__gt=0, - then=F('custom_storage_usage_limit_public'), - ), - output_field=IntegerField() - ), - private_cap=Case( - When( - custom_storage_usage_limit_private=None, - then=settings.STORAGE_LIMIT_PRIVATE, - ), - When( - custom_storage_usage_limit_private__gt=0, - then=F('custom_storage_usage_limit_private'), - ), - output_field=IntegerField() - ) + **STORAGE_USAGE_QUERY ).get() def get_success_url(self): return reverse('nodes:node', kwargs={'guid': self.kwargs['guid']}) +class StorageMixin(View): + + def post(self, request, *args, **kwargs): + object = self.get_object() + new_private_cap = request.POST.get('private-cap-input') + new_public_cap = request.POST.get('public-cap-input') + + object_private_cap = object.custom_storage_usage_limit_private or settings.STORAGE_LIMIT_PRIVATE + object_public_cap = object.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC + + if float(new_private_cap) != object_private_cap: + object.custom_storage_usage_limit_private = new_private_cap + + if float(new_public_cap) != object_public_cap: + object.custom_storage_usage_limit_public = new_public_cap + + object.save() + return redirect(self.get_success_url()) + + class NodeView(NodeMixin, GuidView): """ Allows authorized users to view node info. """ @@ -601,28 +601,11 @@ def post(self, request, *args, **kwargs): return redirect(self.get_success_url()) -class NodeModifyStorageUsage(NodeMixin, View): +class NodeModifyStorageUsage(NodeMixin, StorageMixin): """ Allows an authorized user to view a node's storage usage info and set their public/private storage cap. """ permission_required = 'osf.change_node' - def post(self, request, *args, **kwargs): - node = self.get_object() - new_private_cap = request.POST.get('private-cap-input') - new_public_cap = request.POST.get('public-cap-input') - - node_private_cap = node.custom_storage_usage_limit_private or settings.STORAGE_LIMIT_PRIVATE - node_public_cap = node.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC - - if float(new_private_cap) != node_private_cap: - node.custom_storage_usage_limit_private = new_private_cap - - if float(new_public_cap) != node_public_cap: - node.custom_storage_usage_limit_public = new_public_cap - - node.save() - return redirect(self.get_success_url()) - class NodeRecalculateStorage(NodeMixin, View): """ Allows an authorized user to manually set a node's storage cache by recalculating the value. diff --git a/admin/templates/draft_registrations/detail.html b/admin/templates/draft_registrations/detail.html index e20be95469a..6c88c5c9c76 100644 --- a/admin/templates/draft_registrations/detail.html +++ b/admin/templates/draft_registrations/detail.html @@ -54,12 +54,7 @@

Draft Registration: {{ draft_registration.title }} - Node storage usage - - Current usage: {{ draft_registration.storage_usage }}
- - + {% include "draft_registrations/storage_usage.html" with draft_registration=draft_registration %} diff --git a/admin/templates/draft_registrations/storage_usage.html b/admin/templates/draft_registrations/storage_usage.html new file mode 100644 index 00000000000..3920f82b4e9 --- /dev/null +++ b/admin/templates/draft_registrations/storage_usage.html @@ -0,0 +1,35 @@ +{% load node_extras %} + + + Draft registration storage usage + + Public cap: {{ draft_registration.public_cap|floatformat:0 }} GB
+ Private cap: {{ draft_registration.private_cap|floatformat:0 }} GB
+ + Modify Storage Caps + + + + diff --git a/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py b/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py new file mode 100644 index 00000000000..f6a8d253758 --- /dev/null +++ b/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.15 on 2025-06-17 13:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osf', '0029_remove_abstractnode_keenio_read_key'), + ] + + operations = [ + migrations.AddField( + model_name='draftregistration', + name='custom_storage_usage_limit_private', + field=models.DecimalField(blank=True, decimal_places=9, max_digits=100, null=True), + ), + migrations.AddField( + model_name='draftregistration', + name='custom_storage_usage_limit_public', + field=models.DecimalField(blank=True, decimal_places=9, max_digits=100, null=True), + ), + ] diff --git a/osf/models/registrations.py b/osf/models/registrations.py index b92aed1e8e2..03f7795e7bf 100644 --- a/osf/models/registrations.py +++ b/osf/models/registrations.py @@ -980,6 +980,9 @@ class DraftRegistration(ObjectIDMixin, RegistrationResponseMixin, DirtyFieldsMix default=get_default_id, ) + custom_storage_usage_limit_public = models.DecimalField(decimal_places=9, max_digits=100, null=True, blank=True) + custom_storage_usage_limit_private = models.DecimalField(decimal_places=9, max_digits=100, null=True, blank=True) + # Dictionary field mapping question id to a question's comments and answer # { # : { diff --git a/website/archiver/tasks.py b/website/archiver/tasks.py index f8c3b18feb1..d2da6686cf0 100644 --- a/website/archiver/tasks.py +++ b/website/archiver/tasks.py @@ -1,5 +1,6 @@ import requests import json +import math from furl import furl from rest_framework import status as http_status @@ -279,7 +280,15 @@ def archive_node(stat_results, job_pk): dst.title, targets=stat_results ) - if (NO_ARCHIVE_LIMIT not in job.initiator.system_tags) and (stat_result.disk_usage > settings.MAX_ARCHIVE_SIZE): + + draft_registration = DraftRegistration.objects.get(registered_node=dst) + disk_usage_in_gb = stat_result.disk_usage / math.pow(1024, 3) + if src.is_public: + limit = draft_registration.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC + else: + limit = draft_registration.custom_storage_usage_limit_private or settings.STORAGE_LIMIT_PRIVATE + + if (NO_ARCHIVE_LIMIT not in job.initiator.system_tags) and (disk_usage_in_gb > limit): raise ArchiverSizeExceeded(result=stat_result) else: if not stat_result.targets: From 442b2409b9afae2bc0ee532e5fbe959f3a760a5a Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 17 Jun 2025 19:08:54 +0300 Subject: [PATCH 05/21] added tests --- osf_tests/test_archiver.py | 120 +++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/osf_tests/test_archiver.py b/osf_tests/test_archiver.py index 59c178b839d..cf4721ca3b8 100644 --- a/osf_tests/test_archiver.py +++ b/osf_tests/test_archiver.py @@ -515,6 +515,126 @@ def test_archive_node_no_archive_size_limit(self, mock_archive_addon): job_pk=self.archive_job._id, ) + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail_and_use_updated_public_storage_size_limit(self, mock_archive_addon): + self.src.is_public = True + self.src.save() + draft_reg = DraftRegistration.objects.get(registered_node=self.dst) + draft_reg.custom_storage_usage_limit_public = 2 + draft_reg.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) + 1 # 1GB + 1 kilobyte + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + with self.assertRaises(ArchiverSizeExceeded): + archive_node(results, self.archive_job._id) + + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail_and_use_updated_private_storage_size_limit(self, mock_archive_addon): + draft_reg = DraftRegistration.objects.get(registered_node=self.dst) + draft_reg.custom_storage_usage_limit_private = 2 + draft_reg.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) + 1 # 1GB + 1 kilobyte + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + with self.assertRaises(ArchiverSizeExceeded): + archive_node(results, self.archive_job._id) + + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_success_and_use_updated_public_storage_size_limit(self, mock_archive_addon): + self.src.is_public = True + self.src.save() + draft_reg = DraftRegistration.objects.get(registered_node=self.dst) + draft_reg.custom_storage_usage_limit_public = 3 + draft_reg.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + archive_node(results, self.archive_job._id) + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_success_and_use_updated_private_storage_size_limit(self, mock_archive_addon): + draft_reg = DraftRegistration.objects.get(registered_node=self.dst) + draft_reg.custom_storage_usage_limit_private = 3 + draft_reg.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + archive_node(results, self.archive_job._id) + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail_and_use_default_public_storage_size_limit(self, mock_archive_addon): + self.src.is_public = True + self.src.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + settings.STORAGE_LIMIT_PUBLIC = 4 + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) * 2 # 2GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + with self.assertRaises(ArchiverSizeExceeded): + archive_node(results, self.archive_job._id) + + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail_and_use_default_private_storage_size_limit(self, mock_archive_addon): + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + settings.STORAGE_LIMIT_PRIVATE = 3 + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) * 2 # 2 GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + with self.assertRaises(ArchiverSizeExceeded): + archive_node(results, self.archive_job._id) + + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_success_and_use_default_public_storage_size_limit(self, mock_archive_addon): + self.src.is_public = True + self.src.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + settings.STORAGE_LIMIT_PUBLIC = 4 + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + archive_node(results, self.archive_job._id) + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_success_and_use_default_private_storage_size_limit(self, mock_archive_addon): + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + settings.STORAGE_LIMIT_PRIVATE = 4 + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + archive_node(results, self.archive_job._id) + FILE_TREE['children'][0]['size'] = '128' + @mock.patch('website.archiver.tasks.make_copy_request.delay') def test_archive_addon(self, mock_make_copy_request): archive_addon('osfstorage', self.archive_job._id) From 8d67b3f51c9b9ed38767e8cf1f56567a21901460 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 18 Jun 2025 11:03:00 +0300 Subject: [PATCH 06/21] fixed tests, removed unused MAX_ARCHIVE_SIZE constant --- osf_tests/test_archiver.py | 25 +++++++++++++------------ website/archiver/tasks.py | 3 +-- website/settings/defaults.py | 2 -- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/osf_tests/test_archiver.py b/osf_tests/test_archiver.py index cf4721ca3b8..6b633db82e9 100644 --- a/osf_tests/test_archiver.py +++ b/osf_tests/test_archiver.py @@ -463,7 +463,7 @@ def test_stat_addon(self): @mock.patch('website.archiver.tasks.archive_addon.delay') def test_archive_node_pass(self, mock_archive_addon): - settings.MAX_ARCHIVE_SIZE = 1024 ** 3 + settings.STORAGE_LIMIT_PRIVATE = 1 # 1gb with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage']] @@ -474,8 +474,9 @@ def test_archive_node_pass(self, mock_archive_addon): ) @use_fake_addons - def test_archive_node_fail(self): - settings.MAX_ARCHIVE_SIZE = 100 + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail(self, mock_archive_addon): + settings.STORAGE_LIMIT_PRIVATE = 500 / 1024 ** 3 # 500 KB results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] with pytest.raises(ArchiverSizeExceeded): # Note: Requires task_eager_propagates = True in celery archive_node.apply(args=(results, self.archive_job._id)) @@ -503,7 +504,7 @@ def empty_file_tree(user, version): @use_fake_addons @mock.patch('website.archiver.tasks.archive_addon.delay') def test_archive_node_no_archive_size_limit(self, mock_archive_addon): - settings.MAX_ARCHIVE_SIZE = 100 + settings.STORAGE_LIMIT_PRIVATE = 100 / 1024 ** 3 # 100KB self.archive_job.initiator.add_system_tag(NO_ARCHIVE_LIMIT) self.archive_job.initiator.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: @@ -524,7 +525,7 @@ def test_archive_node_fail_and_use_updated_public_storage_size_limit(self, mock_ draft_reg.custom_storage_usage_limit_public = 2 draft_reg.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) + 1 # 1GB + 1 kilobyte + FILE_TREE['children'][0]['size'] = 1024 ** 3 + 1 # 1GB + 1 kilobyte mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -540,7 +541,7 @@ def test_archive_node_fail_and_use_updated_private_storage_size_limit(self, mock draft_reg.custom_storage_usage_limit_private = 2 draft_reg.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) + 1 # 1GB + 1 kilobyte + FILE_TREE['children'][0]['size'] = 1024 ** 3 + 1 # 1GB + 1 kilobyte mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -558,7 +559,7 @@ def test_archive_node_success_and_use_updated_public_storage_size_limit(self, mo draft_reg.custom_storage_usage_limit_public = 3 draft_reg.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 # 1GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -572,7 +573,7 @@ def test_archive_node_success_and_use_updated_private_storage_size_limit(self, m draft_reg.custom_storage_usage_limit_private = 3 draft_reg.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 # 1GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -586,7 +587,7 @@ def test_archive_node_fail_and_use_default_public_storage_size_limit(self, mock_ self.src.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: settings.STORAGE_LIMIT_PUBLIC = 4 - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) * 2 # 2GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 * 2 # 2GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -600,7 +601,7 @@ def test_archive_node_fail_and_use_default_public_storage_size_limit(self, mock_ def test_archive_node_fail_and_use_default_private_storage_size_limit(self, mock_archive_addon): with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: settings.STORAGE_LIMIT_PRIVATE = 3 - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) * 2 # 2 GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 * 2 # 2 GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -616,7 +617,7 @@ def test_archive_node_success_and_use_default_public_storage_size_limit(self, mo self.src.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: settings.STORAGE_LIMIT_PUBLIC = 4 - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 # 1GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -628,7 +629,7 @@ def test_archive_node_success_and_use_default_public_storage_size_limit(self, mo def test_archive_node_success_and_use_default_private_storage_size_limit(self, mock_archive_addon): with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: settings.STORAGE_LIMIT_PRIVATE = 4 - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 # 1GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] diff --git a/website/archiver/tasks.py b/website/archiver/tasks.py index d2da6686cf0..a3920ef8c96 100644 --- a/website/archiver/tasks.py +++ b/website/archiver/tasks.py @@ -1,6 +1,5 @@ import requests import json -import math from furl import furl from rest_framework import status as http_status @@ -282,7 +281,7 @@ def archive_node(stat_results, job_pk): ) draft_registration = DraftRegistration.objects.get(registered_node=dst) - disk_usage_in_gb = stat_result.disk_usage / math.pow(1024, 3) + disk_usage_in_gb = stat_result.disk_usage / 1024 ** 3 if src.is_public: limit = draft_registration.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC else: diff --git a/website/settings/defaults.py b/website/settings/defaults.py index d891e886873..fb3b1b89272 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -372,8 +372,6 @@ def parent_dir(path): ###### ARCHIVER ########### ARCHIVE_PROVIDER = 'osfstorage' -MAX_ARCHIVE_SIZE = 5 * 1024 ** 3 # == math.pow(1024, 3) == 1 GB - ARCHIVE_TIMEOUT_TIMEDELTA = timedelta(1) # 24 hours STUCK_FILES_DELETE_TIMEOUT = timedelta(days=45) # Registration files stuck for x days are marked as deleted. From 78c89b68e10744a0189a59a32567f29891c1ba2b Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 18 Jun 2025 11:06:19 +0300 Subject: [PATCH 07/21] simplified code readability --- website/archiver/tasks.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/website/archiver/tasks.py b/website/archiver/tasks.py index a3920ef8c96..376d28ac531 100644 --- a/website/archiver/tasks.py +++ b/website/archiver/tasks.py @@ -289,19 +289,19 @@ def archive_node(stat_results, job_pk): if (NO_ARCHIVE_LIMIT not in job.initiator.system_tags) and (disk_usage_in_gb > limit): raise ArchiverSizeExceeded(result=stat_result) - else: - if not stat_result.targets: - job.status = ARCHIVER_SUCCESS - job.save() - for result in stat_result.targets: - if not result['num_files']: - job.update_target(result['target_name'], ARCHIVER_SUCCESS) - else: - archive_addon.delay( - addon_short_name=result['target_name'], - job_pk=job_pk - ) - project_signals.archive_callback.send(dst) + + if not stat_result.targets: + job.status = ARCHIVER_SUCCESS + job.save() + for result in stat_result.targets: + if not result['num_files']: + job.update_target(result['target_name'], ARCHIVER_SUCCESS) + else: + archive_addon.delay( + addon_short_name=result['target_name'], + job_pk=job_pk + ) + project_signals.archive_callback.send(dst) def archive(job_pk): From 383f5ae47b83c9a1cd653feb9caeaef3a0e85377 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 4 Jun 2025 17:07:43 +0300 Subject: [PATCH 08/21] remove caching for ascendants --- osf/models/node.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/osf/models/node.py b/osf/models/node.py index 87f75e8d692..c81e1dc80a1 100644 --- a/osf/models/node.py +++ b/osf/models/node.py @@ -57,7 +57,7 @@ ) from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField from osf.utils.fields import NonNaiveDateTimeField -from osf.utils.requests import get_request_and_user_id, string_type_request_headers, get_current_request +from osf.utils.requests import get_request_and_user_id, string_type_request_headers from osf.utils.workflows import CollectionSubmissionStates from osf.utils import sanitize from website import language, settings @@ -2465,21 +2465,15 @@ def _remove_from_associated_collections(self, auth=None, force=False): ) def _get_addon_from_gv(self, gv_pk, requesting_user_id, auth=None): - request = get_current_request() - # This is to avoid making multiple requests to GV - # within the lifespan of one request on the OSF side - try: - gv_addons = request.gv_addons - except AttributeError: - requesting_user = OSFUser.load(requesting_user_id) - services = gv_translations.get_external_services(requesting_user) - for service in services: - if service.short_name == gv_pk: - break - else: - return None - gv_addons = request.gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) + requesting_user = OSFUser.load(requesting_user_id) + services = gv_translations.get_external_services(requesting_user) + for service in services: + if service.short_name == gv_pk: + break + else: + return None + gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) for item in gv_addons: if item.short_name == gv_pk: return item From a927f7dc9a49c1df18dea244496e3fe9d0aa5d1e Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 10 Jun 2025 13:17:04 +0300 Subject: [PATCH 09/21] Revert "remove caching for ascendants" This reverts commit 15915b0ae60e8967eb4f8fccfb286a3bff04bbe7. --- osf/models/node.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/osf/models/node.py b/osf/models/node.py index c81e1dc80a1..87f75e8d692 100644 --- a/osf/models/node.py +++ b/osf/models/node.py @@ -57,7 +57,7 @@ ) from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField from osf.utils.fields import NonNaiveDateTimeField -from osf.utils.requests import get_request_and_user_id, string_type_request_headers +from osf.utils.requests import get_request_and_user_id, string_type_request_headers, get_current_request from osf.utils.workflows import CollectionSubmissionStates from osf.utils import sanitize from website import language, settings @@ -2465,15 +2465,21 @@ def _remove_from_associated_collections(self, auth=None, force=False): ) def _get_addon_from_gv(self, gv_pk, requesting_user_id, auth=None): - requesting_user = OSFUser.load(requesting_user_id) - services = gv_translations.get_external_services(requesting_user) - for service in services: - if service.short_name == gv_pk: - break - else: - return None + request = get_current_request() + # This is to avoid making multiple requests to GV + # within the lifespan of one request on the OSF side + try: + gv_addons = request.gv_addons + except AttributeError: + requesting_user = OSFUser.load(requesting_user_id) + services = gv_translations.get_external_services(requesting_user) + for service in services: + if service.short_name == gv_pk: + break + else: + return None + gv_addons = request.gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) - gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) for item in gv_addons: if item.short_name == gv_pk: return item From 4b723fb81e44ff7e00cbf44be5969bb866312a21 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 10 Jun 2025 17:01:21 +0300 Subject: [PATCH 10/21] use GV addons caching when it's needed --- osf/models/archive.py | 2 +- osf/models/mixins.py | 4 ++-- osf/models/node.py | 31 +++++++++++++++++++------------ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/osf/models/archive.py b/osf/models/archive.py index 052d918a99c..9e622764ca7 100644 --- a/osf/models/archive.py +++ b/osf/models/archive.py @@ -146,7 +146,7 @@ def _set_target(self, addon_short_name): def set_targets(self): addons = [] - for addon in [self.src_node.get_addon(name) + for addon in [self.src_node.get_addon(name, cached=False) for name in settings.ADDONS_ARCHIVABLE if settings.ADDONS_ARCHIVABLE[name] != 'none']: if not addon or not isinstance(addon, BaseStorageAddon) or not addon.complete: diff --git a/osf/models/mixins.py b/osf/models/mixins.py index 7aaf27b5f89..4b20e5119b1 100644 --- a/osf/models/mixins.py +++ b/osf/models/mixins.py @@ -554,12 +554,12 @@ def get_or_add_addon(self, name, *args, **kwargs): return addon return self.add_addon(name, *args, **kwargs) - def get_addon(self, name, is_deleted=False, auth=None): + def get_addon(self, name, is_deleted=False, auth=None, cached=True): # Avoid test-breakages by avoiding early access to the request context if name not in self.OSF_HOSTED_ADDONS: request, user_id = get_request_and_user_id() if flag_is_active(request, features.ENABLE_GV): - return self._get_addon_from_gv(gv_pk=name, requesting_user_id=user_id, auth=auth) + return self._get_addon_from_gv(gv_pk=name, requesting_user_id=user_id, auth=auth, cached=cached) try: settings_model = self._settings_model(name) diff --git a/osf/models/node.py b/osf/models/node.py index 87f75e8d692..7bc2da542f9 100644 --- a/osf/models/node.py +++ b/osf/models/node.py @@ -2464,21 +2464,28 @@ def _remove_from_associated_collections(self, auth=None, force=False): force=True ) - def _get_addon_from_gv(self, gv_pk, requesting_user_id, auth=None): + def _get_addons_from_gv_without_caching(self, gv_pk, requesting_user_id, auth=None): + requesting_user = OSFUser.load(requesting_user_id) + services = gv_translations.get_external_services(requesting_user) + for service in services: + if service.short_name == gv_pk: + break + else: + return None + + return self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) + + def _get_addon_from_gv(self, gv_pk, requesting_user_id, auth=None, cached=True): request = get_current_request() # This is to avoid making multiple requests to GV # within the lifespan of one request on the OSF side - try: - gv_addons = request.gv_addons - except AttributeError: - requesting_user = OSFUser.load(requesting_user_id) - services = gv_translations.get_external_services(requesting_user) - for service in services: - if service.short_name == gv_pk: - break - else: - return None - gv_addons = request.gv_addons = self._get_addons_from_gv(requesting_user_id, service.type, auth=auth) + if cached: + try: + gv_addons = request.gv_addons + except AttributeError: + gv_addons = request.gv_addons = self._get_addons_from_gv_without_caching(gv_pk, requesting_user_id, auth=auth) + else: + gv_addons = self._get_addons_from_gv_without_caching(gv_pk, requesting_user_id, auth=auth) for item in gv_addons: if item.short_name == gv_pk: From 280c16a114e74c91fc12dd04207f782a1f72fc73 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 17 Jun 2025 17:18:33 +0300 Subject: [PATCH 11/21] added storage allocation for draft registrations --- admin/draft_registrations/urls.py | 2 + admin/draft_registrations/views.py | 14 +++- admin/nodes/queries.py | 29 +++++++++ admin/nodes/views.py | 65 +++++++------------ .../templates/draft_registrations/detail.html | 7 +- .../draft_registrations/storage_usage.html | 35 ++++++++++ ...om_storage_usage_limit_private_and_more.py | 23 +++++++ osf/models/registrations.py | 3 + website/archiver/tasks.py | 11 +++- 9 files changed, 140 insertions(+), 49 deletions(-) create mode 100644 admin/nodes/queries.py create mode 100644 admin/templates/draft_registrations/storage_usage.html create mode 100644 osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py diff --git a/admin/draft_registrations/urls.py b/admin/draft_registrations/urls.py index 84d8ef6525c..abc5b18091f 100644 --- a/admin/draft_registrations/urls.py +++ b/admin/draft_registrations/urls.py @@ -9,4 +9,6 @@ urlpatterns = [ re_path(r'^$', views.UserDraftRegistrationSearchView.as_view(), name='search'), re_path(r'^(?P\w+)/$', views.DraftRegistrationView.as_view(), name='detail'), + re_path(r'^(?P\w+)/modify_storage_usage/$', views.DraftRegisrationModifyStorageUsage.as_view(), + name='adjust-draft-registration-storage-usage'), ] diff --git a/admin/draft_registrations/views.py b/admin/draft_registrations/views.py index 165407e3cc3..14512f476cf 100644 --- a/admin/draft_registrations/views.py +++ b/admin/draft_registrations/views.py @@ -6,13 +6,20 @@ from django.views.generic import DetailView from admin.base.forms import GuidForm +from admin.nodes.queries import STORAGE_USAGE_QUERY +from admin.nodes.views import StorageMixin + from osf.models.registrations import DraftRegistration class DraftRegistrationMixin(PermissionRequiredMixin): def get_object(self): - draft_registration = DraftRegistration.load(self.kwargs['draft_registration_id']) + draft_registration = DraftRegistration.objects.filter( + _id=self.kwargs['draft_registration_id'] + ).annotate( + **STORAGE_USAGE_QUERY + ).first() draft_registration.guid = draft_registration._id return draft_registration @@ -52,3 +59,8 @@ def get_context_data(self, **kwargs): return super().get_context_data(**{ 'draft_registration': draft_registration }, **kwargs) + + +class DraftRegisrationModifyStorageUsage(DraftRegistrationMixin, StorageMixin): + template_name = 'draft_registrations/detail.html' + permission_required = 'osf.change_draftregistration' diff --git a/admin/nodes/queries.py b/admin/nodes/queries.py new file mode 100644 index 00000000000..9cb0b09ff2c --- /dev/null +++ b/admin/nodes/queries.py @@ -0,0 +1,29 @@ +from django.db.models import F, Case, When, IntegerField + +from website import settings + + +STORAGE_USAGE_QUERY = { + 'public_cap': Case( + When( + custom_storage_usage_limit_public=None, + then=settings.STORAGE_LIMIT_PUBLIC, + ), + When( + custom_storage_usage_limit_public__gt=0, + then=F('custom_storage_usage_limit_public'), + ), + output_field=IntegerField() + ), + 'private_cap': Case( + When( + custom_storage_usage_limit_private=None, + then=settings.STORAGE_LIMIT_PRIVATE, + ), + When( + custom_storage_usage_limit_private__gt=0, + then=F('custom_storage_usage_limit_private'), + ), + output_field=IntegerField() + ) +} diff --git a/admin/nodes/views.py b/admin/nodes/views.py index 40cf261945d..bd7b1633038 100644 --- a/admin/nodes/views.py +++ b/admin/nodes/views.py @@ -6,7 +6,7 @@ from django.utils import timezone from django.core.exceptions import PermissionDenied, ValidationError from django.urls import NoReverseMatch -from django.db.models import F, Case, When, IntegerField +from django.db.models import F from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin from django.http import HttpResponse @@ -23,6 +23,7 @@ from admin.base.views import GuidView from admin.base.forms import GuidForm from admin.notifications.views import detect_duplicate_notifications, delete_selected_notifications +from admin.nodes.queries import STORAGE_USAGE_QUERY from api.share.utils import update_share from api.caching.tasks import update_storage_usage_cache @@ -62,34 +63,33 @@ def get_object(self): guids___id=self.kwargs['guid'] ).annotate( guid=F('guids___id'), - public_cap=Case( - When( - custom_storage_usage_limit_public=None, - then=settings.STORAGE_LIMIT_PUBLIC, - ), - When( - custom_storage_usage_limit_public__gt=0, - then=F('custom_storage_usage_limit_public'), - ), - output_field=IntegerField() - ), - private_cap=Case( - When( - custom_storage_usage_limit_private=None, - then=settings.STORAGE_LIMIT_PRIVATE, - ), - When( - custom_storage_usage_limit_private__gt=0, - then=F('custom_storage_usage_limit_private'), - ), - output_field=IntegerField() - ) + **STORAGE_USAGE_QUERY ).get() def get_success_url(self): return reverse('nodes:node', kwargs={'guid': self.kwargs['guid']}) +class StorageMixin(View): + + def post(self, request, *args, **kwargs): + object = self.get_object() + new_private_cap = request.POST.get('private-cap-input') + new_public_cap = request.POST.get('public-cap-input') + + object_private_cap = object.custom_storage_usage_limit_private or settings.STORAGE_LIMIT_PRIVATE + object_public_cap = object.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC + + if float(new_private_cap) != object_private_cap: + object.custom_storage_usage_limit_private = new_private_cap + + if float(new_public_cap) != object_public_cap: + object.custom_storage_usage_limit_public = new_public_cap + + object.save() + return redirect(self.get_success_url()) + + class NodeView(NodeMixin, GuidView): """ Allows authorized users to view node info. """ @@ -624,28 +624,11 @@ def post(self, request, *args, **kwargs): return redirect(self.get_success_url()) -class NodeModifyStorageUsage(NodeMixin, View): +class NodeModifyStorageUsage(NodeMixin, StorageMixin): """ Allows an authorized user to view a node's storage usage info and set their public/private storage cap. """ permission_required = 'osf.change_node' - def post(self, request, *args, **kwargs): - node = self.get_object() - new_private_cap = request.POST.get('private-cap-input') - new_public_cap = request.POST.get('public-cap-input') - - node_private_cap = node.custom_storage_usage_limit_private or settings.STORAGE_LIMIT_PRIVATE - node_public_cap = node.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC - - if float(new_private_cap) != node_private_cap: - node.custom_storage_usage_limit_private = new_private_cap - - if float(new_public_cap) != node_public_cap: - node.custom_storage_usage_limit_public = new_public_cap - - node.save() - return redirect(self.get_success_url()) - class NodeRecalculateStorage(NodeMixin, View): """ Allows an authorized user to manually set a node's storage cache by recalculating the value. diff --git a/admin/templates/draft_registrations/detail.html b/admin/templates/draft_registrations/detail.html index e20be95469a..6c88c5c9c76 100644 --- a/admin/templates/draft_registrations/detail.html +++ b/admin/templates/draft_registrations/detail.html @@ -54,12 +54,7 @@

Draft Registration: {{ draft_registration.title }} - Node storage usage - - Current usage: {{ draft_registration.storage_usage }}
- - + {% include "draft_registrations/storage_usage.html" with draft_registration=draft_registration %} diff --git a/admin/templates/draft_registrations/storage_usage.html b/admin/templates/draft_registrations/storage_usage.html new file mode 100644 index 00000000000..3920f82b4e9 --- /dev/null +++ b/admin/templates/draft_registrations/storage_usage.html @@ -0,0 +1,35 @@ +{% load node_extras %} + + + Draft registration storage usage + + Public cap: {{ draft_registration.public_cap|floatformat:0 }} GB
+ Private cap: {{ draft_registration.private_cap|floatformat:0 }} GB
+ + Modify Storage Caps + + + + diff --git a/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py b/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py new file mode 100644 index 00000000000..f6a8d253758 --- /dev/null +++ b/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.15 on 2025-06-17 13:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osf', '0029_remove_abstractnode_keenio_read_key'), + ] + + operations = [ + migrations.AddField( + model_name='draftregistration', + name='custom_storage_usage_limit_private', + field=models.DecimalField(blank=True, decimal_places=9, max_digits=100, null=True), + ), + migrations.AddField( + model_name='draftregistration', + name='custom_storage_usage_limit_public', + field=models.DecimalField(blank=True, decimal_places=9, max_digits=100, null=True), + ), + ] diff --git a/osf/models/registrations.py b/osf/models/registrations.py index 2a7d5f501e2..ea9a9a6376f 100644 --- a/osf/models/registrations.py +++ b/osf/models/registrations.py @@ -988,6 +988,9 @@ class DraftRegistration(ObjectIDMixin, RegistrationResponseMixin, DirtyFieldsMix default=get_default_id, ) + custom_storage_usage_limit_public = models.DecimalField(decimal_places=9, max_digits=100, null=True, blank=True) + custom_storage_usage_limit_private = models.DecimalField(decimal_places=9, max_digits=100, null=True, blank=True) + # Dictionary field mapping question id to a question's comments and answer # { # : { diff --git a/website/archiver/tasks.py b/website/archiver/tasks.py index f8c3b18feb1..d2da6686cf0 100644 --- a/website/archiver/tasks.py +++ b/website/archiver/tasks.py @@ -1,5 +1,6 @@ import requests import json +import math from furl import furl from rest_framework import status as http_status @@ -279,7 +280,15 @@ def archive_node(stat_results, job_pk): dst.title, targets=stat_results ) - if (NO_ARCHIVE_LIMIT not in job.initiator.system_tags) and (stat_result.disk_usage > settings.MAX_ARCHIVE_SIZE): + + draft_registration = DraftRegistration.objects.get(registered_node=dst) + disk_usage_in_gb = stat_result.disk_usage / math.pow(1024, 3) + if src.is_public: + limit = draft_registration.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC + else: + limit = draft_registration.custom_storage_usage_limit_private or settings.STORAGE_LIMIT_PRIVATE + + if (NO_ARCHIVE_LIMIT not in job.initiator.system_tags) and (disk_usage_in_gb > limit): raise ArchiverSizeExceeded(result=stat_result) else: if not stat_result.targets: From 0f24479973080bc4c06c0eb9d04aaecef4524d5e Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 17 Jun 2025 19:08:54 +0300 Subject: [PATCH 12/21] added tests --- osf_tests/test_archiver.py | 120 +++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/osf_tests/test_archiver.py b/osf_tests/test_archiver.py index 59c178b839d..cf4721ca3b8 100644 --- a/osf_tests/test_archiver.py +++ b/osf_tests/test_archiver.py @@ -515,6 +515,126 @@ def test_archive_node_no_archive_size_limit(self, mock_archive_addon): job_pk=self.archive_job._id, ) + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail_and_use_updated_public_storage_size_limit(self, mock_archive_addon): + self.src.is_public = True + self.src.save() + draft_reg = DraftRegistration.objects.get(registered_node=self.dst) + draft_reg.custom_storage_usage_limit_public = 2 + draft_reg.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) + 1 # 1GB + 1 kilobyte + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + with self.assertRaises(ArchiverSizeExceeded): + archive_node(results, self.archive_job._id) + + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail_and_use_updated_private_storage_size_limit(self, mock_archive_addon): + draft_reg = DraftRegistration.objects.get(registered_node=self.dst) + draft_reg.custom_storage_usage_limit_private = 2 + draft_reg.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) + 1 # 1GB + 1 kilobyte + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + with self.assertRaises(ArchiverSizeExceeded): + archive_node(results, self.archive_job._id) + + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_success_and_use_updated_public_storage_size_limit(self, mock_archive_addon): + self.src.is_public = True + self.src.save() + draft_reg = DraftRegistration.objects.get(registered_node=self.dst) + draft_reg.custom_storage_usage_limit_public = 3 + draft_reg.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + archive_node(results, self.archive_job._id) + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_success_and_use_updated_private_storage_size_limit(self, mock_archive_addon): + draft_reg = DraftRegistration.objects.get(registered_node=self.dst) + draft_reg.custom_storage_usage_limit_private = 3 + draft_reg.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + archive_node(results, self.archive_job._id) + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail_and_use_default_public_storage_size_limit(self, mock_archive_addon): + self.src.is_public = True + self.src.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + settings.STORAGE_LIMIT_PUBLIC = 4 + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) * 2 # 2GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + with self.assertRaises(ArchiverSizeExceeded): + archive_node(results, self.archive_job._id) + + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail_and_use_default_private_storage_size_limit(self, mock_archive_addon): + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + settings.STORAGE_LIMIT_PRIVATE = 3 + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) * 2 # 2 GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + with self.assertRaises(ArchiverSizeExceeded): + archive_node(results, self.archive_job._id) + + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_success_and_use_default_public_storage_size_limit(self, mock_archive_addon): + self.src.is_public = True + self.src.save() + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + settings.STORAGE_LIMIT_PUBLIC = 4 + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + archive_node(results, self.archive_job._id) + FILE_TREE['children'][0]['size'] = '128' + + @use_fake_addons + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_success_and_use_default_private_storage_size_limit(self, mock_archive_addon): + with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: + settings.STORAGE_LIMIT_PRIVATE = 4 + FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + mock_file_tree.return_value = FILE_TREE + results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] + + archive_node(results, self.archive_job._id) + FILE_TREE['children'][0]['size'] = '128' + @mock.patch('website.archiver.tasks.make_copy_request.delay') def test_archive_addon(self, mock_make_copy_request): archive_addon('osfstorage', self.archive_job._id) From 0bc8f6f1530a4a033b3626efe183c8b983f90a3e Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 18 Jun 2025 11:03:00 +0300 Subject: [PATCH 13/21] fixed tests, removed unused MAX_ARCHIVE_SIZE constant --- osf_tests/test_archiver.py | 25 +++++++++++++------------ website/archiver/tasks.py | 3 +-- website/settings/defaults.py | 2 -- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/osf_tests/test_archiver.py b/osf_tests/test_archiver.py index cf4721ca3b8..6b633db82e9 100644 --- a/osf_tests/test_archiver.py +++ b/osf_tests/test_archiver.py @@ -463,7 +463,7 @@ def test_stat_addon(self): @mock.patch('website.archiver.tasks.archive_addon.delay') def test_archive_node_pass(self, mock_archive_addon): - settings.MAX_ARCHIVE_SIZE = 1024 ** 3 + settings.STORAGE_LIMIT_PRIVATE = 1 # 1gb with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage']] @@ -474,8 +474,9 @@ def test_archive_node_pass(self, mock_archive_addon): ) @use_fake_addons - def test_archive_node_fail(self): - settings.MAX_ARCHIVE_SIZE = 100 + @mock.patch('website.archiver.tasks.archive_addon.delay') + def test_archive_node_fail(self, mock_archive_addon): + settings.STORAGE_LIMIT_PRIVATE = 500 / 1024 ** 3 # 500 KB results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] with pytest.raises(ArchiverSizeExceeded): # Note: Requires task_eager_propagates = True in celery archive_node.apply(args=(results, self.archive_job._id)) @@ -503,7 +504,7 @@ def empty_file_tree(user, version): @use_fake_addons @mock.patch('website.archiver.tasks.archive_addon.delay') def test_archive_node_no_archive_size_limit(self, mock_archive_addon): - settings.MAX_ARCHIVE_SIZE = 100 + settings.STORAGE_LIMIT_PRIVATE = 100 / 1024 ** 3 # 100KB self.archive_job.initiator.add_system_tag(NO_ARCHIVE_LIMIT) self.archive_job.initiator.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: @@ -524,7 +525,7 @@ def test_archive_node_fail_and_use_updated_public_storage_size_limit(self, mock_ draft_reg.custom_storage_usage_limit_public = 2 draft_reg.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) + 1 # 1GB + 1 kilobyte + FILE_TREE['children'][0]['size'] = 1024 ** 3 + 1 # 1GB + 1 kilobyte mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -540,7 +541,7 @@ def test_archive_node_fail_and_use_updated_private_storage_size_limit(self, mock draft_reg.custom_storage_usage_limit_private = 2 draft_reg.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) + 1 # 1GB + 1 kilobyte + FILE_TREE['children'][0]['size'] = 1024 ** 3 + 1 # 1GB + 1 kilobyte mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -558,7 +559,7 @@ def test_archive_node_success_and_use_updated_public_storage_size_limit(self, mo draft_reg.custom_storage_usage_limit_public = 3 draft_reg.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 # 1GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -572,7 +573,7 @@ def test_archive_node_success_and_use_updated_private_storage_size_limit(self, m draft_reg.custom_storage_usage_limit_private = 3 draft_reg.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 # 1GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -586,7 +587,7 @@ def test_archive_node_fail_and_use_default_public_storage_size_limit(self, mock_ self.src.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: settings.STORAGE_LIMIT_PUBLIC = 4 - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) * 2 # 2GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 * 2 # 2GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -600,7 +601,7 @@ def test_archive_node_fail_and_use_default_public_storage_size_limit(self, mock_ def test_archive_node_fail_and_use_default_private_storage_size_limit(self, mock_archive_addon): with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: settings.STORAGE_LIMIT_PRIVATE = 3 - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) * 2 # 2 GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 * 2 # 2 GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -616,7 +617,7 @@ def test_archive_node_success_and_use_default_public_storage_size_limit(self, mo self.src.save() with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: settings.STORAGE_LIMIT_PUBLIC = 4 - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 # 1GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] @@ -628,7 +629,7 @@ def test_archive_node_success_and_use_default_public_storage_size_limit(self, mo def test_archive_node_success_and_use_default_private_storage_size_limit(self, mock_archive_addon): with mock.patch.object(BaseStorageAddon, '_get_file_tree') as mock_file_tree: settings.STORAGE_LIMIT_PRIVATE = 4 - FILE_TREE['children'][0]['size'] = math.pow(1024, 3) # 1GB + FILE_TREE['children'][0]['size'] = 1024 ** 3 # 1GB mock_file_tree.return_value = FILE_TREE results = [stat_addon(addon, self.archive_job._id) for addon in ['osfstorage', 'dropbox']] diff --git a/website/archiver/tasks.py b/website/archiver/tasks.py index d2da6686cf0..a3920ef8c96 100644 --- a/website/archiver/tasks.py +++ b/website/archiver/tasks.py @@ -1,6 +1,5 @@ import requests import json -import math from furl import furl from rest_framework import status as http_status @@ -282,7 +281,7 @@ def archive_node(stat_results, job_pk): ) draft_registration = DraftRegistration.objects.get(registered_node=dst) - disk_usage_in_gb = stat_result.disk_usage / math.pow(1024, 3) + disk_usage_in_gb = stat_result.disk_usage / 1024 ** 3 if src.is_public: limit = draft_registration.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC else: diff --git a/website/settings/defaults.py b/website/settings/defaults.py index d891e886873..fb3b1b89272 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -372,8 +372,6 @@ def parent_dir(path): ###### ARCHIVER ########### ARCHIVE_PROVIDER = 'osfstorage' -MAX_ARCHIVE_SIZE = 5 * 1024 ** 3 # == math.pow(1024, 3) == 1 GB - ARCHIVE_TIMEOUT_TIMEDELTA = timedelta(1) # 24 hours STUCK_FILES_DELETE_TIMEOUT = timedelta(days=45) # Registration files stuck for x days are marked as deleted. From 581cf5a7bd1c6d799ce14966d383ec8e2ff446be Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 18 Jun 2025 11:06:19 +0300 Subject: [PATCH 14/21] simplified code readability --- website/archiver/tasks.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/website/archiver/tasks.py b/website/archiver/tasks.py index a3920ef8c96..376d28ac531 100644 --- a/website/archiver/tasks.py +++ b/website/archiver/tasks.py @@ -289,19 +289,19 @@ def archive_node(stat_results, job_pk): if (NO_ARCHIVE_LIMIT not in job.initiator.system_tags) and (disk_usage_in_gb > limit): raise ArchiverSizeExceeded(result=stat_result) - else: - if not stat_result.targets: - job.status = ARCHIVER_SUCCESS - job.save() - for result in stat_result.targets: - if not result['num_files']: - job.update_target(result['target_name'], ARCHIVER_SUCCESS) - else: - archive_addon.delay( - addon_short_name=result['target_name'], - job_pk=job_pk - ) - project_signals.archive_callback.send(dst) + + if not stat_result.targets: + job.status = ARCHIVER_SUCCESS + job.save() + for result in stat_result.targets: + if not result['num_files']: + job.update_target(result['target_name'], ARCHIVER_SUCCESS) + else: + archive_addon.delay( + addon_short_name=result['target_name'], + job_pk=job_pk + ) + project_signals.archive_callback.send(dst) def archive(job_pk): From 1cb0be3f2f23887745cc0c79dc9a6253c3dc6452 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Mon, 23 Jun 2025 13:16:25 +0300 Subject: [PATCH 15/21] fixed migration conflict --- ...registration_custom_storage_usage_limit_private_and_more.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename osf/migrations/{0030_draftregistration_custom_storage_usage_limit_private_and_more.py => 0031_draftregistration_custom_storage_usage_limit_private_and_more.py} (91%) diff --git a/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py b/osf/migrations/0031_draftregistration_custom_storage_usage_limit_private_and_more.py similarity index 91% rename from osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py rename to osf/migrations/0031_draftregistration_custom_storage_usage_limit_private_and_more.py index f6a8d253758..b109159d593 100644 --- a/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py +++ b/osf/migrations/0031_draftregistration_custom_storage_usage_limit_private_and_more.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('osf', '0029_remove_abstractnode_keenio_read_key'), + ('osf', '0030_abstractnode__manual_guid'), ] operations = [ From bf2296efe1dc610d5865806a32a9fa59a4a7654e Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Mon, 23 Jun 2025 13:25:23 +0300 Subject: [PATCH 16/21] remove redundant migration --- ...om_storage_usage_limit_private_and_more.py | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py diff --git a/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py b/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py deleted file mode 100644 index f6a8d253758..00000000000 --- a/osf/migrations/0030_draftregistration_custom_storage_usage_limit_private_and_more.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 4.2.15 on 2025-06-17 13:42 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('osf', '0029_remove_abstractnode_keenio_read_key'), - ] - - operations = [ - migrations.AddField( - model_name='draftregistration', - name='custom_storage_usage_limit_private', - field=models.DecimalField(blank=True, decimal_places=9, max_digits=100, null=True), - ), - migrations.AddField( - model_name='draftregistration', - name='custom_storage_usage_limit_public', - field=models.DecimalField(blank=True, decimal_places=9, max_digits=100, null=True), - ), - ] From bfcccc99f0be1051c17d993c07db6c2f10be5f1f Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Mon, 23 Jun 2025 15:09:28 +0300 Subject: [PATCH 17/21] replace constant by a hardcoded actual value --- website/templates/emails/archive_size_exceeded_desk.html.mako | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/templates/emails/archive_size_exceeded_desk.html.mako b/website/templates/emails/archive_size_exceeded_desk.html.mako index 8b4376c1c0d..919530cdbaa 100644 --- a/website/templates/emails/archive_size_exceeded_desk.html.mako +++ b/website/templates/emails/archive_size_exceeded_desk.html.mako @@ -11,7 +11,7 @@ User: ${user.fullname} (${user.username}) [${user._id}] - Tried to register ${src.title} (${url}), but the resulting archive would have exceeded our caps for disk usage (${settings.MAX_ARCHIVE_SIZE / 1024 ** 3}GB). + Tried to register ${src.title} (${url}), but the resulting archive would have exceeded our caps for disk usage 5 GB.
A report is included below: From fe98fc5da4d4016f8b2aa198bf5573556d615f2f Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 24 Jun 2025 17:12:46 +0300 Subject: [PATCH 18/21] refactor storage allocation to use a common limit for private and public drafts --- admin/draft_registrations/views.py | 29 ++++++-- admin/nodes/queries.py | 29 -------- admin/nodes/views.py | 65 ++++++++++------- .../draft_registrations/storage_usage.html | 6 +- ...egistration_custom_storage_usage_limit.py} | 9 +-- osf/models/registrations.py | 3 +- osf_tests/test_archiver.py | 70 ++----------------- website/archiver/tasks.py | 5 +- 8 files changed, 78 insertions(+), 138 deletions(-) delete mode 100644 admin/nodes/queries.py rename osf/migrations/{0031_draftregistration_custom_storage_usage_limit_private_and_more.py => 0031_draftregistration_custom_storage_usage_limit.py} (52%) diff --git a/admin/draft_registrations/views.py b/admin/draft_registrations/views.py index 14512f476cf..e9462be4c6f 100644 --- a/admin/draft_registrations/views.py +++ b/admin/draft_registrations/views.py @@ -1,13 +1,13 @@ from django.urls import NoReverseMatch, reverse from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin +from django.db.models import F, Case, When, IntegerField from django.shortcuts import redirect from django.views.generic import FormView from django.views.generic import DetailView from admin.base.forms import GuidForm -from admin.nodes.queries import STORAGE_USAGE_QUERY -from admin.nodes.views import StorageMixin +from website import settings from osf.models.registrations import DraftRegistration @@ -18,7 +18,17 @@ def get_object(self): draft_registration = DraftRegistration.objects.filter( _id=self.kwargs['draft_registration_id'] ).annotate( - **STORAGE_USAGE_QUERY + cap=Case( + When( + custom_storage_usage_limit=None, + then=settings.STORAGE_LIMIT_PRIVATE, + ), + When( + custom_storage_usage_limit__gt=0, + then=F('custom_storage_usage_limit'), + ), + output_field=IntegerField() + ) ).first() draft_registration.guid = draft_registration._id return draft_registration @@ -61,6 +71,17 @@ def get_context_data(self, **kwargs): }, **kwargs) -class DraftRegisrationModifyStorageUsage(DraftRegistrationMixin, StorageMixin): +class DraftRegisrationModifyStorageUsage(DraftRegistrationMixin, DetailView): template_name = 'draft_registrations/detail.html' permission_required = 'osf.change_draftregistration' + + def post(self, request, *args, **kwargs): + draft = self.get_object() + new_cap = request.POST.get('cap-input') + + draft_cap = draft.custom_storage_usage_limit or settings.STORAGE_LIMIT_PRIVATE + if float(new_cap) != draft_cap: + draft.custom_storage_usage_limit = new_cap + + draft.save() + return redirect(self.get_success_url()) diff --git a/admin/nodes/queries.py b/admin/nodes/queries.py deleted file mode 100644 index 9cb0b09ff2c..00000000000 --- a/admin/nodes/queries.py +++ /dev/null @@ -1,29 +0,0 @@ -from django.db.models import F, Case, When, IntegerField - -from website import settings - - -STORAGE_USAGE_QUERY = { - 'public_cap': Case( - When( - custom_storage_usage_limit_public=None, - then=settings.STORAGE_LIMIT_PUBLIC, - ), - When( - custom_storage_usage_limit_public__gt=0, - then=F('custom_storage_usage_limit_public'), - ), - output_field=IntegerField() - ), - 'private_cap': Case( - When( - custom_storage_usage_limit_private=None, - then=settings.STORAGE_LIMIT_PRIVATE, - ), - When( - custom_storage_usage_limit_private__gt=0, - then=F('custom_storage_usage_limit_private'), - ), - output_field=IntegerField() - ) -} diff --git a/admin/nodes/views.py b/admin/nodes/views.py index bd7b1633038..40cf261945d 100644 --- a/admin/nodes/views.py +++ b/admin/nodes/views.py @@ -6,7 +6,7 @@ from django.utils import timezone from django.core.exceptions import PermissionDenied, ValidationError from django.urls import NoReverseMatch -from django.db.models import F +from django.db.models import F, Case, When, IntegerField from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin from django.http import HttpResponse @@ -23,7 +23,6 @@ from admin.base.views import GuidView from admin.base.forms import GuidForm from admin.notifications.views import detect_duplicate_notifications, delete_selected_notifications -from admin.nodes.queries import STORAGE_USAGE_QUERY from api.share.utils import update_share from api.caching.tasks import update_storage_usage_cache @@ -63,33 +62,34 @@ def get_object(self): guids___id=self.kwargs['guid'] ).annotate( guid=F('guids___id'), - **STORAGE_USAGE_QUERY + public_cap=Case( + When( + custom_storage_usage_limit_public=None, + then=settings.STORAGE_LIMIT_PUBLIC, + ), + When( + custom_storage_usage_limit_public__gt=0, + then=F('custom_storage_usage_limit_public'), + ), + output_field=IntegerField() + ), + private_cap=Case( + When( + custom_storage_usage_limit_private=None, + then=settings.STORAGE_LIMIT_PRIVATE, + ), + When( + custom_storage_usage_limit_private__gt=0, + then=F('custom_storage_usage_limit_private'), + ), + output_field=IntegerField() + ) ).get() def get_success_url(self): return reverse('nodes:node', kwargs={'guid': self.kwargs['guid']}) -class StorageMixin(View): - - def post(self, request, *args, **kwargs): - object = self.get_object() - new_private_cap = request.POST.get('private-cap-input') - new_public_cap = request.POST.get('public-cap-input') - - object_private_cap = object.custom_storage_usage_limit_private or settings.STORAGE_LIMIT_PRIVATE - object_public_cap = object.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC - - if float(new_private_cap) != object_private_cap: - object.custom_storage_usage_limit_private = new_private_cap - - if float(new_public_cap) != object_public_cap: - object.custom_storage_usage_limit_public = new_public_cap - - object.save() - return redirect(self.get_success_url()) - - class NodeView(NodeMixin, GuidView): """ Allows authorized users to view node info. """ @@ -624,11 +624,28 @@ def post(self, request, *args, **kwargs): return redirect(self.get_success_url()) -class NodeModifyStorageUsage(NodeMixin, StorageMixin): +class NodeModifyStorageUsage(NodeMixin, View): """ Allows an authorized user to view a node's storage usage info and set their public/private storage cap. """ permission_required = 'osf.change_node' + def post(self, request, *args, **kwargs): + node = self.get_object() + new_private_cap = request.POST.get('private-cap-input') + new_public_cap = request.POST.get('public-cap-input') + + node_private_cap = node.custom_storage_usage_limit_private or settings.STORAGE_LIMIT_PRIVATE + node_public_cap = node.custom_storage_usage_limit_public or settings.STORAGE_LIMIT_PUBLIC + + if float(new_private_cap) != node_private_cap: + node.custom_storage_usage_limit_private = new_private_cap + + if float(new_public_cap) != node_public_cap: + node.custom_storage_usage_limit_public = new_public_cap + + node.save() + return redirect(self.get_success_url()) + class NodeRecalculateStorage(NodeMixin, View): """ Allows an authorized user to manually set a node's storage cache by recalculating the value. diff --git a/admin/templates/draft_registrations/storage_usage.html b/admin/templates/draft_registrations/storage_usage.html index 3920f82b4e9..47c944f6d6a 100644 --- a/admin/templates/draft_registrations/storage_usage.html +++ b/admin/templates/draft_registrations/storage_usage.html @@ -3,8 +3,7 @@ Draft registration storage usage - Public cap: {{ draft_registration.public_cap|floatformat:0 }} GB
- Private cap: {{ draft_registration.private_cap|floatformat:0 }} GB
+ Public and private cap: {{ draft_registration.cap|floatformat:0 }} GB
@@ -19,8 +18,7 @@

Adjusting the storage caps for {{ draft_registration.guid }}

- Public cap: GB
- Private cap: GB + Public and Private cap: GB