Merge pull request #4818 from zeeshanakram3/qn_add_storage_bag_size_f…

…ield

[Query Node] Adds `StorageBag.objectsSize` field

distributor-node/src/services/networking/query-node/schema.graphql

@@ -1047,6 +1047,9 @@ type StorageBag implements BaseGraphQLObject {
   deletedById: ID
   version: Int!
   objects: [StorageDataObject!]!
+
+  """Total size of data objects in Bag"""
+  objectsSize: BigInt!
   storageBuckets: [StorageBucket!]!
   distributionBuckets: [DistributionBucket!]!
 
@@ -12476,6 +12479,12 @@ input StorageBagWhereInput {
   deletedAt_gte: DateTime
   deletedById_eq: ID
   deletedById_in: [ID!]
+  objectsSize_eq: BigInt
+  objectsSize_gt: BigInt
+  objectsSize_gte: BigInt
+  objectsSize_lt: BigInt
+  objectsSize_lte: BigInt
+  objectsSize_in: [BigInt!]
   owner_json: JSONObject
   objects_none: StorageDataObjectWhereInput
   objects_some: StorageDataObjectWhereInput
@@ -12496,10 +12505,12 @@ input StorageBagWhereUniqueInput {
 }
 
 input StorageBagCreateInput {
+  objectsSize: String!
   owner: JSONObject!
 }
 
 input StorageBagUpdateInput {
+  objectsSize: String
   owner: JSONObject
 }
 
@@ -31872,6 +31883,8 @@ enum StorageBagOrderByInput {
   updatedAt_DESC
   deletedAt_ASC
   deletedAt_DESC
+  objectsSize_ASC
+  objectsSize_DESC
 }
 
 enum StorageBucketOperatorMetadataOrderByInput {

query-node/CHANGELOG.md

@@ -1,3 +1,7 @@
+### 1.4.0
+
+- Adds `StorageBag.objectsSize` field to `StorageBag` entity schema. This enables to query the total size of all objects in a storage bag.
+
 ### 1.3.0
 
 - Fix external resources mapping of membership metadata to ignore unrecognized type.

query-node/mappings/src/storage/index.ts

@@ -2,77 +2,77 @@
 eslint-disable @typescript-eslint/naming-convention
 */
 import { DatabaseManager, EventContext, StoreContext } from '@joystream/hydra-common'
+import BN from 'bn.js'
+import {
+  DistributionBucket,
+  DistributionBucketFamily,
+  DistributionBucketOperator,
+  DistributionBucketOperatorMetadata,
+  DistributionBucketOperatorStatus,
+  GeoCoordinates,
+  NodeLocationMetadata,
+  StorageBag,
+  StorageBucket,
+  StorageBucketOperatorStatusActive,
+  StorageBucketOperatorStatusInvited,
+  StorageBucketOperatorStatusMissing,
+  StorageDataObject,
+} from 'query-node/dist/model'
 import {
   Storage_DataObjectsDeletedEvent_V1001 as DataObjectsDeletedEvent_V1001,
   Storage_DataObjectsMovedEvent_V1001 as DataObjectsMovedEvent_V1001,
   Storage_DataObjectsUpdatedEvent_V1001 as DataObjectsUpdatedEvent_V1001,
   Storage_DataObjectsUploadedEvent_V1001 as DataObjectsUploadedEvent_V1001,
   Storage_DistributionBucketCreatedEvent_V1001 as DistributionBucketCreatedEvent_V1001,
   Storage_DistributionBucketDeletedEvent_V1001 as DistributionBucketDeletedEvent_V1001,
-  Storage_DistributionBucketsUpdatedForBagEvent_V1001 as DistributionBucketsUpdatedForBagEvent_V1001,
   Storage_DistributionBucketFamilyCreatedEvent_V1001 as DistributionBucketFamilyCreatedEvent_V1001,
   Storage_DistributionBucketFamilyDeletedEvent_V1001 as DistributionBucketFamilyDeletedEvent_V1001,
   Storage_DistributionBucketFamilyMetadataSetEvent_V1001 as DistributionBucketFamilyMetadataSetEvent_V1001,
-  Storage_DistributionBucketMetadataSetEvent_V1001 as DistributionBucketMetadataSetEvent_V1001,
   Storage_DistributionBucketInvitationAcceptedEvent_V1001 as DistributionBucketInvitationAcceptedEvent_V1001,
   Storage_DistributionBucketInvitationCancelledEvent_V1001 as DistributionBucketInvitationCancelledEvent_V1001,
-  Storage_StorageOperatorMetadataSetEvent_V1001 as StorageOperatorMetadataSetEvent_V1001,
-  Storage_StorageBucketCreatedEvent_V1001 as StorageBucketCreatedEvent_V1001,
-  Storage_StorageBucketDeletedEvent_V1001 as StorageBucketDeletedEvent_V1001,
-  Storage_StorageBucketInvitationAcceptedEvent_V1001 as StorageBucketInvitationAcceptedEvent_V1001,
-  Storage_StorageBucketInvitationCancelledEvent_V1001 as StorageBucketInvitationCancelledEvent_V1001,
+  Storage_DistributionBucketMetadataSetEvent_V1001 as DistributionBucketMetadataSetEvent_V1001,
   Storage_DistributionBucketModeUpdatedEvent_V1001 as DistributionBucketModeUpdatedEvent_V1001,
-  Storage_DynamicBagCreatedEvent_V1001 as DynamicBagCreatedEvent_V1001,
-  Storage_DynamicBagDeletedEvent_V1001 as DynamicBagDeletedEvent_V1001,
-  Storage_VoucherChangedEvent_V1001 as VoucherChangedEvent_V1001,
   Storage_DistributionBucketOperatorInvitedEvent_V1001 as DistributionBucketOperatorInvitedEvent_V1001,
   Storage_DistributionBucketOperatorRemovedEvent_V1001 as DistributionBucketOperatorRemovedEvent_V1001,
   Storage_DistributionBucketStatusUpdatedEvent_V1001 as DistributionBucketStatusUpdatedEvent_V1001,
+  Storage_DistributionBucketsUpdatedForBagEvent_V1001 as DistributionBucketsUpdatedForBagEvent_V1001,
+  Storage_DynamicBagCreatedEvent_V1001 as DynamicBagCreatedEvent_V1001,
+  Storage_DynamicBagDeletedEvent_V1001 as DynamicBagDeletedEvent_V1001,
   Storage_PendingDataObjectsAcceptedEvent_V1001 as PendingDataObjectsAcceptedEvent_V1001,
+  Storage_StorageBucketCreatedEvent_V1001 as StorageBucketCreatedEvent_V1001,
+  Storage_StorageBucketDeletedEvent_V1001 as StorageBucketDeletedEvent_V1001,
+  Storage_StorageBucketInvitationAcceptedEvent_V1001 as StorageBucketInvitationAcceptedEvent_V1001,
+  Storage_StorageBucketInvitationCancelledEvent_V1001 as StorageBucketInvitationCancelledEvent_V1001,
   Storage_StorageBucketOperatorInvitedEvent_V1001 as StorageBucketOperatorInvitedEvent_V1001,
   Storage_StorageBucketStatusUpdatedEvent_V1001 as StorageBucketStatusUpdatedEvent_V1001,
   Storage_StorageBucketVoucherLimitsSetEvent_V1001 as StorageBucketVoucherLimitsSetEvent_V1001,
   Storage_StorageBucketsUpdatedForBagEvent_V1001 as StorageBucketsUpdatedForBagEvent_V1001,
+  Storage_StorageOperatorMetadataSetEvent_V1001 as StorageOperatorMetadataSetEvent_V1001,
+  Storage_VoucherChangedEvent_V1001 as VoucherChangedEvent_V1001,
 } from '../../generated/types'
-import {
-  DistributionBucket,
-  DistributionBucketFamily,
-  DistributionBucketOperator,
-  DistributionBucketOperatorMetadata,
-  DistributionBucketOperatorStatus,
-  NodeLocationMetadata,
-  StorageBag,
-  StorageBucket,
-  StorageBucketOperatorStatusActive,
-  StorageBucketOperatorStatusInvited,
-  StorageBucketOperatorStatusMissing,
-  StorageDataObject,
-  GeoCoordinates,
-} from 'query-node/dist/model'
-import BN from 'bn.js'
 import { getById, inconsistentState } from '../common'
-import { videoRelationsForCounters, unsetAssetRelations } from '../content/utils'
+import { unsetAssetRelations, videoRelationsForCounters } from '../content/utils'
+import { getAllManagers } from '../derivedPropertiesManager/applications'
 import {
   processDistributionBucketFamilyMetadata,
   processDistributionOperatorMetadata,
   processStorageOperatorMetadata,
 } from './metadata'
 import {
   createDataObjects,
-  getStorageBucketWithOperatorMetadata,
+  deleteDataObjects,
+  distributionBucketId,
+  distributionBucketIdByFamilyAndIndex,
+  distributionOperatorId,
   getBag,
-  getDynamicBagId,
-  getDynamicBagOwner,
   getDataObjectsInBag,
-  getDynamicBag,
   getDistributionBucketFamilyWithMetadata,
   getDistributionBucketOperatorWithMetadata,
-  distributionBucketId,
-  distributionOperatorId,
-  distributionBucketIdByFamilyAndIndex,
-  deleteDataObjects,
+  getDynamicBag,
+  getDynamicBagId,
+  getDynamicBagOwner,
+  getStorageBucketWithOperatorMetadata,
 } from './utils'
-import { getAllManagers } from '../derivedPropertiesManager/applications'
 
 // STORAGE BUCKETS
 
@@ -233,6 +233,7 @@ export async function storage_DynamicBagCreated({ event, store }: EventContext &
   const storageBag = new StorageBag({
     id: getDynamicBagId(bagId),
     owner: getDynamicBagOwner(bagId),
+    objectsSize: new BN(0),
     storageBuckets: Array.from(storageBuckets).map((id) => new StorageBucket({ id: id.toString() })),
     distributionBuckets: Array.from(distributionBuckets).map(
       (id) => new DistributionBucket({ id: distributionBucketId(id) })
@@ -364,13 +365,21 @@ export async function storage_PendingDataObjectsAccepted({ event, store }: Event
 export async function storage_DataObjectsMoved({ event, store }: EventContext & StoreContext): Promise<void> {
   const [srcBagId, destBagId, dataObjectIds] = new DataObjectsMovedEvent_V1001(event).params
   const dataObjects = await getDataObjectsInBag(store, srcBagId, dataObjectIds)
+  const srcBag = await getBag(store, srcBagId)
   const destBag = await getBag(store, destBagId)
   await Promise.all(
     dataObjects.map(async (dataObject) => {
       dataObject.storageBag = destBag
       await store.save<StorageDataObject>(dataObject)
     })
   )
+
+  // Update source & destination bags size
+  const movedDataObjectsSize = dataObjects.reduce((acc, dataObject) => acc.add(dataObject.size), new BN(0))
+  srcBag.objectsSize = srcBag.objectsSize.sub(movedDataObjectsSize)
+  destBag.objectsSize = destBag.objectsSize.add(movedDataObjectsSize)
+  await store.save<StorageBag>(srcBag)
+  await store.save<StorageBag>(destBag)
 }
 
 export async function storage_DataObjectsDeleted({ event, store }: EventContext & StoreContext): Promise<void> {

query-node/mappings/src/storage/utils.ts

@@ -1,39 +1,39 @@
 import { DatabaseManager } from '@joystream/hydra-common'
 import {
   PalletStorageBagIdType as BagId,
+  PalletStorageDataObjectCreationParameters as DataObjectCreationParameters,
+  PalletStorageDistributionBucketIdRecord as DistributionBucketId,
   PalletStorageDynamicBagIdType as DynamicBagId,
   PalletStorageStaticBagId as StaticBagId,
-  PalletStorageDistributionBucketIdRecord as DistributionBucketId,
-  PalletStorageDataObjectCreationParameters as DataObjectCreationParameters,
 } from '@polkadot/types/lookup'
+import BN from 'bn.js'
 import {
   DataObjectTypeUnknown,
+  DistributionBucketFamily,
+  DistributionBucketOperator,
   StorageBag,
-  StorageDataObject,
   StorageBagOwner,
   StorageBagOwnerChannel,
   StorageBagOwnerCouncil,
   StorageBagOwnerMember,
   StorageBagOwnerWorkingGroup,
   StorageBucket,
-  DistributionBucketOperator,
-  DistributionBucketFamily,
+  StorageDataObject,
 } from 'query-node/dist/model'
-import BN from 'bn.js'
-import { bytesToString, inconsistentState, getById, RelationsArr } from '../common'
 import { In } from 'typeorm'
+import { RelationsArr, bytesToString, getById, inconsistentState } from '../common'
 
-import { BTreeSet } from '@polkadot/types'
-import _ from 'lodash'
 import {
-  WorkerId,
   DataObjectId,
   DistributionBucketFamilyId,
   DistributionBucketIndex,
+  WorkerId,
 } from '@joystream/types/primitives'
+import { BTreeSet } from '@polkadot/types'
 import { Balance } from '@polkadot/types/interfaces'
+import _ from 'lodash'
+import { unsetAssetRelations, videoRelationsForCounters } from '../content/utils'
 import { getAllManagers } from '../derivedPropertiesManager/applications'
-import { videoRelationsForCounters, unsetAssetRelations } from '../content/utils'
 
 export type StorageDataObjectParams = {
   storageBagOrId: StorageBag | BagId
@@ -155,6 +155,7 @@ export async function getStaticBag(
     console.log(`Creating new static bag: ${id}`)
     const newBag = new StorageBag({
       id,
+      objectsSize: new BN(0),
       owner: getStaticBagOwner(bagId),
     })
     await store.save<StorageBag>(newBag)
@@ -234,6 +235,11 @@ export async function createDataObjects(
 
   await Promise.all(dataObjects.map((o) => store.save<StorageDataObject>(o)))
 
+  // Update bag size
+  const createdDataObjectsSize = dataObjects.reduce((acc, dataObject) => acc.add(dataObject.size), new BN(0))
+  storageBag.objectsSize = storageBag.objectsSize.add(createdDataObjectsSize)
+  await store.save<StorageBag>(storageBag)
+
   return dataObjects
 }
 
@@ -242,6 +248,7 @@ export async function deleteDataObjects(
   bagId: BagId,
   dataObjectIds: BTreeSet<DataObjectId>
 ): Promise<void> {
+  const storageBag = await getBag(store, bagId)
   const dataObjects = await getDataObjectsInBag(store, bagId, dataObjectIds, [
     'videoThumbnail',
     ...videoRelationsForCounters.map((item) => `videoThumbnail.${item}`),
@@ -255,6 +262,11 @@ export async function deleteDataObjects(
 
     await unsetAssetRelations(store, dataObject)
   }
+
+  // Update bag size
+  const deletedDataObjectsSize = dataObjects.reduce((acc, dataObject) => acc.add(dataObject.size), new BN(0))
+  storageBag.objectsSize = storageBag.objectsSize.sub(deletedDataObjectsSize)
+  await store.save<StorageBag>(storageBag)
 }
 
 export function distributionBucketId(runtimeBucketId: DistributionBucketId): string {

query-node/schemas/storage.graphql

@@ -133,6 +133,9 @@ type StorageBag @entity {
   "Data objects in the bag"
   objects: [StorageDataObject!] @derivedFrom(field: "storageBag")
 
+  "Total size of data objects in Bag"
+  objectsSize: BigInt!
+
   "Storage buckets assigned to the bag"
   storageBuckets: [StorageBucket!]
 

storage-node/src/services/queryNode/schema.graphql

@@ -1047,6 +1047,9 @@ type StorageBag implements BaseGraphQLObject {
   deletedById: ID
   version: Int!
   objects: [StorageDataObject!]!
+
+  """Total size of data objects in Bag"""
+  objectsSize: BigInt!
   storageBuckets: [StorageBucket!]!
   distributionBuckets: [DistributionBucket!]!
 
@@ -12476,6 +12479,12 @@ input StorageBagWhereInput {
   deletedAt_gte: DateTime
   deletedById_eq: ID
   deletedById_in: [ID!]
+  objectsSize_eq: BigInt
+  objectsSize_gt: BigInt
+  objectsSize_gte: BigInt
+  objectsSize_lt: BigInt
+  objectsSize_lte: BigInt
+  objectsSize_in: [BigInt!]
   owner_json: JSONObject
   objects_none: StorageDataObjectWhereInput
   objects_some: StorageDataObjectWhereInput
@@ -12496,10 +12505,12 @@ input StorageBagWhereUniqueInput {
 }
 
 input StorageBagCreateInput {
+  objectsSize: String!
   owner: JSONObject!
 }
 
 input StorageBagUpdateInput {
+  objectsSize: String
   owner: JSONObject
 }
 
@@ -31872,6 +31883,8 @@ enum StorageBagOrderByInput {
   updatedAt_DESC
   deletedAt_ASC
   deletedAt_DESC
+  objectsSize_ASC
+  objectsSize_DESC
 }
 
 enum StorageBucketOperatorMetadataOrderByInput {