From ed592ea3f7b07bda47953c372aa5cc0cd5a5c2ba Mon Sep 17 00:00:00 2001 From: Matej Holicky <10matejholicky@gmail.com> Date: Thu, 6 Apr 2023 13:14:14 +0200 Subject: [PATCH 1/4] feat: Add resources on lastEvent query for rmrk2 --- src/server-extension/model/event.model.ts | 38 +++++++- src/server-extension/query/event.ts | 109 +++++++++++++--------- src/server-extension/resolvers/event.ts | 52 +++++++---- 3 files changed, 130 insertions(+), 69 deletions(-) diff --git a/src/server-extension/model/event.model.ts b/src/server-extension/model/event.model.ts index 861f4ae8..1c3ffef8 100644 --- a/src/server-extension/model/event.model.ts +++ b/src/server-extension/model/event.model.ts @@ -1,4 +1,4 @@ -import { Field, ObjectType } from 'type-graphql'; +import { Field, Int, ObjectType } from "type-graphql"; @ObjectType() export class EventEntity { @@ -48,22 +48,50 @@ export class LastEventEntity { @Field(() => String, { nullable: false }) value!: String; - @Field(() => String, { nullable: false, name: 'currentOwner' }) + @Field(() => String, { nullable: false, name: "currentOwner" }) current_owner!: String; @Field(() => String, { nullable: true }) image!: String; - @Field(() => String, { nullable: true, name: 'animationUrl' }) + @Field(() => String, { nullable: true, name: "animationUrl" }) animation_url!: string | undefined | null; - @Field(() => String, { nullable: false, name: 'collectionId' }) + @Field(() => String, { nullable: false, name: "collectionId" }) collection_id!: string; - @Field(() => String, { nullable: false, name: 'collectionName' }) + @Field(() => String, { nullable: false, name: "collectionName" }) collection_name!: string; constructor(props: Partial) { Object.assign(this, props); } } + +@ObjectType() +export class Resource { + @Field(() => String, { nullable: false }) + id!: string; + + @Field(() => String, { nullable: true }) + src!: string; + + @Field(() => String, { nullable: true }) + metadata!: string; + + @Field(() => String, { nullable: true }) + slot!: string; + + @Field(() => String, { nullable: true }) + thumb!: string; + + @Field(() => Int, { nullable: false }) + priority!: number; + + @Field(() => Boolean, { nullable: false }) + pending!: boolean; + + constructor(props: Partial) { + Object.assign(this, props); + } +} diff --git a/src/server-extension/query/event.ts b/src/server-extension/query/event.ts index e10d39f2..11adaf32 100644 --- a/src/server-extension/query/event.ts +++ b/src/server-extension/query/event.ts @@ -1,47 +1,68 @@ -export const buyEvent = `SELECT - COUNT(e.*) as count, - COALESCE(MAX(e.meta::decimal), 0) as max - FROM event e - LEFT JOIN nft_entity ne on ne.id = e.nft_id - WHERE e.interaction = 'BUY' AND ne.collection_id = $1;`; +export const buyEvent = ` + SELECT COUNT(e.*) AS COUNT, + COALESCE(MAX(e.meta::decimal), 0) AS MAX + FROM event e + LEFT JOIN nft_entity ne ON ne.id = e.nft_id + WHERE e.interaction = 'BUY' + AND ne.collection_id = $1; +`; -export const collectionEventHistory = ( - idList: string, - dateRange: string -) => `SELECT - ce.id as id, - DATE(e.timestamp), - count(e) -FROM nft_entity ne -JOIN collection_entity ce on ce.id = ne.collection_id -JOIN event e on e.nft_id = ne.id -WHERE e.interaction = 'BUY' -and ce.id in (${idList}) -${dateRange} -GROUP BY ce.id, DATE(e.timestamp) -ORDER BY DATE(e.timestamp)`; +export const collectionEventHistory = (idList: string, dateRange: string) => ` + SELECT ce.id AS id, + DATE(e.timestamp), + count(e) + FROM nft_entity ne + JOIN collection_entity ce ON ce.id = ne.collection_id + JOIN event e ON e.nft_id = ne.id + WHERE e.interaction = 'BUY' + AND ce.id in (${idList}) + ${dateRange} + GROUP BY ce.id, + DATE(e.timestamp) + ORDER BY DATE(e.timestamp) +`; -export const lastEventQuery = (whereCondition: string) => `SELECT - DISTINCT ne.id as id, - ne.name as name, - ne.issuer as issuer, - ne.metadata as metadata, - e.current_owner, - me.image as image, - me.animation_url, - MAX(e.timestamp) as timestamp, - MAX(e.meta::decimal) as value, - ne.collection_id as collection_id, - ce.name as collection_name - -FROM event e - JOIN nft_entity ne on e.nft_id = ne.id - LEFT join metadata_entity me on me.id = ne.metadata - LEFT JOIN collection_entity ce on ne.collection_id = ce.id -WHERE - e.interaction = $1 - AND ne.burned = false +export const lastEventQuery = (whereCondition: string) => ` + SELECT DISTINCT ne.id AS id, + ne.name AS name, + ne.issuer AS issuer, + ne.metadata AS metadata, + e.current_owner, + me.image AS image, + me.animation_url, + MAX(e.timestamp) AS timestamp, + MAX(e.meta::decimal) AS value, + ne.collection_id AS collection_id, + ce.name AS collection_name + FROM event e + JOIN nft_entity ne ON e.nft_id = ne.id + LEFT JOIN metadata_entity me ON me.id = ne.metadata + LEFT JOIN collection_entity ce ON ne.collection_id = ce.id + WHERE e.interaction = $1 + AND ne.burned = FALSE ${whereCondition} -GROUP BY ne.id, me.id, e.current_owner, me.image, ce.name -ORDER BY MAX(e.timestamp) DESC -LIMIT $2 OFFSET $3`; + GROUP BY ne.id, + me.id, + e.current_owner, + me.image, + ce.name + ORDER BY MAX(e.timestamp) DESC + LIMIT $2 + OFFSET $3 +`; + +export const resourcesByNFT = (nftId: string) => ` + SELECT r.id as id, + r.src as src, + r.metadata as metadata, + r.slot as slot, + r.thumb as thumb, + r.priority as priority, + r.pending as pending, + r.meta_id as meta_id, + r.nft_id as nft_id + FROM resource r + WHERE r.nft_id = '${nftId}' + LIMIT $1 + OFFSET $2 + `; diff --git a/src/server-extension/resolvers/event.ts b/src/server-extension/resolvers/event.ts index f48bbc1a..8d3b597d 100644 --- a/src/server-extension/resolvers/event.ts +++ b/src/server-extension/resolvers/event.ts @@ -1,34 +1,46 @@ -import { Arg, Query, Resolver } from 'type-graphql' -import type { EntityManager } from 'typeorm' -import { NFTEntity } from '../../model/generated' -import { LastEventEntity } from '../model/event.model' -import { lastEventQuery } from '../query/event' +import { Arg, FieldResolver, Query, Resolver, Root } from "type-graphql"; +import type { EntityManager } from "typeorm"; +import { NFTEntity } from "../../model/generated"; +import { LastEventEntity, Resource } from "../model/event.model"; +import { lastEventQuery, resourcesByNFT } from "../query/event"; import { makeQuery, toSqlInParams } from "../utils"; -import { Interaction } from '../../model' +import { Interaction } from "../../model"; import { passionQuery } from "../query/nft"; -import { PassionFeedEntity } from '../model/passion.model' +import { PassionFeedEntity } from "../model/passion.model"; -@Resolver() +@Resolver((of) => LastEventEntity) export class EventResolver { constructor(private tx: () => Promise) {} @Query(() => [LastEventEntity]) async lastEvent( - @Arg('interaction', { nullable: true, defaultValue: Interaction.LIST }) interaction: Interaction, - @Arg('passionAccount', { nullable: true, }) account: string, - @Arg('limit', { nullable: true, defaultValue: 20 }) limit: number, - @Arg('offset', { nullable: true, defaultValue: 0 }) offset: number, + @Arg("interaction", { nullable: true, defaultValue: Interaction.LIST }) interaction: Interaction, + @Arg("passionAccount", { nullable: true }) account: string, + @Arg("limit", { nullable: true, defaultValue: 20 }) limit: number, + @Arg("offset", { nullable: true, defaultValue: 0 }) offset: number ): Promise<[LastEventEntity]> { + const passionResult: [PassionFeedEntity] = await makeQuery(this.tx, NFTEntity, passionQuery, [account]); + const passionList = passionResult.map((passion) => passion.id); + const selectFromPassionList = + passionList && passionList.length > 0 ? `AND ne.issuer in (${toSqlInParams(passionList)})` : ""; + const result: [LastEventEntity] = await makeQuery(this.tx, NFTEntity, lastEventQuery(selectFromPassionList), [ + interaction, + limit, + offset, + ]); - const passionResult: [PassionFeedEntity] = await makeQuery(this.tx, NFTEntity, passionQuery, [account]) - const passionList = passionResult.map(passion => passion.id) - - const selectFromPassionList = passionList && passionList.length > 0 - ? `AND ne.issuer in (${toSqlInParams(passionList)})` - : '' - const result: [LastEventEntity] = await makeQuery(this.tx, NFTEntity, lastEventQuery(selectFromPassionList), [interaction, limit, offset]) - return result + return result; } + @FieldResolver(() => [Resource]) + async resources( + @Root() event: LastEventEntity, + @Arg("limit", { nullable: true, defaultValue: 20 }) limit: number, + @Arg("offset", { nullable: true, defaultValue: 0 }) offset: number + ) { + const result: [Resource] = await makeQuery(this.tx, NFTEntity, resourcesByNFT(String(event.id)), [limit, offset]); + + return result; + } } From 4da6bea08d39189325df4d519d15fb63af8e6fd0 Mon Sep 17 00:00:00 2001 From: Matej Holicky <10matejholicky@gmail.com> Date: Fri, 7 Apr 2023 00:30:47 +0200 Subject: [PATCH 2/4] feat: Fixed nft field on Resource --- src/server-extension/model/event.model.ts | 4 +-- src/server-extension/query/event.ts | 32 ++++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/server-extension/model/event.model.ts b/src/server-extension/model/event.model.ts index 1c3ffef8..5cf43e36 100644 --- a/src/server-extension/model/event.model.ts +++ b/src/server-extension/model/event.model.ts @@ -68,7 +68,7 @@ export class LastEventEntity { } } -@ObjectType() +@ObjectType({ description: "only supports 1 level of nested attributes" }) export class Resource { @Field(() => String, { nullable: false }) id!: string; @@ -91,7 +91,7 @@ export class Resource { @Field(() => Boolean, { nullable: false }) pending!: boolean; - constructor(props: Partial) { + constructor(props: Partial) { Object.assign(this, props); } } diff --git a/src/server-extension/query/event.ts b/src/server-extension/query/event.ts index 11adaf32..3095aead 100644 --- a/src/server-extension/query/event.ts +++ b/src/server-extension/query/event.ts @@ -59,10 +59,36 @@ export const resourcesByNFT = (nftId: string) => ` r.thumb as thumb, r.priority as priority, r.pending as pending, - r.meta_id as meta_id, - r.nft_id as nft_id + json_build_object( + 'block_number', ne.block_number, + 'burned', ne.burned, + 'collection_id', ne.collection_id, + 'created_at', ne.created_at, + 'current_owner', ne.current_owner, + 'emote_count', ne.emote_count, + 'hash', ne.hash, + 'id', ne.id, + 'image', ne.image, + 'instance', ne.instance, + 'issuer', ne.issuer, + 'media', ne.media, + 'meta_id', ne.meta_id, + 'metadata', ne.metadata, + 'name', ne.name, + 'parent_id', ne.parent_id, + 'pending', ne.pending, + 'price', ne.price, + 'recipient', ne.recipient, + 'royalty', ne.royalty, + 'sn', ne.sn, + 'transferable', ne.transferable, + 'updated_at', ne.updated_at, + 'version', ne.version + ) as nft FROM resource r - WHERE r.nft_id = '${nftId}' + JOIN nft_entity ne ON ne.id = r.nft_id + WHERE ne.id = '${nftId}' + GROUP BY r.id, ne.id LIMIT $1 OFFSET $2 `; From 93f683864220c73ecb664242a52013d41636ec26 Mon Sep 17 00:00:00 2001 From: Matej Holicky <10matejholicky@gmail.com> Date: Fri, 7 Apr 2023 13:25:13 +0200 Subject: [PATCH 3/4] feat: Fixed meta field on Resource --- .gitignore | 1 + src/server-extension/query/event.ts | 71 ++++++++++++++++++----------- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index b2381b72..c394145a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/.vscode /.idea /node_modules /lib diff --git a/src/server-extension/query/event.ts b/src/server-extension/query/event.ts index 3095aead..fd5c31d2 100644 --- a/src/server-extension/query/event.ts +++ b/src/server-extension/query/event.ts @@ -59,36 +59,53 @@ export const resourcesByNFT = (nftId: string) => ` r.thumb as thumb, r.priority as priority, r.pending as pending, - json_build_object( - 'block_number', ne.block_number, - 'burned', ne.burned, - 'collection_id', ne.collection_id, - 'created_at', ne.created_at, - 'current_owner', ne.current_owner, - 'emote_count', ne.emote_count, - 'hash', ne.hash, - 'id', ne.id, - 'image', ne.image, - 'instance', ne.instance, - 'issuer', ne.issuer, - 'media', ne.media, - 'meta_id', ne.meta_id, - 'metadata', ne.metadata, - 'name', ne.name, - 'parent_id', ne.parent_id, - 'pending', ne.pending, - 'price', ne.price, - 'recipient', ne.recipient, - 'royalty', ne.royalty, - 'sn', ne.sn, - 'transferable', ne.transferable, - 'updated_at', ne.updated_at, - 'version', ne.version + CASE + WHEN me.id IS NULL THEN NULL + ELSE json_strip_nulls( + json_build_object( + 'attributes', me.attributes, + 'name', me.name, + 'description', me.description, + 'id', me.id, + 'animation_url', me.animation_url, + 'type', me.type, + 'image', me.image + ) + ) + END AS meta, + json_strip_nulls( + json_build_object( + 'block_number', ne.block_number, + 'burned', ne.burned, + 'collection_id', ne.collection_id, + 'created_at', ne.created_at, + 'current_owner', ne.current_owner, + 'emote_count', ne.emote_count, + 'hash', ne.hash, + 'id', ne.id, + 'image', ne.image, + 'instance', ne.instance, + 'issuer', ne.issuer, + 'media', ne.media, + 'meta_id', ne.meta_id, + 'metadata', ne.metadata, + 'name', ne.name, + 'parent_id', ne.parent_id, + 'pending', ne.pending, + 'price', ne.price, + 'recipient', ne.recipient, + 'royalty', ne.royalty, + 'sn', ne.sn, + 'transferable', ne.transferable, + 'updated_at', ne.updated_at, + 'version', ne.version + ) ) as nft FROM resource r - JOIN nft_entity ne ON ne.id = r.nft_id + LEFT JOIN nft_entity ne ON ne.id = r.nft_id + LEFT JOIN metadata_entity me ON me.id = r.meta_id WHERE ne.id = '${nftId}' - GROUP BY r.id, ne.id + GROUP BY r.id, ne.id, me.id LIMIT $1 OFFSET $2 `; From 666dc795126b270d65271df454d0509276f5d601 Mon Sep 17 00:00:00 2001 From: Matej Holicky <10matejholicky@gmail.com> Date: Tue, 11 Apr 2023 17:17:54 +0200 Subject: [PATCH 4/4] feat: Fix n+1 issue with resources field --- package-lock.json | 13 ++++---- package.json | 1 + src/server-extension/model/event.model.ts | 7 ++++- src/server-extension/query/event.ts | 9 +++--- src/server-extension/resolvers/event.ts | 38 +++++++++++++++-------- 5 files changed, 43 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index caa08288..ce3cf37c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@subsquid/typeorm-migration": "0.1.6", "@subsquid/typeorm-store": "0.2.2", "dotenv": "^16.0.3", + "lodash": "^4.17.21", "md5": "^2.3.0", "nanoid": "3.3.4", "pg": "^8.10.0", @@ -4151,9 +4152,9 @@ } }, "node_modules/dataloader": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/dataloader/-/dataloader-2.1.0.tgz", - "integrity": "sha512-qTcEYLen3r7ojZNgVUaRggOI+KM7jrKxXeSHhogh/TWxYMeONEMqY+hmkobiYQozsGIyg9OYVzO4ZIfoB4I0pQ==" + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/dataloader/-/dataloader-2.2.2.tgz", + "integrity": "sha512-8YnDaaf7N3k/q5HnTJVuzSyLETjoZjVmHc4AeKAzOvKHEFQKcn64OKBfzHYtE9zGjctNM7V9I0MfnUVLpi7M5g==" }, "node_modules/date-fns": { "version": "2.29.3", @@ -14430,9 +14431,9 @@ "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" }, "dataloader": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/dataloader/-/dataloader-2.1.0.tgz", - "integrity": "sha512-qTcEYLen3r7ojZNgVUaRggOI+KM7jrKxXeSHhogh/TWxYMeONEMqY+hmkobiYQozsGIyg9OYVzO4ZIfoB4I0pQ==" + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/dataloader/-/dataloader-2.2.2.tgz", + "integrity": "sha512-8YnDaaf7N3k/q5HnTJVuzSyLETjoZjVmHc4AeKAzOvKHEFQKcn64OKBfzHYtE9zGjctNM7V9I0MfnUVLpi7M5g==" }, "date-fns": { "version": "2.29.3", diff --git a/package.json b/package.json index 7676a81a..57ff54a2 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "@subsquid/typeorm-migration": "0.1.6", "@subsquid/typeorm-store": "0.2.2", "dotenv": "^16.0.3", + "lodash": "^4.17.21", "md5": "^2.3.0", "nanoid": "3.3.4", "pg": "^8.10.0", diff --git a/src/server-extension/model/event.model.ts b/src/server-extension/model/event.model.ts index 5cf43e36..25ecccbd 100644 --- a/src/server-extension/model/event.model.ts +++ b/src/server-extension/model/event.model.ts @@ -63,12 +63,15 @@ export class LastEventEntity { @Field(() => String, { nullable: false, name: "collectionName" }) collection_name!: string; + @Field(() => [Resource], { nullable: true }) + resources!: Resource[]; + constructor(props: Partial) { Object.assign(this, props); } } -@ObjectType({ description: "only supports 1 level of nested attributes" }) +@ObjectType() export class Resource { @Field(() => String, { nullable: false }) id!: string; @@ -91,6 +94,8 @@ export class Resource { @Field(() => Boolean, { nullable: false }) pending!: boolean; + nftId!: string; + constructor(props: Partial) { Object.assign(this, props); } diff --git a/src/server-extension/query/event.ts b/src/server-extension/query/event.ts index fd5c31d2..fe7ea508 100644 --- a/src/server-extension/query/event.ts +++ b/src/server-extension/query/event.ts @@ -51,7 +51,7 @@ export const lastEventQuery = (whereCondition: string) => ` OFFSET $3 `; -export const resourcesByNFT = (nftId: string) => ` +export const resourcesByNFT = (whereCondition: string) => ` SELECT r.id as id, r.src as src, r.metadata as metadata, @@ -100,12 +100,11 @@ export const resourcesByNFT = (nftId: string) => ` 'updated_at', ne.updated_at, 'version', ne.version ) - ) as nft + ) as nft, + r.nft_id as nft_id FROM resource r LEFT JOIN nft_entity ne ON ne.id = r.nft_id LEFT JOIN metadata_entity me ON me.id = r.meta_id - WHERE ne.id = '${nftId}' + WHERE ${whereCondition} GROUP BY r.id, ne.id, me.id - LIMIT $1 - OFFSET $2 `; diff --git a/src/server-extension/resolvers/event.ts b/src/server-extension/resolvers/event.ts index 8d3b597d..b03ca3c2 100644 --- a/src/server-extension/resolvers/event.ts +++ b/src/server-extension/resolvers/event.ts @@ -1,4 +1,5 @@ -import { Arg, FieldResolver, Query, Resolver, Root } from "type-graphql"; +import { Arg, Info, Query, Resolver } from "type-graphql"; +import { GraphQLResolveInfo } from "graphql"; import type { EntityManager } from "typeorm"; import { NFTEntity } from "../../model/generated"; import { LastEventEntity, Resource } from "../model/event.model"; @@ -7,6 +8,13 @@ import { makeQuery, toSqlInParams } from "../utils"; import { Interaction } from "../../model"; import { passionQuery } from "../query/nft"; import { PassionFeedEntity } from "../model/passion.model"; +import { groupBy } from "lodash"; + +type FieldName = { + name: { + value: string; + }; +}; @Resolver((of) => LastEventEntity) export class EventResolver { @@ -17,30 +25,34 @@ export class EventResolver { @Arg("interaction", { nullable: true, defaultValue: Interaction.LIST }) interaction: Interaction, @Arg("passionAccount", { nullable: true }) account: string, @Arg("limit", { nullable: true, defaultValue: 20 }) limit: number, - @Arg("offset", { nullable: true, defaultValue: 0 }) offset: number + @Arg("offset", { nullable: true, defaultValue: 0 }) offset: number, + @Info() info: GraphQLResolveInfo ): Promise<[LastEventEntity]> { const passionResult: [PassionFeedEntity] = await makeQuery(this.tx, NFTEntity, passionQuery, [account]); const passionList = passionResult.map((passion) => passion.id); const selectFromPassionList = passionList && passionList.length > 0 ? `AND ne.issuer in (${toSqlInParams(passionList)})` : ""; - const result: [LastEventEntity] = await makeQuery(this.tx, NFTEntity, lastEventQuery(selectFromPassionList), [ + let lastEvents: [LastEventEntity] = await makeQuery(this.tx, NFTEntity, lastEventQuery(selectFromPassionList), [ interaction, limit, offset, ]); - return result; - } + // TODO: Refactor this to use proper dataloader with FieldResolver, currently dataloader is not supported + // ref https://github.com/MichalLytek/type-graphql/issues/51 + const isResourcesQueried = Object.values(info.fieldNodes[0]?.selectionSet?.selections as unknown[] as FieldName[]) + .map((i) => i.name.value) + .includes("resources"); - @FieldResolver(() => [Resource]) - async resources( - @Root() event: LastEventEntity, - @Arg("limit", { nullable: true, defaultValue: 20 }) limit: number, - @Arg("offset", { nullable: true, defaultValue: 0 }) offset: number - ) { - const result: [Resource] = await makeQuery(this.tx, NFTEntity, resourcesByNFT(String(event.id)), [limit, offset]); + if (isResourcesQueried) { + const whereCondition = `r.nft_id IN (${toSqlInParams(lastEvents.map((i) => String(i.id)))})`; + const resources: [Resource] = await makeQuery(this.tx, NFTEntity, resourcesByNFT(whereCondition)); + const resourcesById = groupBy(resources, "nft_id"); + + lastEvents.map((event) => (event.resources = resourcesById[String(event.id)] ?? [])); + } - return result; + return lastEvents; } }