diff --git a/.env.default b/.env.default index a592d94a..beb22688 100644 --- a/.env.default +++ b/.env.default @@ -10,23 +10,37 @@ ARCHIVE_GQL_URL=https://archive.dev.uniquenetwork.dev/graphql #ARCHIVE_GQL_URL=https://archive.opal.uniquenetwork.dev/graphql CHAIN_WS_URL=wss://ws-rc.unique.network SCAN_TYPES_BUNDLE=quartz -SCAN_RANGE_FROM_DEFAULT=0 -SCAN_RANGE_FROM=111000 -# SCAN_RANGE_TO=1000000 -SCAN_FORCE_RESCAN=true +#SCAN_RANGE_FROM=111000 +#SCAN_RANGE_TO=1000000 +#SCAN_FORCE_RESCAN=true PROMETHEUS_PORT=3003 +BATCH_SIZE=10 ## Subscquid based subscribers + #ACCOUNTS_SUBSCRIBER_DISABLE=true #BLOCKS_SUBSCRIBER_DISABLE=true #COLLECTIONS_SUBSCRIBER_DISABLE=true #TOKENS_SUBSCRIBER_DISABLE=true -#Environment variables for tests +## Environment variables for tests + TESTS_UNIQUE_WS_ENDPOINT=wss://ws-rc.unique.network TESTS_GRAPHQL_URL='http://localhost:3031/v1/graphql' TESTS_IPFS_URL=https://ipfs.unique.network/ipfs/ +## Sentry variables + SENTRY_DSN=SENTRY_DSN SENTRY_DEBUG=0 SENTRY_LOG_LEVELS='error,warning' + +## Cache variables + +### Cache ttl in seconds. +CACHE_TTL=60 + +### Redis cache connection. If not set, memory cache is in use. +#REDIS_HOST= +#REDIS_PORT= +#REDIS_DB= diff --git a/apps/crawler/src/cache/cache-provider.module.ts b/apps/crawler/src/cache/cache-provider.module.ts new file mode 100644 index 00000000..40a9d142 --- /dev/null +++ b/apps/crawler/src/cache/cache-provider.module.ts @@ -0,0 +1,40 @@ +import { CACHE_MANAGER, Global, Module, Provider } from '@nestjs/common'; +import * as redisStore from 'cache-manager-redis-store'; +import { ConfigService } from '@nestjs/config'; +import { caching, Cache } from 'cache-manager'; +import { CacheConfig, CacheType } from '../config/cache.config'; + +const cacheProvider: Provider = { + inject: [ConfigService], + provide: CACHE_MANAGER, + useFactory: (configService: ConfigService): Cache => { + const cacheConfig: CacheConfig = configService.get('cache'); + + switch (cacheConfig.type) { + case CacheType.DEFAULT: + return caching({ + ttl: cacheConfig.ttl, + + store: 'memory', + }); + case CacheType.REDIS: + return caching({ + ttl: cacheConfig.ttl, + + store: redisStore, + host: cacheConfig.host, + port: cacheConfig.port, + db: cacheConfig.db, + }); + default: + throw new Error('Invalid cache config'); + } + }, +}; + +@Global() +@Module({ + providers: [cacheProvider], + exports: [CACHE_MANAGER], +}) +export class CacheProviderModule {} diff --git a/apps/crawler/src/config/cache.config.ts b/apps/crawler/src/config/cache.config.ts new file mode 100644 index 00000000..a267a9e4 --- /dev/null +++ b/apps/crawler/src/config/cache.config.ts @@ -0,0 +1,42 @@ +export enum CacheType { + DEFAULT = 'Default', + REDIS = 'Redis', +} + +interface CacheConfigBase { + type: CacheType; + ttl: number; +} + +export interface DefaultCacheConfig extends CacheConfigBase { + type: CacheType.DEFAULT; +} + +export interface RedisCacheConfig extends CacheConfigBase { + type: CacheType.REDIS; + host: string; + port: number; + db: number; +} + +export type CacheConfig = DefaultCacheConfig | RedisCacheConfig; + +export function createCacheConfig(env: Record): CacheConfig { + const { CACHE_TTL, REDIS_HOST, REDIS_PORT, REDIS_DB } = env; + const ttl = +CACHE_TTL || 600; + + if (REDIS_HOST) { + return { + type: CacheType.REDIS, + host: REDIS_HOST, + port: +REDIS_PORT || 6379, + db: +REDIS_DB || 0, + ttl, + }; + } + + return { + type: CacheType.DEFAULT, + ttl, + }; +} diff --git a/apps/crawler/src/config/config.module.ts b/apps/crawler/src/config/config.module.ts new file mode 100644 index 00000000..d5c24220 --- /dev/null +++ b/apps/crawler/src/config/config.module.ts @@ -0,0 +1,67 @@ +import * as process from 'process'; +import { ConfigModule } from '@nestjs/config'; +import { CacheConfig, createCacheConfig } from './cache.config'; +import { SentryConfig, createSentryConfig } from './sentry.config'; +import { + SubscribersConfig, + createSubscribersConfig, +} from './subscribers.config'; + +export type Config = { + logLevels: Array; + + chainWsUrl: string; + + archiveGqlUrl: string; + + sentry: SentryConfig; + + cache: CacheConfig; + + subscribers: SubscribersConfig; + + scanTypesBundle: string; + + scanRangeFrom: number; + + scanRangeTo?: number; + + rescan: boolean; + + prometheusPort: number; + + batchSize: number; +}; + +const loadConfig = (): Config => ({ + logLevels: process.env.LOG_LEVELS + ? process.env.LOG_LEVELS.split(',') + : ['log', 'error', 'warn'], + + chainWsUrl: process.env.CHAIN_WS_URL, + + archiveGqlUrl: process.env.ARCHIVE_GQL_URL, + + sentry: createSentryConfig(process.env), + + cache: createCacheConfig(process.env), + + subscribers: createSubscribersConfig(process.env), + + scanTypesBundle: process.env.SCAN_TYPES_BUNDLE || 'quartz', + + scanRangeFrom: +process.env.SCAN_RANGE_FROM || 0, + + scanRangeTo: +process.env.SCAN_RANGE_TO || undefined, + + rescan: process.env.SCAN_FORCE_RESCAN === 'true', + + prometheusPort: +process.env.PROMETHEUS_PORT || 9090, + + batchSize: +process.env.BATCH_SIZE || 10, +}); + +export const GlobalConfigModule = ConfigModule.forRoot({ + isGlobal: true, + load: [loadConfig], +}); diff --git a/apps/crawler/src/processor.config.service.ts b/apps/crawler/src/config/processor.config.service.ts similarity index 59% rename from apps/crawler/src/processor.config.service.ts rename to apps/crawler/src/config/processor.config.service.ts index 8edde6d3..f61148b7 100644 --- a/apps/crawler/src/processor.config.service.ts +++ b/apps/crawler/src/config/processor.config.service.ts @@ -2,28 +2,28 @@ import { Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { DataSource as SubscquidDataSource } from '@subsquid/substrate-processor'; import { Range } from '@subsquid/substrate-processor/lib/util/range'; +import { Config } from './config.module'; @Injectable() export class ProcessorConfigService { - constructor(private configService: ConfigService) {} + constructor(private configService: ConfigService) {} public getDataSource(): SubscquidDataSource { return { - archive: this.configService.get('ARCHIVE_GQL_URL'), - chain: this.configService.get('CHAIN_WS_URL'), + archive: this.configService.get('archiveGqlUrl'), + chain: this.configService.get('chainWsUrl'), }; } public getRange(): Range { - const to = this.configService.get('SCAN_RANGE_TO'); return { - from: this.configService.get('SCAN_RANGE_FROM', 0), - to, + from: this.configService.get('scanRangeFrom'), + to: this.configService.get('scanRangeTo'), }; } public getTypesBundle(): string { - return this.configService.get('SCAN_TYPES_BUNDLE'); + return this.configService.get('scanTypesBundle'); } public getAllParams(): { @@ -38,15 +38,15 @@ export class ProcessorConfigService { }; } - public getForceMode() { - return this.configService.get('SCAN_FORCE_RESCAN'); + public isRescan() { + return this.configService.get('rescan'); } public getPrometheusPort(): number { - return this.configService.get('PROMETHEUS_PORT', 9090); + return this.configService.get('prometheusPort'); } public getBatchSize(): number { - return Number(this.configService.get('BATCH_SIZE', 10)); + return this.configService.get('batchSize'); } } diff --git a/apps/crawler/src/config/sentry.config.ts b/apps/crawler/src/config/sentry.config.ts new file mode 100644 index 00000000..431c6480 --- /dev/null +++ b/apps/crawler/src/config/sentry.config.ts @@ -0,0 +1,19 @@ +export type SentryConfig = { + dsn: string; + debug: boolean; + environment: string; + logLevels: string[]; + enabled: boolean; +}; + +export function createSentryConfig(env: Record): SentryConfig { + const { NODE_ENV, SENTRY_DSN, SENTRY_DEBUG, SENTRY_LOG_LEVELS } = env; + + return { + dsn: SENTRY_DSN, + debug: SENTRY_DEBUG === '1', + environment: NODE_ENV ?? 'development', + logLevels: SENTRY_LOG_LEVELS ? SENTRY_LOG_LEVELS.split(',') : ['error'], + enabled: !!SENTRY_DSN, + }; +} diff --git a/apps/crawler/src/config/subscribers.config.ts b/apps/crawler/src/config/subscribers.config.ts new file mode 100644 index 00000000..b7467cc3 --- /dev/null +++ b/apps/crawler/src/config/subscribers.config.ts @@ -0,0 +1,21 @@ +import { SubscriberName } from '@common/constants'; + +export type SubscribersConfig = { [key in SubscriberName]: boolean }; + +export function createSubscribersConfig( + env: Record, +): SubscribersConfig { + const { + ACCOUNTS_SUBSCRIBER_DISABLE, + BLOCKS_SUBSCRIBER_DISABLE, + COLLECTIONS_SUBSCRIBER_DISABLE, + TOKENS_SUBSCRIBER_DISABLE, + } = env; + + return { + [SubscriberName.ACCOUNTS]: ACCOUNTS_SUBSCRIBER_DISABLE !== 'true', + [SubscriberName.BLOCKS]: BLOCKS_SUBSCRIBER_DISABLE !== 'true', + [SubscriberName.COLLECTIONS]: COLLECTIONS_SUBSCRIBER_DISABLE !== 'true', + [SubscriberName.TOKENS]: TOKENS_SUBSCRIBER_DISABLE !== 'true', + }; +} diff --git a/apps/crawler/src/crawler.module.ts b/apps/crawler/src/crawler.module.ts index 31fea068..991d242a 100644 --- a/apps/crawler/src/crawler.module.ts +++ b/apps/crawler/src/crawler.module.ts @@ -1,32 +1,27 @@ -import typeormConfig from '@common/typeorm.config'; -import { Logger, Module } from '@nestjs/common'; -import { ConfigModule, ConfigService } from '@nestjs/config'; +import { Module } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { ProcessorConfigService } from './processor.config.service'; +import { SentryModule } from '@ntegral/nestjs-sentry'; +import typeormConfig from '@common/typeorm.config'; import { CrawlerService } from './crawler.service'; import { SubscribersModule } from './subscribers/subscribers.module'; -import { SentryModule } from '@ntegral/nestjs-sentry'; +import { Config, GlobalConfigModule } from './config/config.module'; +import { CacheProviderModule } from './cache/cache-provider.module'; @Module({ imports: [ - ConfigModule.forRoot(), + GlobalConfigModule, + CacheProviderModule, TypeOrmModule.forRoot(typeormConfig), SentryModule.forRootAsync({ - imports: [ConfigModule], - useFactory: async (config: ConfigService) => ({ - dsn: config.get('SENTRY_DSN'), - debug: config.get('SENTRY_DEBUG') === '1', - environment: process.env.NODE_ENV ?? 'development', - logLevels: config.get('SENTRY_LOG_LEVELS') - ? config.get('SENTRY_LOG_LEVELS').split(',') - : ['error'], - enabled: !!config.get('SENTRY_DSN'), - }), + useFactory: async (configService: ConfigService) => { + return configService.get('sentry'); + }, inject: [ConfigService], }), SubscribersModule, ], controllers: [], - providers: [Logger, CrawlerService, ProcessorConfigService], + providers: [CrawlerService], }) export class CrawlerModule {} diff --git a/apps/crawler/src/crawler.service.ts b/apps/crawler/src/crawler.service.ts index 02dd7aaf..0db05391 100644 --- a/apps/crawler/src/crawler.service.ts +++ b/apps/crawler/src/crawler.service.ts @@ -1,39 +1,11 @@ import { Injectable } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { ProcessorService } from './subscribers/processor.service'; -import { AccountsSubscriberService } from './subscribers/accounts-subscriber.service'; -import { BlocksSubscriberService } from './subscribers/blocks-subscriber.service'; -import { CollectionsSubscriberService } from './subscribers/collections-subscriber.service'; -import { TokensSubscriberService } from './subscribers/tokens-subscriber.service'; +import { SubscribersService } from './subscribers/subscribers.service'; @Injectable() export class CrawlerService { - constructor( - private configService: ConfigService, - private processorService: ProcessorService, - private accountsSubscriberService: AccountsSubscriberService, - private blocksSubscriberService: BlocksSubscriberService, - private collectionsSubscriberService: CollectionsSubscriberService, - private tokensSubscriberService: TokensSubscriberService, - ) {} + constructor(private subscribersService: SubscribersService) {} - run(forceRescan = false) { - if (this.configService.get('ACCOUNTS_SUBSCRIBER_DISABLE') !== 'true') { - this.accountsSubscriberService.subscribe(); - } - - if (this.configService.get('BLOCKS_SUBSCRIBER_DISABLE') !== 'true') { - this.blocksSubscriberService.subscribe(); - } - - if (this.configService.get('COLLECTIONS_SUBSCRIBER_DISABLE') !== 'true') { - this.collectionsSubscriberService.subscribe(); - } - - if (this.configService.get('TOKENS_SUBSCRIBER_DISABLE') !== 'true') { - this.tokensSubscriberService.subscribe(); - } - - return this.processorService.run(forceRescan); + run() { + return this.subscribersService.run(); } } diff --git a/apps/crawler/src/main.ts b/apps/crawler/src/main.ts index 10d3340b..1a1b4c02 100644 --- a/apps/crawler/src/main.ts +++ b/apps/crawler/src/main.ts @@ -1,24 +1,23 @@ import { Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { NestFactory } from '@nestjs/core'; +import { Config } from './config/config.module'; import { CrawlerModule } from './crawler.module'; import { CrawlerService } from './crawler.service'; async function bootstrap() { const app = await NestFactory.create(CrawlerModule); - const configService = app.get(ConfigService); + const configService = app.get(ConfigService) as ConfigService; - const logLevels = configService.get('LOG_LEVELS') - ? configService.get('LOG_LEVELS').split(',') - : ['log', 'error', 'warn']; + const logLevels = configService.get('logLevels'); Logger.overrideLogger(logLevels); try { const crawlerService = app.get(CrawlerService); - await crawlerService.run(process.env.SCAN_FORCE_RESCAN === 'true'); + await crawlerService.run(); } catch (err) { // eslint-disable-next-line no-console console.error(err); diff --git a/apps/crawler/src/sdk/sdk-cache.decorator.ts b/apps/crawler/src/sdk/sdk-cache.decorator.ts new file mode 100644 index 00000000..832a48b3 --- /dev/null +++ b/apps/crawler/src/sdk/sdk-cache.decorator.ts @@ -0,0 +1,48 @@ +import { CACHE_MANAGER, Inject } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Cache } from 'cache-manager'; +import { Config } from '../config/config.module'; + +export function SdkCache(key?: string) { + const cacheManagerInjection = Inject(CACHE_MANAGER); + const configServiceInjection = Inject(ConfigService); + + return function ( + target: Record, + _, + descriptor: PropertyDescriptor, + ) { + configServiceInjection(target, 'configService'); + + cacheManagerInjection(target, 'cacheManager'); + + const method = descriptor.value; + + descriptor.value = async function (...args: Array) { + const configService = this.configService as ConfigService; + + const cacheManager = this.cacheManager as Cache; + + const entryKey = `${key}[${args + .map((res) => JSON.stringify(res)) + .join(',')}]`; + + // Get data from cache only while rescan mode + if (configService.get('rescan')) { + const cachedValue = await cacheManager.get(entryKey); + + if (cachedValue !== undefined) { + return cachedValue; + } + } + + // If no cache found, get data by real sdk call. + const result = await method.apply(this, args); + + // Set cache value. + await cacheManager.set(entryKey, result); + + return result; + }; + }; +} diff --git a/apps/crawler/src/sdk/factory.ts b/apps/crawler/src/sdk/sdk-factory.ts similarity index 100% rename from apps/crawler/src/sdk/factory.ts rename to apps/crawler/src/sdk/sdk-factory.ts diff --git a/apps/crawler/src/sdk/sdk.module.ts b/apps/crawler/src/sdk/sdk.module.ts index 6a5bcca8..8c98f97e 100644 --- a/apps/crawler/src/sdk/sdk.module.ts +++ b/apps/crawler/src/sdk/sdk.module.ts @@ -1,8 +1,9 @@ import { Global, Module } from '@nestjs/common'; import { ConfigModule, ConfigService } from '@nestjs/config'; -import { sdkFactory } from './factory'; -import { SdkService } from './sdk.service'; import { Sdk } from '@unique-nft/substrate-client'; +import { sdkFactory } from './sdk-factory'; +import { SdkService } from './sdk.service'; +import { Config } from '../config/config.module'; import '@unique-nft/substrate-client/extrinsics'; import '@unique-nft/substrate-client/tokens'; import '@unique-nft/substrate-client/balance'; @@ -13,8 +14,8 @@ import '@unique-nft/substrate-client/balance'; providers: [ { provide: Sdk, - useFactory: async (configService: ConfigService) => - sdkFactory(configService.get('CHAIN_WS_URL')), + useFactory: async (configService: ConfigService) => + sdkFactory(configService.get('chainWsUrl')), inject: [ConfigService], }, SdkService, diff --git a/apps/crawler/src/sdk/sdk.service.ts b/apps/crawler/src/sdk/sdk.service.ts index 95ff6ed4..5c02b7e1 100644 --- a/apps/crawler/src/sdk/sdk.service.ts +++ b/apps/crawler/src/sdk/sdk.service.ts @@ -1,26 +1,44 @@ -import { Injectable } from '@nestjs/common'; +import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { Sdk } from '@unique-nft/substrate-client'; import { CollectionInfoWithSchema, TokenByIdResult, TokenPropertiesResult, } from '@unique-nft/substrate-client/tokens'; +import { Config } from '../config/config.module'; +import { SdkCache } from './sdk-cache.decorator'; @Injectable() export class SdkService { - constructor(private sdk: Sdk) {} + constructor( + private sdk: Sdk, + private configService: ConfigService, + @Inject(CACHE_MANAGER) private cacheManager: Cache, + ) {} + @SdkCache('getCollection') getCollection( collectionId: number, ): Promise { return this.sdk.collections.get_new({ collectionId }); } + @SdkCache('getCollectionLimits') async getCollectionLimits(collectionId: number) { const result = await this.sdk.collections.getLimits({ collectionId }); return result?.limits; } + @SdkCache('getTokenPropertyPermissions') + async getTokenPropertyPermissions(collectionId: number) { + const result = await this.sdk.collections.propertyPermissions({ + collectionId, + }); + return result?.propertyPermissions ?? []; + } + + @SdkCache('getToken') getToken( collectionId: number, tokenId: number, @@ -28,6 +46,7 @@ export class SdkService { return this.sdk.tokens.get_new({ collectionId, tokenId }); } + @SdkCache('getTokenProperties') getTokenProperties( collectionId: number, tokenId: number, @@ -35,6 +54,7 @@ export class SdkService { return this.sdk.tokens.properties({ collectionId, tokenId }); } + @SdkCache('getBalances') getBalances(rawAddress: string) { return this.sdk.balance.get({ address: rawAddress }); } diff --git a/apps/crawler/src/subscribers/accounts-subscriber.service.ts b/apps/crawler/src/subscribers/accounts.subscriber.service.ts similarity index 77% rename from apps/crawler/src/subscribers/accounts-subscriber.service.ts rename to apps/crawler/src/subscribers/accounts.subscriber.service.ts index c7e09bc2..c6e82fdb 100644 --- a/apps/crawler/src/subscribers/accounts-subscriber.service.ts +++ b/apps/crawler/src/subscribers/accounts.subscriber.service.ts @@ -1,33 +1,31 @@ import { Injectable, Logger } from '@nestjs/common'; -import { Repository } from 'typeorm'; -import { InjectRepository } from '@nestjs/typeorm'; import { Store } from '@subsquid/typeorm-store'; import { EventHandlerContext } from '@subsquid/substrate-processor'; -import { Account } from '@entities/Account'; -import { SdkService } from '../sdk/sdk.service'; -import { ProcessorService } from './processor.service'; -import { EventName } from '@common/constants'; -import { normalizeSubstrateAddress, normalizeTimestamp } from '@common/utils'; -import ISubscriberService from './subscriber.interface'; +import { Severity } from '@sentry/node'; import { AllBalances } from '@unique-nft/substrate-client/types'; +import { EventName } from '@common/constants'; +import { SdkService } from '../sdk/sdk.service'; import { InjectSentry, SentryService } from '@ntegral/nestjs-sentry'; -import { Severity } from '@sentry/node'; +import { ProcessorService } from './processor/processor.service'; +import { ISubscriberService } from './subscribers.service'; +import { AccountWriterService } from '../writers/account.writer.service'; @Injectable() export class AccountsSubscriberService implements ISubscriberService { private readonly logger = new Logger(AccountsSubscriberService.name); constructor( - @InjectRepository(Account) - private accountsRepository: Repository, - private processorService: ProcessorService, private sdkService: SdkService, - @InjectSentry() private readonly sentry: SentryService, + + private accountWriterService: AccountWriterService, + + @InjectSentry() + private readonly sentry: SentryService, ) { this.sentry.setContext(AccountsSubscriberService.name); } - subscribe() { + subscribe(processorService: ProcessorService) { [ EventName.NEW_ACCOUNT, EventName.COLLECTION_CREATED, @@ -40,7 +38,7 @@ export class AccountsSubscriberService implements ISubscriberService { EventName.BALANCES_WITHDRAW, EventName.BALANCES_TRANSFER, ].forEach((eventName) => - this.processorService.processor.addEventHandler( + processorService.processor.addEventHandler( eventName, this.upsertHandler.bind(this), ), @@ -99,7 +97,7 @@ export class AccountsSubscriberService implements ISubscriberService { * "0xc89axxx" * */ - private getAddressValues( + private extractAddressValues( eventName: string, args: string | object | (string | number)[], ): string[] { @@ -149,46 +147,20 @@ export class AccountsSubscriberService implements ISubscriberService { return addresses; } - /** - * Prepares event and balances data to write into db. - */ - private prepareDataForDb(params: { - timestamp: number; - blockNumber: number; - balances: AllBalances; - }): Account { - const { - blockNumber, - timestamp, - balances: { address, availableBalance, lockedBalance, freeBalance }, - } = params; - - return { - block_height: String(blockNumber), - timestamp: String(timestamp), - account_id: address, - account_id_normalized: normalizeSubstrateAddress(address), - available_balance: availableBalance.amount, - free_balance: freeBalance.amount, - locked_balance: lockedBalance.amount, - }; - } - private async upsertHandler(ctx: EventHandlerContext): Promise { const { - block: { height: blockNumber, timestamp: rawTimestamp }, + block: { height: blockNumber, timestamp: blockTimestamp }, event: { name: eventName, args }, } = ctx; const log = { eventName, - blockNumber, rawAddressValues: [], processedAccounts: [], }; try { - const rawAddressValues = this.getAddressValues(eventName, args); + const rawAddressValues = this.extractAddressValues(eventName, args); log.rawAddressValues = rawAddressValues; if (!rawAddressValues.length) { @@ -198,8 +170,6 @@ export class AccountsSubscriberService implements ISubscriberService { // Get balances and converted address from sdk const balancesData = await this.getBalances(rawAddressValues); - const timestamp = normalizeTimestamp(rawTimestamp); - await Promise.all( balancesData.map((balances, addressIndex) => { if (!balances) { @@ -219,14 +189,11 @@ export class AccountsSubscriberService implements ISubscriberService { log.processedAccounts.push(balances.address); - const dataToWrite = this.prepareDataForDb({ + return this.accountWriterService.upsert({ blockNumber, - timestamp, + blockTimestamp, balances, }); - - // Write data into db - return this.accountsRepository.upsert(dataToWrite, ['account_id']); }), ); diff --git a/apps/crawler/src/subscribers/blocks-subscriber.service.ts b/apps/crawler/src/subscribers/blocks-subscriber.service.ts deleted file mode 100644 index b7fde1cd..00000000 --- a/apps/crawler/src/subscribers/blocks-subscriber.service.ts +++ /dev/null @@ -1,381 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { Repository } from 'typeorm'; -import { InjectRepository } from '@nestjs/typeorm'; -import { Store } from '@subsquid/typeorm-store'; -import { - BlockHandlerContext, - SubstrateBlock, - SubstrateExtrinsic, -} from '@subsquid/substrate-processor'; -import { InjectSentry, SentryService } from '@ntegral/nestjs-sentry'; -import { ProcessorService } from './processor.service'; -import ISubscriberService from './subscriber.interface'; -import { Block } from '@entities/Block'; -import { - getAmount, - normalizeSubstrateAddress, - normalizeTimestamp, -} from '@common/utils'; -import { - EventMethod, - EventSection, - ExtrinsicMethod, - ExtrinsicSection, -} from '@common/constants'; -import { Extrinsic } from '@entities/Extrinsic'; -import { Event } from '@entities/Event'; -import { Prefix } from '@unique-nft/api/.'; - -const EVENT_TRANSFER = `${EventSection.BALANCES}.${EventMethod.TRANSFER}`; -const EVENT_ENDOWED = `${EventSection.BALANCES}.${EventMethod.ENDOWED}`; - -const EXTRINSICS_TRANSFER_METHODS = [ - ExtrinsicMethod.TRANSFER, - ExtrinsicMethod.TRANSFER_FROM, - ExtrinsicMethod.TRANSFER_ALL, - ExtrinsicMethod.TRANSFER_KEEP_ALIVE, - ExtrinsicMethod.VESTED_TRANSFER, -]; - -interface IExtrinsicExtended extends SubstrateExtrinsic { - name: string; -} - -interface IBlockItem { - kind: 'event' | 'call'; - name: string; - event?: object; - extrinsic?: object; -} - -interface IEvent { - name: string; - extrinsic: SubstrateExtrinsic; - indexInBlock: number; - phase: string; - args: { value?: string; amount?: string }; -} - -interface IExtrinsicRecipient { - // Unique.transfer - recipient?: { value: string }; - - // Balances.transfer - // Balances.transfer_all - // Balances.transfer_keep_alive - // Vesting.vested_transfer - dest?: { value: string }; -} - -interface IBlockCommonData { - timestamp: number; - blockNumber: number; - ss58Prefix: Prefix; -} - -@Injectable() -export class BlocksSubscriberService implements ISubscriberService { - private readonly logger = new Logger(BlocksSubscriberService.name); - - constructor( - @InjectRepository(Block) private blocksRepository: Repository, - @InjectRepository(Extrinsic) - private extrinsicsRepository: Repository, - @InjectRepository(Event) private eventsRepository: Repository, - @InjectSentry() private readonly sentry: SentryService, - private processorService: ProcessorService, - ) { - this.sentry.setContext(BlocksSubscriberService.name); - } - - subscribe() { - this.processorService.processor.addPreHook( - { - data: { - includeAllBlocks: true, - items: true, - }, - } as const, - this.upsertHandler.bind(this), - ); - } - - private processExtrinsicItems(items: IBlockItem[]) { - const extrinsics = items - .filter(({ kind }) => kind === 'call') - .map((item) => { - const { name, extrinsic } = item; - return { name, ...extrinsic } as IExtrinsicExtended; - }); - - return { count: extrinsics.length, extrinsics }; - } - - private processEventItems(items: IBlockItem[]) { - const events = items - .filter(({ kind }) => kind === 'event') - .map((item) => item.event as IEvent); - - // Save 'amount' and 'fee' for extrinsic's events - const extrinsicsValues = events.reduce((acc, curr) => { - const { name, extrinsic, args } = curr; - - const extrinsicId = extrinsic?.id; - if (!extrinsicId) { - return acc; - } - - const rawAmount = - typeof args === 'string' ? args : args?.amount || args?.value; - - if (name === `${EventSection.BALANCES}.${EventMethod.TRANSFER}`) { - // Save extrinsic amount - acc[extrinsicId] = acc[extrinsicId] || {}; - acc[extrinsicId].amount = getAmount(rawAmount); - } else if (name === `${EventSection.TREASURY}.${EventMethod.DEPOSIT}`) { - // Save extrinsic fee - acc[extrinsicId] = acc[extrinsicId] || {}; - acc[extrinsicId].fee = getAmount(rawAmount); - } else { - return acc; - } - - return { ...acc }; - }, {}); - - return { - count: events.length, - events, - extrinsicsValues, - numTransfers: events.filter(({ name }) => name === EVENT_TRANSFER).length, - newAccounts: events.filter(({ name }) => name === EVENT_ENDOWED).length, - }; - } - - private processContextData({ - block, - items, - ss58Prefix, - }: { - block: SubstrateBlock; - items: IBlockItem[]; - ss58Prefix: Prefix; - }) { - const processedEventsItems = this.processEventItems(items); - - const processedExtrinsicsItems = this.processExtrinsicItems(items); - - const { height: blockNumber, timestamp: rawTimestamp } = block; - const timestamp = normalizeTimestamp(rawTimestamp); - const blockCommonData = { - blockNumber, - timestamp, - ss58Prefix, - } as IBlockCommonData; - - // Create extrinsics data to write - const { extrinsicsValues } = processedEventsItems; - const { extrinsics } = processedExtrinsicsItems; - const extrinsicsDataToWrite = this.getExtrinsicsDataToWrite( - extrinsics, - extrinsicsValues, - blockCommonData, - ); - - // Create events data to write - const { events } = processedEventsItems; - const eventsDataToWrite = this.getEventsDataToWrite( - events, - blockCommonData, - ); - - // Create block data to write - const { specId, hash, parentHash } = block; - const [specName, specVersion] = specId.split('@') as [string, number]; - - const { - count: eventsCount, - numTransfers, - newAccounts, - } = processedEventsItems; - - const { count: extrinsicsCount } = processedExtrinsicsItems; - - const blockDataToWrite = { - block_number: blockNumber, - block_hash: hash, - parent_hash: parentHash, - spec_name: specName, - spec_version: specVersion, - timestamp: String(timestamp), - - // events info - total_events: eventsCount, - num_transfers: numTransfers, - new_accounts: newAccounts, - - // extrinsics info - total_extrinsics: extrinsicsCount, - - // todo or not todo - extrinsics_root: '', - state_root: '', - session_length: '0', - total_issuance: '', - need_rescan: false, - }; - - return { - blockData: blockDataToWrite, - extrinsicsData: extrinsicsDataToWrite, - eventsData: eventsDataToWrite, - }; - } - - private getExtrinsicsDataToWrite( - extrinsics: IExtrinsicExtended[], - extrinsicsEventValues: { - [key: string]: { amount?: string; fee?: string }; - }, - blockCommonData: IBlockCommonData, - ) { - return extrinsics - .map((extrinsic) => { - const { name } = extrinsic; - const [section, method] = name.split('.') as [ - ExtrinsicSection, - ExtrinsicMethod, - ]; - - const { timestamp, blockNumber, ss58Prefix } = blockCommonData; - - let signer = null; - const { signature } = extrinsic; - if (signature) { - const { - address: { value: rawSigner }, - } = signature; - - signer = normalizeSubstrateAddress(rawSigner, ss58Prefix); - } - - const { - call: { args }, - } = extrinsic; - - let toOwner = null; - if (EXTRINSICS_TRANSFER_METHODS.includes(method)) { - const recipientAddress = args as IExtrinsicRecipient; - const rawToOwner = - recipientAddress?.recipient?.value || recipientAddress?.dest?.value; - toOwner = normalizeSubstrateAddress(rawToOwner, ss58Prefix); - } - - const { id, hash, indexInBlock, success } = extrinsic; - - const { amount = '0', fee = '0' } = extrinsicsEventValues[id] || {}; - - return { - timestamp: String(timestamp), - block_number: String(blockNumber), - block_index: `${blockNumber}-${indexInBlock}`, - extrinsic_index: indexInBlock, - section, - method, - hash, - success, - is_signed: !!signature, - signer, - signer_normalized: signer && normalizeSubstrateAddress(signer), - to_owner: toOwner, - to_owner_normalized: toOwner && normalizeSubstrateAddress(toOwner), - amount, - fee, - }; - }) - .filter((item) => !!item); - } - - private getEventsDataToWrite( - events: IEvent[], - blockCommonData: IBlockCommonData, - ) { - return events - .map((event) => { - const { name, indexInBlock, phase, extrinsic, args } = event; - const { timestamp, blockNumber } = blockCommonData; - - const [section, method] = name.split('.') as [ - EventSection, - EventMethod, - ]; - - const rawAmount = args?.amount || args?.value; - - return { - timestamp: String(timestamp), - block_number: String(blockNumber), - event_index: indexInBlock, - block_index: `${blockNumber}-${ - extrinsic ? extrinsic.indexInBlock : '' - }`, - section, - method, - // todo: Make more clean connect to extrinsic - phase: - phase === 'ApplyExtrinsic' ? String(extrinsic.indexInBlock) : phase, - data: JSON.stringify(args), - amount: rawAmount ? getAmount(rawAmount) : null, - }; - }) - .filter((item) => !!item); - } - - private async upsertHandler(ctx: BlockHandlerContext): Promise { - const { block, items } = ctx; - const { height: blockNumber } = block; - - const log = { - blockNumber, - }; - - try { - const ss58Prefix = ctx._chain?.getConstant( - 'System', - 'SS58Prefix', - ) as Prefix; - - const { blockData, extrinsicsData, eventsData } = this.processContextData( - { - block, - items, - ss58Prefix, - }, - ); - - await Promise.all([ - this.blocksRepository.upsert(blockData, ['block_number']), - this.extrinsicsRepository.upsert(extrinsicsData, [ - 'block_number', - 'extrinsic_index', - ]), - this.eventsRepository.upsert(eventsData, [ - 'block_number', - 'event_index', - ]), - ]); - - const { total_events: totalEvents, total_extrinsics: totalExtrinsics } = - blockData; - - this.logger.verbose({ - ...log, - totalEvents, - totalExtrinsics, - }); - } catch (error) { - this.logger.error({ ...log, error: error.message || error }); - this.sentry.instance().captureException({ ...log, error }); - } - } -} diff --git a/apps/crawler/src/subscribers/blocks.subscriber.service.ts b/apps/crawler/src/subscribers/blocks.subscriber.service.ts new file mode 100644 index 00000000..7d0fa036 --- /dev/null +++ b/apps/crawler/src/subscribers/blocks.subscriber.service.ts @@ -0,0 +1,122 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Store } from '@subsquid/typeorm-store'; +import { + BlockHandlerContext, + SubstrateExtrinsic, +} from '@subsquid/substrate-processor'; +import { InjectSentry, SentryService } from '@ntegral/nestjs-sentry'; +import { ISubscriberService } from './subscribers.service'; +import { Prefix } from '@unique-nft/api/.'; +import { ProcessorService } from './processor/processor.service'; +import { BlockWriterService } from '../writers/block.writer.service'; +import { ExtrinsicWriterService } from '../writers/extrinsic.writer.service'; +import { EventWriterService } from '../writers/event.writer.service'; + +export interface IEvent { + name: string; + extrinsic: SubstrateExtrinsic; + indexInBlock: number; + phase: string; + args: { value?: string; amount?: string }; +} + +export type IBlockItem = + | { + kind: 'event'; + name: string; + event: IEvent; + } + | { + kind: 'call'; + name: string; + extrinsic: SubstrateExtrinsic; + }; + +export interface IBlockCommonData { + blockNumber: number; + blockTimestamp: number; + ss58Prefix: Prefix; +} + +export interface IItemCounts { + totalEvents: number; + totalExtrinsics: number; + numTransfers: number; + newAccounts: number; +} + +@Injectable() +export class BlocksSubscriberService implements ISubscriberService { + private readonly logger = new Logger(BlocksSubscriberService.name); + + constructor( + private blockWriterService: BlockWriterService, + + private extrinsicWriterService: ExtrinsicWriterService, + + private eventWriterService: EventWriterService, + + @InjectSentry() + private readonly sentry: SentryService, + ) { + this.sentry.setContext(BlocksSubscriberService.name); + } + + subscribe(processorService: ProcessorService) { + processorService.processor.addPreHook( + { + data: { + includeAllBlocks: true, + items: true, + }, + } as const, + this.upsertHandler.bind(this), + ); + } + + private async upsertHandler(ctx: BlockHandlerContext): Promise { + const { block, items } = ctx; + const { height: blockNumber, timestamp: blockTimestamp } = block; + const blockItems = items as unknown as IBlockItem[]; + + const log = { + blockNumber, + }; + + try { + const ss58Prefix = ctx._chain?.getConstant( + 'System', + 'SS58Prefix', + ) as Prefix; + + const blockCommonData = { + blockNumber, + blockTimestamp, + ss58Prefix, + } as IBlockCommonData; + + const [itemCounts] = await Promise.all([ + this.blockWriterService.upsert({ + block, + blockItems, + }), + this.extrinsicWriterService.upsert({ + blockCommonData, + blockItems, + }), + this.eventWriterService.upsert({ + blockCommonData, + blockItems, + }), + ]); + + this.logger.verbose({ + ...log, + ...itemCounts, + }); + } catch (error) { + this.logger.error({ ...log, error: error.message || error }); + this.sentry.instance().captureException({ ...log, error }); + } + } +} diff --git a/apps/crawler/src/subscribers/collections.subscriber.service.ts b/apps/crawler/src/subscribers/collections.subscriber.service.ts new file mode 100644 index 00000000..33e11ef8 --- /dev/null +++ b/apps/crawler/src/subscribers/collections.subscriber.service.ts @@ -0,0 +1,146 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { EventHandlerContext } from '@subsquid/substrate-processor'; +import { Store } from '@subsquid/typeorm-store'; +import { EventName, SubscriberAction } from '@common/constants'; +import { SdkService } from '../sdk/sdk.service'; +import { InjectSentry, SentryService } from '@ntegral/nestjs-sentry'; +import { + ICollectionData, + CollectionWriterService, +} from '../writers/collection.writer.service'; +import { ProcessorService } from './processor/processor.service'; +import { ISubscriberService } from './subscribers.service'; + +@Injectable() +export class CollectionsSubscriberService implements ISubscriberService { + private readonly logger = new Logger(CollectionsSubscriberService.name); + + constructor( + private sdkService: SdkService, + private collectionWriterService: CollectionWriterService, + @InjectSentry() + private readonly sentry: SentryService, + ) { + this.sentry.setContext(CollectionsSubscriberService.name); + } + + subscribe(processorService: ProcessorService) { + [ + EventName.COLLECTION_CREATED, + EventName.COLLECTION_PROPERTY_SET, + EventName.COLLECTION_PROPERTY_DELETED, + EventName.PROPERTY_PERMISSION_SET, + EventName.COLLECTION_SPONSOR_REMOVED, + EventName.COLLECTION_OWNED_CHANGED, + EventName.SPONSORSHIP_CONFIRMED, + EventName.COLLECTION_LIMIT_SET, + EventName.COLLECTION_SPONSOR_SET, + ].forEach((eventName) => + processorService.processor.addEventHandler( + eventName, + this.upsertHandler.bind(this), + ), + ); + + processorService.processor.addEventHandler( + EventName.COLLECTION_DESTROYED, + this.destroyHandler.bind(this), + ); + } + + /** + * Extracts collection id from archive event. + */ + private extractCollectionId(args): number { + return typeof args === 'number' ? args : (args[0] as number); + } + + /** + * Recieves collection data from sdk. + */ + private async getCollectionData( + collectionId: number, + ): Promise { + const [collectionDecoded, collectionLimits, tokenPropertyPermissions] = + await Promise.all([ + this.sdkService.getCollection(collectionId), + this.sdkService.getCollectionLimits(collectionId), + this.sdkService.getTokenPropertyPermissions(collectionId), + ]); + + return { + collectionDecoded, + collectionLimits, + tokenPropertyPermissions, + }; + } + + private async upsertHandler(ctx: EventHandlerContext): Promise { + const { + block: { height: blockNumber, timestamp: blockTimestamp }, + event: { name: eventName, args }, + } = ctx; + + const log = { + eventName, + blockNumber, + collectionId: null as null | number, + action: null as null | SubscriberAction, + }; + + try { + const collectionId = this.extractCollectionId(args); + + log.collectionId = collectionId; + + const collectionData = await this.getCollectionData(collectionId); + + if (collectionData.collectionDecoded) { + await this.collectionWriterService.upsert({ + eventName, + blockTimestamp, + collectionData, + }); + + log.action = SubscriberAction.UPSERT; + } else { + // No entity returned from sdk. Most likely it was destroyed in a future block. + await this.collectionWriterService.delete(collectionId); + + log.action = SubscriberAction.DELETE_NOT_FOUND; + } + + this.logger.verbose({ ...log }); + } catch (error) { + this.logger.error({ ...log, error: error.message }); + this.sentry.instance().captureException({ ...log, error }); + } + } + + private async destroyHandler(ctx: EventHandlerContext): Promise { + const { + block: { height: blockNumber }, + event: { name: eventName, args }, + } = ctx; + + const log = { + eventName, + blockNumber, + collectionId: null as null | number, + action: SubscriberAction.DELETE, + }; + + try { + const collectionId = this.extractCollectionId(args); + + log.collectionId = collectionId; + + await this.collectionWriterService.delete(collectionId); + + this.logger.verbose({ ...log }); + } catch (error) { + this.logger.error({ ...log, error: error.message }); + this.sentry.instance().captureException({ ...log, error }); + } + } +} diff --git a/apps/crawler/src/subscribers/processor.service.ts b/apps/crawler/src/subscribers/processor/processor.service.ts similarity index 93% rename from apps/crawler/src/subscribers/processor.service.ts rename to apps/crawler/src/subscribers/processor/processor.service.ts index 8435b381..975d6b71 100644 --- a/apps/crawler/src/subscribers/processor.service.ts +++ b/apps/crawler/src/subscribers/processor/processor.service.ts @@ -8,7 +8,7 @@ import { } from '@subsquid/typeorm-store'; import { Connection, DataSource } from 'typeorm'; import { InjectSentry, SentryService } from '@ntegral/nestjs-sentry'; -import { ProcessorConfigService } from '../processor.config.service'; +import { ProcessorConfigService } from '../../config/processor.config.service'; interface IScanDatabaseOptions extends TypeormDatabaseOptions { stateSchema: string; @@ -70,10 +70,9 @@ export class ProcessorService { private processorConfigService: ProcessorConfigService, @InjectSentry() private readonly sentry: SentryService, ) { - this.stateSchema = - this.processorConfigService.getForceMode() === 'true' - ? STATE_SCHEMA_NAME_BY_MODE.RESCAN - : STATE_SCHEMA_NAME_BY_MODE.SCAN; + this.stateSchema = this.processorConfigService.isRescan() + ? STATE_SCHEMA_NAME_BY_MODE.RESCAN + : STATE_SCHEMA_NAME_BY_MODE.SCAN; const db = new ScanDatabase({ stateSchema: this.stateSchema, diff --git a/apps/crawler/src/subscribers/subscriber.interface.ts b/apps/crawler/src/subscribers/subscriber.interface.ts deleted file mode 100644 index 8e726088..00000000 --- a/apps/crawler/src/subscribers/subscriber.interface.ts +++ /dev/null @@ -1,3 +0,0 @@ -export default interface ISubscriberService { - subscribe(); -} diff --git a/apps/crawler/src/subscribers/subscribers.module.ts b/apps/crawler/src/subscribers/subscribers.module.ts index e891e8bf..77be5739 100644 --- a/apps/crawler/src/subscribers/subscribers.module.ts +++ b/apps/crawler/src/subscribers/subscribers.module.ts @@ -1,49 +1,26 @@ -import { Block } from '@entities/Block'; -import { Collections } from '@entities/Collections'; -import { Tokens } from '@entities/Tokens'; -import { Event } from '@entities/Event'; import { Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { ProcessorConfigService } from '../processor.config.service'; -import { ProcessorService } from './processor.service'; -import { SubstrateProcessor } from '@subsquid/substrate-processor'; -import { CollectionsSubscriberService } from './collections-subscriber.service'; -import { TokensSubscriberService } from './tokens-subscriber.service'; -import { BlocksSubscriberService } from './blocks-subscriber.service'; -import { Extrinsic } from '@entities/Extrinsic'; -import { Account } from '@entities/Account'; -import { AccountsSubscriberService } from './accounts-subscriber.service'; +import { CollectionsSubscriberService } from './collections.subscriber.service'; +import { TokensSubscriberService } from './tokens.subscriber.service'; +import { BlocksSubscriberService } from './blocks.subscriber.service'; +import { AccountsSubscriberService } from './accounts.subscriber.service'; import { SdkModule } from '../sdk/sdk.module'; +import { WritersModule } from '../writers/writers.module'; +import { ProcessorConfigService } from '../config/processor.config.service'; +import { ProcessorService } from './processor/processor.service'; +import { SubscribersService } from './subscribers.service'; @Module({ - imports: [ - TypeOrmModule.forFeature([ - Account, - Block, - Collections, - Event, - Extrinsic, - Tokens, - ]), - ConfigModule, - SdkModule, - ], + imports: [ConfigModule, SdkModule, WritersModule], providers: [ - SubstrateProcessor, ProcessorService, ProcessorConfigService, AccountsSubscriberService, BlocksSubscriberService, CollectionsSubscriberService, TokensSubscriberService, + SubscribersService, ], - exports: [ - ProcessorService, - AccountsSubscriberService, - BlocksSubscriberService, - CollectionsSubscriberService, - TokensSubscriberService, - ], + exports: [SubscribersService], }) export class SubscribersModule {} diff --git a/apps/crawler/src/subscribers/subscribers.service.ts b/apps/crawler/src/subscribers/subscribers.service.ts new file mode 100644 index 00000000..292e4ff6 --- /dev/null +++ b/apps/crawler/src/subscribers/subscribers.service.ts @@ -0,0 +1,47 @@ +import { SubscriberName } from '@common/constants'; +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Config } from '../config/config.module'; +import { AccountsSubscriberService } from './accounts.subscriber.service'; +import { BlocksSubscriberService } from './blocks.subscriber.service'; +import { CollectionsSubscriberService } from './collections.subscriber.service'; +import { ProcessorService } from './processor/processor.service'; +import { TokensSubscriberService } from './tokens.subscriber.service'; + +export interface ISubscriberService { + subscribe(processorService: ProcessorService); +} + +@Injectable() +export class SubscribersService { + constructor( + private configService: ConfigService, + private processorService: ProcessorService, + private accountsSubscriberService: AccountsSubscriberService, + private blocksSubscriberService: BlocksSubscriberService, + private collectionsSubscriberService: CollectionsSubscriberService, + private tokensSubscriberService: TokensSubscriberService, + ) {} + + run() { + const subscribersConfig = this.configService.get('subscribers'); + + if (subscribersConfig[SubscriberName.ACCOUNTS]) { + this.accountsSubscriberService.subscribe(this.processorService); + } + + if (subscribersConfig[SubscriberName.BLOCKS]) { + this.blocksSubscriberService.subscribe(this.processorService); + } + + if (subscribersConfig[SubscriberName.COLLECTIONS]) { + this.collectionsSubscriberService.subscribe(this.processorService); + } + + if (subscribersConfig[SubscriberName.TOKENS]) { + this.tokensSubscriberService.subscribe(this.processorService); + } + + return this.processorService.run(this.configService.get('rescan')); + } +} diff --git a/apps/crawler/src/subscribers/tokens-subscriber.service.ts b/apps/crawler/src/subscribers/tokens-subscriber.service.ts deleted file mode 100644 index f68c5df8..00000000 --- a/apps/crawler/src/subscribers/tokens-subscriber.service.ts +++ /dev/null @@ -1,219 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { Repository } from 'typeorm'; -import { InjectRepository } from '@nestjs/typeorm'; -import { Store } from '@subsquid/typeorm-store'; -import { EventHandlerContext } from '@subsquid/substrate-processor'; -import { Tokens } from '@entities/Tokens'; -import { SdkService } from '../sdk/sdk.service'; -import { ProcessorService } from './processor.service'; -import { EventName } from '@common/constants'; -import { - normalizeSubstrateAddress, - normalizeTimestamp, - sanitizePropertiesValues, -} from '@common/utils'; -import ISubscriberService from './subscriber.interface'; -import { - CollectionInfoWithSchema, - TokenPropertiesResult, - TokenByIdResult, -} from '@unique-nft/substrate-client/tokens'; -import { InjectSentry, SentryService } from '@ntegral/nestjs-sentry'; - -@Injectable() -export class TokensSubscriberService implements ISubscriberService { - private readonly logger = new Logger(TokensSubscriberService.name); - - constructor( - @InjectRepository(Tokens) - private tokensRepository: Repository, - private processorService: ProcessorService, - private sdkService: SdkService, - @InjectSentry() private readonly sentry: SentryService, - ) { - this.sentry.setContext(TokensSubscriberService.name); - } - - subscribe() { - const EVENTS_TO_UPDATE = [ - // Insert - EventName.ITEM_CREATED, - - // Update - EventName.TRANSFER, - - // todo: Or maybe these events are reletad to collection? - EventName.TOKEN_PROPERTY_SET, - EventName.TOKEN_PROPERTY_DELETED, - ]; - - EVENTS_TO_UPDATE.forEach((eventName) => - this.processorService.processor.addEventHandler( - eventName, - this.upsertHandler.bind(this), - ), - ); - - this.processorService.processor.addEventHandler( - EventName.ITEM_DESTROYED, - this.destroyHandler.bind(this), - ); - } - - private async getTokenData( - collectionId: number, - tokenId: number, - ): Promise<{ - tokenDecoded: TokenByIdResult | null; - tokenProperties: TokenPropertiesResult | null; - collection: CollectionInfoWithSchema | null; - }> { - const [tokenDecoded, tokenProperties, collection] = await Promise.all([ - this.sdkService.getToken(collectionId, tokenId), - this.sdkService.getTokenProperties(collectionId, tokenId), - this.sdkService.getCollection(collectionId), - ]); - - return { - tokenDecoded, - tokenProperties, - collection, - }; - } - - prepareDataToWrite( - tokenDecoded: TokenByIdResult, - tokenProperties: TokenPropertiesResult, - collection: CollectionInfoWithSchema, - ) { - const { - tokenId: token_id, - collectionId: collection_id, - image, - attributes, - nestingParentToken, - owner, - } = tokenDecoded; - - const { owner: collectionOwner, tokenPrefix } = collection; - - let parentId = null; - if (nestingParentToken) { - const { collectionId, tokenId } = nestingParentToken; - parentId = `${collectionId}_${tokenId}`; - } - - return { - token_id, - collection_id, - owner, - owner_normalized: normalizeSubstrateAddress(owner), - image, - attributes, - properties: tokenProperties - ? sanitizePropertiesValues(tokenProperties.properties) - : [], - parent_id: parentId, - is_sold: owner !== collectionOwner, - token_name: `${tokenPrefix} #${token_id}`, - }; - } - - private async upsertHandler(ctx: EventHandlerContext): Promise { - const { - block: { height: blockNumber, timestamp: blockTimestamp }, - event: { name: eventName, args }, - } = ctx; - - const log = { - eventName, - blockNumber, - blockTimestamp, - entity: null as null | object | string, - collectionId: null as null | number, - tokenId: null as null | number, - }; - - try { - const [collectionId, tokenId] = args as [number, number]; - - log.collectionId = collectionId; - log.tokenId = tokenId; - - if (tokenId === 0) { - throw new Error('Bad tokenId'); - } - - const { tokenDecoded, tokenProperties, collection } = - await this.getTokenData(collectionId, tokenId); - - if (tokenDecoded) { - const dataToWrite = this.prepareDataToWrite( - tokenDecoded, - tokenProperties, - collection, - ); - - log.entity = String(dataToWrite); // Just to know that data is not null - - // Write collection data into db - await this.tokensRepository.upsert( - { - ...dataToWrite, - date_of_creation: - eventName === EventName.ITEM_CREATED - ? normalizeTimestamp(blockTimestamp) - : undefined, - }, - ['collection_id', 'token_id'], - ); - } else { - // No entity returned from sdk. Most likely it was destroyed in a future block. - log.entity = null; - - // Delete db record - await this.tokensRepository.delete({ - collection_id: collectionId, - token_id: tokenId, - }); - } - - this.logger.verbose({ ...log }); - } catch (err) { - this.logger.error({ ...log, error: err.message }); - } - } - - private async destroyHandler(ctx: EventHandlerContext): Promise { - const { - block: { height: blockNumber, timestamp: blockTimestamp }, - event: { name: eventName, args }, - } = ctx; - - const log = { - eventName, - blockNumber, - blockTimestamp, - collectionId: null as null | number, - tokenId: null as null | number, - }; - - try { - const [collectionId, tokenId] = args as [number, number]; - - log.collectionId = collectionId; - log.tokenId = tokenId; - - // Delete db record - await this.tokensRepository.delete({ - collection_id: collectionId, - token_id: tokenId, - }); - - this.logger.verbose({ ...log }); - } catch (error) { - this.logger.error({ ...log, error: error.message }); - this.sentry.instance().captureException({ ...log, error }); - } - } -} diff --git a/apps/crawler/src/subscribers/tokens.subscriber.service.ts b/apps/crawler/src/subscribers/tokens.subscriber.service.ts new file mode 100644 index 00000000..57a9c4c7 --- /dev/null +++ b/apps/crawler/src/subscribers/tokens.subscriber.service.ts @@ -0,0 +1,162 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Store } from '@subsquid/typeorm-store'; +import { EventHandlerContext } from '@subsquid/substrate-processor'; +import { SdkService } from '../sdk/sdk.service'; +import { EventName, SubscriberAction } from '@common/constants'; +import { InjectSentry, SentryService } from '@ntegral/nestjs-sentry'; +import { ProcessorService } from './processor/processor.service'; +import { ISubscriberService } from './subscribers.service'; +import { + ITokenData, + TokenWriterService, +} from '../writers/token.writer.service'; + +@Injectable() +export class TokensSubscriberService implements ISubscriberService { + private readonly logger = new Logger(TokensSubscriberService.name); + + constructor( + private sdkService: SdkService, + + private tokenWriterService: TokenWriterService, + + @InjectSentry() + private readonly sentry: SentryService, + ) { + this.sentry.setContext(TokensSubscriberService.name); + } + + subscribe(processorService: ProcessorService) { + [ + // Insert + EventName.ITEM_CREATED, + + // Update + EventName.TRANSFER, + EventName.TOKEN_PROPERTY_SET, + EventName.TOKEN_PROPERTY_DELETED, + ].forEach((eventName) => + processorService.processor.addEventHandler( + eventName, + this.upsertHandler.bind(this), + ), + ); + + processorService.processor.addEventHandler( + EventName.ITEM_DESTROYED, + this.destroyHandler.bind(this), + ); + } + + /** + * Extracts collection id and token id from archive event. + */ + private extractCollectionAndTokenId(args: [number, number]): { + collectionId: number; + tokenId: number; + } { + const [collectionId, tokenId] = args; + + return { + collectionId, + tokenId, + }; + } + + private async getTokenData( + collectionId: number, + tokenId: number, + ): Promise { + const [tokenDecoded, tokenProperties, collectionDecoded] = + await Promise.all([ + this.sdkService.getToken(collectionId, tokenId), + this.sdkService.getTokenProperties(collectionId, tokenId), + this.sdkService.getCollection(collectionId), + ]); + + return { + tokenDecoded, + tokenProperties, + collectionDecoded, + }; + } + + private async upsertHandler(ctx: EventHandlerContext): Promise { + const { + block: { height: blockNumber, timestamp: blockTimestamp }, + event: { name: eventName, args }, + } = ctx; + + const log = { + eventName, + blockNumber, + collectionId: null as null | number, + tokenId: null as null | number, + action: null as null | SubscriberAction, + }; + + try { + const { collectionId, tokenId } = this.extractCollectionAndTokenId(args); + + log.collectionId = collectionId; + log.tokenId = tokenId; + + if (tokenId === 0) { + throw new Error('Bad tokenId'); + } + + const tokenData = await this.getTokenData(collectionId, tokenId); + + if (tokenData.tokenDecoded) { + await this.tokenWriterService.upsert({ + eventName, + blockTimestamp, + tokenData, + }); + + log.action = SubscriberAction.UPSERT; + } else { + // No entity returned from sdk. Most likely it was destroyed in a future block. + + // Delete db record + await this.tokenWriterService.delete(collectionId, tokenId); + + log.action = SubscriberAction.DELETE_NOT_FOUND; + } + + this.logger.verbose({ ...log }); + } catch (err) { + this.logger.error({ ...log, error: err.message }); + } + } + + private async destroyHandler(ctx: EventHandlerContext): Promise { + const { + block: { height: blockNumber }, + event: { name: eventName, args }, + } = ctx; + + const log = { + eventName, + blockNumber, + collectionId: null as null | number, + tokenId: null as null | number, + action: SubscriberAction.DELETE, + }; + + try { + const { collectionId, tokenId } = this.extractCollectionAndTokenId(args); + + log.collectionId = collectionId; + log.tokenId = tokenId; + + // Delete db record + await this.tokenWriterService.delete(collectionId, tokenId); + + this.logger.verbose({ ...log }); + } catch (error) { + this.logger.error({ ...log, error: error.message }); + this.sentry.instance().captureException({ ...log, error }); + } + } +} diff --git a/apps/crawler/src/writers/account.writer.service.ts b/apps/crawler/src/writers/account.writer.service.ts new file mode 100644 index 00000000..38188e52 --- /dev/null +++ b/apps/crawler/src/writers/account.writer.service.ts @@ -0,0 +1,58 @@ +import { normalizeSubstrateAddress, normalizeTimestamp } from '@common/utils'; +import { Account } from '@entities/Account'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { AllBalances } from '@unique-nft/substrate-client/types'; +import { Repository } from 'typeorm'; + +@Injectable() +export class AccountWriterService { + constructor( + @InjectRepository(Account) + private accountsRepository: Repository, + ) {} + + /** + * Prepares event and balances data to write into db. + */ + private prepareDataForDb(params: { + timestamp: number; + blockNumber: number; + balances: AllBalances; + }): Account { + const { + blockNumber, + timestamp, + balances: { address, availableBalance, lockedBalance, freeBalance }, + } = params; + + return { + block_height: String(blockNumber), + timestamp: String(timestamp), + account_id: address, + account_id_normalized: normalizeSubstrateAddress(address), + available_balance: availableBalance.amount, + free_balance: freeBalance.amount, + locked_balance: lockedBalance.amount, + }; + } + + async upsert({ + blockNumber, + blockTimestamp, + balances, + }: { + blockNumber: number; + blockTimestamp: number; + balances: AllBalances; + }) { + const dataToWrite = this.prepareDataForDb({ + blockNumber, + timestamp: normalizeTimestamp(blockTimestamp), + balances, + }); + + // Write data into db + return this.accountsRepository.upsert(dataToWrite, ['account_id']); + } +} diff --git a/apps/crawler/src/writers/block.writer.service.ts b/apps/crawler/src/writers/block.writer.service.ts new file mode 100644 index 00000000..eceed79e --- /dev/null +++ b/apps/crawler/src/writers/block.writer.service.ts @@ -0,0 +1,108 @@ +import { EventName } from '@common/constants'; +import { normalizeTimestamp } from '@common/utils'; +import { Block } from '@entities/Block'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { SubstrateBlock } from '@subsquid/substrate-processor'; +import { Repository } from 'typeorm'; +import { + IBlockItem, + IItemCounts, +} from '../subscribers/blocks.subscriber.service'; + +@Injectable() +export class BlockWriterService { + constructor( + @InjectRepository(Block) + private blocksRepository: Repository, + ) {} + + private collectItemCounts(blockItems: IBlockItem[]) { + const itemCounts = { + totalEvents: 0, + totalExtrinsics: 0, + numTransfers: 0, + newAccounts: 0, + }; + + blockItems.forEach((item) => { + const { kind } = item; + + if (kind === 'event') { + // Event + itemCounts.totalEvents += 1; + + const { + event: { name }, + } = item; + + if (name === EventName.BALANCES_TRANSFER) { + itemCounts.numTransfers += 1; + } else if (name === EventName.BALANCES_ENDOWED) { + itemCounts.newAccounts += 1; + } + } else { + // Extrinsic + itemCounts.totalExtrinsics += 1; + } + }); + + return itemCounts; + } + + private prepareDataForDb({ + block, + itemCounts, + }: { + block: SubstrateBlock; + itemCounts: IItemCounts; + }): Block { + const { + specId, + parentHash, + stateRoot, + extrinsicsRoot, + hash: blockHash, + height: blockNumber, + timestamp: blockTimestamp, + } = block; + + const [specName, specVersion] = specId.split('@') as [string, number]; + + const { totalEvents, totalExtrinsics, numTransfers, newAccounts } = + itemCounts; + + return { + block_number: blockNumber, + block_hash: blockHash, + parent_hash: parentHash, + extrinsics_root: extrinsicsRoot, + state_root: stateRoot, + spec_name: specName, + spec_version: specVersion, + timestamp: String(normalizeTimestamp(blockTimestamp)), + + // Item counts + total_events: totalEvents, + num_transfers: numTransfers, + new_accounts: newAccounts, + total_extrinsics: totalExtrinsics, + }; + } + + async upsert({ + block, + blockItems, + }: { + block: SubstrateBlock; + blockItems: IBlockItem[]; + }): Promise { + const itemCounts = this.collectItemCounts(blockItems); + + const blockData = this.prepareDataForDb({ block, itemCounts }); + + await this.blocksRepository.upsert(blockData, ['block_number']); + + return itemCounts; + } +} diff --git a/apps/crawler/src/subscribers/collections-subscriber.service.ts b/apps/crawler/src/writers/collection.writer.service.ts similarity index 53% rename from apps/crawler/src/subscribers/collections-subscriber.service.ts rename to apps/crawler/src/writers/collection.writer.service.ts index 7904c52d..a047b364 100644 --- a/apps/crawler/src/subscribers/collections-subscriber.service.ts +++ b/apps/crawler/src/writers/collection.writer.service.ts @@ -1,26 +1,21 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { Repository } from 'typeorm'; -import { InjectRepository } from '@nestjs/typeorm'; -import { EventHandlerContext } from '@subsquid/substrate-processor'; -import { Store } from '@subsquid/typeorm-store'; -import { Collections } from '@entities/Collections'; -import { Tokens } from '@entities/Tokens'; import { EventName } from '@common/constants'; import { normalizeSubstrateAddress, normalizeTimestamp, sanitizePropertiesValues, } from '@common/utils'; +import { Collections } from '@entities/Collections'; +import { Tokens } from '@entities/Tokens'; +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; import { CollectionInfoWithSchema, CollectionLimits, CollectionProperty, UniqueCollectionSchemaDecoded, + PropertyKeyPermission, } from '@unique-nft/substrate-client/tokens'; -import { SdkService } from '../sdk/sdk.service'; -import { ProcessorService } from './processor.service'; -import ISubscriberService from './subscriber.interface'; -import { InjectSentry, SentryService } from '@ntegral/nestjs-sentry'; +import { Repository } from 'typeorm'; type ParsedSchemaFields = { collectionCover?: string; @@ -30,62 +25,22 @@ type ParsedSchemaFields = { variableOnChainSchema?: object; }; +export interface ICollectionData { + collectionDecoded: CollectionInfoWithSchema | null; + collectionLimits: CollectionLimits | null; + tokenPropertyPermissions: PropertyKeyPermission[]; +} + @Injectable() -export class CollectionsSubscriberService implements ISubscriberService { - private readonly logger = new Logger(CollectionsSubscriberService.name); +export class CollectionWriterService { + private readonly logger = new Logger(CollectionWriterService.name); constructor( @InjectRepository(Collections) private collectionsRepository: Repository, @InjectRepository(Tokens) private tokensRepository: Repository, - private processorService: ProcessorService, - private sdkService: SdkService, - @InjectSentry() private readonly sentry: SentryService, - ) { - this.sentry.setContext(CollectionsSubscriberService.name); - } - - subscribe() { - // todo: Remove some items when models rework is done - const EVENTS_TO_UPDATE_COLLECTION = [ - // Insert - EventName.COLLECTION_CREATED, - - // Update - EventName.COLLECTION_PROPERTY_SET, - EventName.COLLECTION_PROPERTY_DELETED, - EventName.PROPERTY_PERMISSION_SET, - EventName.COLLECTION_SPONSOR_REMOVED, - EventName.COLLECTION_OWNED_CHANGED, - EventName.SPONSORSHIP_CONFIRMED, - // EventName.ALLOW_LIST_ADDRESS_ADDED, // todo: Too many events. Do we really need to process this event? - EventName.ALLOW_LIST_ADDRESS_REMOVED, - EventName.COLLECTION_LIMIT_SET, - EventName.COLLECTION_SPONSOR_SET, - ]; - - EVENTS_TO_UPDATE_COLLECTION.forEach((eventName) => - this.processorService.processor.addEventHandler( - eventName, - this.upsertHandler.bind(this), - ), - ); - - this.processorService.processor.addEventHandler( - EventName.COLLECTION_DESTROYED, - this.destroyHandler.bind(this), - ); - } - - private async getCollectionData( - collectionId: number, - ): Promise<[CollectionInfoWithSchema | null, CollectionLimits | null]> { - return Promise.all([ - this.sdkService.getCollection(collectionId), - this.sdkService.getCollectionLimits(collectionId), - ]); - } + ) {} private processJsonStringifiedValue(rawValue) { let result: object | null = null; @@ -178,10 +133,10 @@ export class CollectionsSubscriberService implements ISubscriberService { return result as UniqueCollectionSchemaDecoded; } - prepareDataToWrite( - collectionInfo: CollectionInfoWithSchema, - collectionLimits: CollectionLimits, - ) { + private prepareDataForDb(collectionData: ICollectionData): Collections { + const { collectionDecoded, collectionLimits, tokenPropertyPermissions } = + collectionData; + const { id: collection_id, owner, @@ -191,9 +146,9 @@ export class CollectionsSubscriberService implements ISubscriberService { tokenPrefix: token_prefix, mode, schema, - permissions: { mintMode: mint_mode }, + permissions, properties = [], - } = collectionInfo; + } = collectionDecoded; let schemaFromProperties = {} as UniqueCollectionSchemaDecoded; if (!schema) { @@ -231,6 +186,7 @@ export class CollectionsSubscriberService implements ISubscriberService { offchain_schema: offchainSchema, token_limit: token_limit || 0, properties: sanitizePropertiesValues(properties), + token_property_permissions: tokenPropertyPermissions, attributes_schema: attributesSchema, const_chain_schema: constOnChainSchema, variable_on_chain_schema: variableOnChainSchema, @@ -243,105 +199,44 @@ export class CollectionsSubscriberService implements ISubscriberService { schema_version: schemaVersion, token_prefix, mode, - mint_mode, + mint_mode: permissions.mintMode, owner_normalized: normalizeSubstrateAddress(owner), collection_cover: collectionCover, }; } - private async upsertHandler(ctx: EventHandlerContext): Promise { - const { - block: { height: blockNumber, timestamp: blockTimestamp }, - event: { name: eventName, args }, - } = ctx; - - const log = { - eventName, - blockNumber, - blockTimestamp, - entity: null as null | object | string, - collectionId: null as null | number, - }; - - try { - const collectionId = this.getCollectionIdFromArgs(args); - - log.collectionId = collectionId; - - const [collectionInfo, collectionLimits] = await this.getCollectionData( - collectionId, - ); - - if (collectionInfo) { - const dataToWrite = this.prepareDataToWrite( - collectionInfo, - collectionLimits, - ); - - // console.log('dataToWrite', dataToWrite); - - // Do not log the full entity because this object is quite big - log.entity = dataToWrite.name; - - await this.collectionsRepository.upsert( - { - ...dataToWrite, - date_of_creation: - eventName === EventName.COLLECTION_CREATED - ? normalizeTimestamp(blockTimestamp) - : undefined, - }, - ['collection_id'], - ); - } else { - // No entity returned from sdk. Most likely it was destroyed in a future block. - log.entity = null; - - await this.deleteCollection(collectionId); - } - - this.logger.verbose({ ...log }); - } catch (err) { - this.logger.error({ ...log, error: err.message }); - } + async upsert({ + eventName, + blockTimestamp, + collectionData, + }: { + eventName: string; + blockTimestamp: number; + collectionData: ICollectionData; + }) { + const preparedData = this.prepareDataForDb(collectionData); + + return this.collectionsRepository.upsert( + { + ...preparedData, + date_of_creation: + eventName === EventName.COLLECTION_CREATED + ? normalizeTimestamp(blockTimestamp) + : undefined, + }, + ['collection_id'], + ); } - private async destroyHandler(ctx: EventHandlerContext): Promise { - const { - block: { height: blockNumber, timestamp: blockTimestamp }, - event: { name: eventName, args }, - } = ctx; - - const log = { - eventName, - blockNumber, - blockTimestamp, - collectionId: null as null | number, - }; - - try { - const collectionId = this.getCollectionIdFromArgs(args); - - log.collectionId = collectionId; - - await this.deleteCollection(collectionId); - - this.logger.verbose({ ...log }); - } catch (error) { - this.logger.error({ ...log, error: error.message }); - this.sentry.instance().captureException({ ...log, error }); - } + async delete(collectionId: number) { + return this.deleteCollectionWithTokens(collectionId); } // Delete db collection record and related tokens - private deleteCollection(collectionId) { + private async deleteCollectionWithTokens(collectionId: number) { return Promise.all([ this.collectionsRepository.delete(collectionId), this.tokensRepository.delete({ collection_id: collectionId }), ]); } - - private getCollectionIdFromArgs(args): number { - return typeof args === 'number' ? args : (args[0] as number); - } } diff --git a/apps/crawler/src/writers/event.writer.service.ts b/apps/crawler/src/writers/event.writer.service.ts new file mode 100644 index 00000000..9f9a16ea --- /dev/null +++ b/apps/crawler/src/writers/event.writer.service.ts @@ -0,0 +1,91 @@ +import { Repository } from 'typeorm'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { EventMethod, EventSection } from '@common/constants'; +import { Event } from '@entities/Event'; +import { getAmount, normalizeTimestamp } from '@common/utils'; +import { + IBlockCommonData, + IBlockItem, + IEvent, +} from '../subscribers/blocks.subscriber.service'; + +@Injectable() +export class EventWriterService { + constructor( + @InjectRepository(Event) + private eventsRepository: Repository, + ) {} + + static extractEventItems(blockItems: IBlockItem[]): IEvent[] { + return blockItems + .map((item) => { + if (item.kind == 'event') { + return item.event as IEvent; + } + return null; + }) + .filter((v) => !!v); + } + + static extractRawAmountValue( + args: string | { amount?: string; value?: string }, + ) { + return typeof args === 'string' ? args : args?.amount || args?.value; + } + + private prepareDataForDb({ + eventItems, + blockCommonData, + }: { + eventItems: IEvent[]; + blockCommonData: IBlockCommonData; + }): Event[] { + return eventItems.map((event) => { + const { name, indexInBlock, phase, extrinsic, args: rawArgs } = event; + const { blockNumber, blockTimestamp } = blockCommonData; + + const [section, method] = name.split('.') as [EventSection, EventMethod]; + + const rawAmount = EventWriterService.extractRawAmountValue(rawArgs); + + const args = typeof rawArgs === 'object' ? rawArgs : [rawArgs]; + + return { + timestamp: String(normalizeTimestamp(blockTimestamp)), + block_number: String(blockNumber), + event_index: indexInBlock, + block_index: `${blockNumber}-${ + extrinsic ? extrinsic.indexInBlock : '' + }`, + section, + method, + // todo: Make more clean connect to extrinsic + phase: + phase === 'ApplyExtrinsic' ? String(extrinsic.indexInBlock) : phase, + data: JSON.stringify(args), + amount: rawAmount ? getAmount(rawAmount) : null, + }; + }); + } + + async upsert({ + blockItems, + blockCommonData, + }: { + blockItems: IBlockItem[]; + blockCommonData: IBlockCommonData; + }) { + const eventItems = EventWriterService.extractEventItems(blockItems); + + const eventsData = this.prepareDataForDb({ + blockCommonData, + eventItems, + }); + + return this.eventsRepository.upsert(eventsData, [ + 'block_number', + 'event_index', + ]); + } +} diff --git a/apps/crawler/src/writers/extrinsic.writer.service.ts b/apps/crawler/src/writers/extrinsic.writer.service.ts new file mode 100644 index 00000000..a05141fb --- /dev/null +++ b/apps/crawler/src/writers/extrinsic.writer.service.ts @@ -0,0 +1,186 @@ +import { + EventName, + ExtrinsicMethod, + ExtrinsicSection, +} from '@common/constants'; +import { + getAmount, + normalizeSubstrateAddress, + normalizeTimestamp, +} from '@common/utils'; +import { Extrinsic } from '@entities/Extrinsic'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { SubstrateExtrinsic } from '@subsquid/substrate-processor'; +import { Repository } from 'typeorm'; +import { + IBlockCommonData, + IBlockItem, +} from '../subscribers/blocks.subscriber.service'; +import { EventWriterService } from './event.writer.service'; + +const EXTRINSICS_TRANSFER_METHODS = [ + ExtrinsicMethod.TRANSFER, + ExtrinsicMethod.TRANSFER_FROM, + ExtrinsicMethod.TRANSFER_ALL, + ExtrinsicMethod.TRANSFER_KEEP_ALIVE, + ExtrinsicMethod.VESTED_TRANSFER, +]; + +export interface IExtrinsicExtended extends SubstrateExtrinsic { + name: string; +} + +interface IExtrinsicRecipient { + // Unique.transfer + recipient?: { value: string }; + + // Balances.transfer + // Balances.transfer_all + // Balances.transfer_keep_alive + // Vesting.vested_transfer + dest?: { value: string }; +} + +const EVENT_NAMES_WITH_AMOUNTS = [ + EventName.BALANCES_TRANSFER, + EventName.TREASURY_DEPOSIT, +]; + +@Injectable() +export class ExtrinsicWriterService { + constructor( + @InjectRepository(Extrinsic) + private extrinsicsRepository: Repository, + ) {} + + static extractExtrinsicItems(items: IBlockItem[]): IExtrinsicExtended[] { + return items + .map((item) => { + const { kind } = item; + if (kind === 'call') { + const { name, extrinsic } = item; + return { name, ...extrinsic } as IExtrinsicExtended; + } + return null; + }) + .filter((v) => !!v); + } + + private getAmountValues(blockItems: IBlockItem[]) { + const eventItems = EventWriterService.extractEventItems(blockItems); + + // Save 'amount' and 'fee' for extrinsic's events + return eventItems.reduce((acc, curr) => { + const { name, extrinsic, args } = curr; + + const extrinsicId = extrinsic?.id; + if (!extrinsicId || !EVENT_NAMES_WITH_AMOUNTS.includes(name)) { + return acc; + } + + const rawAmount = EventWriterService.extractRawAmountValue(args); + const amount = getAmount(rawAmount); + + acc[extrinsicId] = acc[extrinsicId] || {}; + + if (name === EventName.BALANCES_TRANSFER) { + acc[extrinsicId].amount = amount; + } else if (name === EventName.TREASURY_DEPOSIT) { + acc[extrinsicId].fee = amount; + } + + return { ...acc }; + }, {}); + } + + private prepareDataForDb({ + extrinsicItems, + amountValues, + blockCommonData, + }: { + extrinsicItems: IExtrinsicExtended[]; + amountValues: { + [key: string]: { amount?: string; fee?: string }; + }; + blockCommonData: IBlockCommonData; + }): Extrinsic[] { + return extrinsicItems.map((extrinsic) => { + const { name } = extrinsic; + const [section, method] = name.split('.') as [ + ExtrinsicSection, + ExtrinsicMethod, + ]; + + const { blockTimestamp, blockNumber, ss58Prefix } = blockCommonData; + + let signer = null; + const { signature } = extrinsic; + if (signature) { + const { + address: { value: rawSigner }, + } = signature; + + signer = normalizeSubstrateAddress(rawSigner, ss58Prefix); + } + + const { + call: { args }, + } = extrinsic; + + let toOwner = null; + if (EXTRINSICS_TRANSFER_METHODS.includes(method)) { + const recipientAddress = args as IExtrinsicRecipient; + const rawToOwner = + recipientAddress?.recipient?.value || recipientAddress?.dest?.value; + toOwner = normalizeSubstrateAddress(rawToOwner, ss58Prefix); + } + + const { id, hash, indexInBlock, success } = extrinsic; + + const { amount = '0', fee = '0' } = amountValues[id] || {}; + + return { + timestamp: String(normalizeTimestamp(blockTimestamp)), + block_number: String(blockNumber), + block_index: `${blockNumber}-${indexInBlock}`, + extrinsic_index: indexInBlock, + section, + method, + hash, + success, + is_signed: !!signature, + signer, + signer_normalized: signer && normalizeSubstrateAddress(signer), + to_owner: toOwner, + to_owner_normalized: toOwner && normalizeSubstrateAddress(toOwner), + amount, + fee, + }; + }); + } + + async upsert({ + blockItems, + blockCommonData, + }: { + blockItems: IBlockItem[]; + blockCommonData: IBlockCommonData; + }) { + const extrinsicItems = + ExtrinsicWriterService.extractExtrinsicItems(blockItems); + + const amountValues = this.getAmountValues(blockItems); + + const extrinsicsData = this.prepareDataForDb({ + blockCommonData, + extrinsicItems, + amountValues, + }); + + return this.extrinsicsRepository.upsert(extrinsicsData, [ + 'block_number', + 'extrinsic_index', + ]); + } +} diff --git a/apps/crawler/src/writers/token.writer.service.ts b/apps/crawler/src/writers/token.writer.service.ts new file mode 100644 index 00000000..485b9479 --- /dev/null +++ b/apps/crawler/src/writers/token.writer.service.ts @@ -0,0 +1,94 @@ +import { EventName } from '@common/constants'; +import { + normalizeSubstrateAddress, + normalizeTimestamp, + sanitizePropertiesValues, +} from '@common/utils'; +import { Tokens } from '@entities/Tokens'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { + CollectionInfoWithSchema, + TokenByIdResult, + TokenPropertiesResult, +} from '@unique-nft/substrate-client/tokens'; +import { Repository } from 'typeorm'; + +export interface ITokenData { + tokenDecoded: TokenByIdResult | null; + tokenProperties: TokenPropertiesResult | null; + collectionDecoded: CollectionInfoWithSchema | null; +} +@Injectable() +export class TokenWriterService { + constructor( + @InjectRepository(Tokens) + private tokensRepository: Repository, + ) {} + + prepareDataForDb(tokenData: ITokenData): Omit { + const { tokenDecoded, tokenProperties, collectionDecoded } = tokenData; + const { + tokenId: token_id, + collectionId: collection_id, + image, + attributes, + nestingParentToken, + owner, + } = tokenDecoded; + + const { owner: collectionOwner, tokenPrefix } = collectionDecoded; + + let parentId = null; + if (nestingParentToken) { + const { collectionId, tokenId } = nestingParentToken; + parentId = `${collectionId}_${tokenId}`; + } + + return { + token_id, + collection_id, + owner, + owner_normalized: normalizeSubstrateAddress(owner), + image, + attributes, + properties: tokenProperties + ? sanitizePropertiesValues(tokenProperties.properties) + : [], + parent_id: parentId, + is_sold: owner !== collectionOwner, + token_name: `${tokenPrefix} #${token_id}`, + }; + } + + async upsert({ + eventName, + blockTimestamp, + tokenData, + }: { + eventName: string; + blockTimestamp: number; + tokenData: ITokenData; + }) { + const preparedData = this.prepareDataForDb(tokenData); + + // Write token data into db + return this.tokensRepository.upsert( + { + ...preparedData, + date_of_creation: + eventName === EventName.ITEM_CREATED + ? normalizeTimestamp(blockTimestamp) + : undefined, + }, + ['collection_id', 'token_id'], + ); + } + + async delete(collectionId: number, tokenId: number) { + return this.tokensRepository.delete({ + collection_id: collectionId, + token_id: tokenId, + }); + } +} diff --git a/apps/crawler/src/writers/writers.module.ts b/apps/crawler/src/writers/writers.module.ts new file mode 100644 index 00000000..3d8bb33d --- /dev/null +++ b/apps/crawler/src/writers/writers.module.ts @@ -0,0 +1,46 @@ +import { Block } from '@entities/Block'; +import { Collections } from '@entities/Collections'; +import { Tokens } from '@entities/Tokens'; +import { Event } from '@entities/Event'; +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { Extrinsic } from '@entities/Extrinsic'; +import { Account } from '@entities/Account'; +import { CollectionWriterService } from './collection.writer.service'; +import { AccountWriterService } from './account.writer.service'; +import { TokenWriterService } from './token.writer.service'; +import { BlockWriterService } from './block.writer.service'; +import { EventWriterService } from './event.writer.service'; +import { ExtrinsicWriterService } from './extrinsic.writer.service'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ + Account, + Block, + Collections, + Event, + Extrinsic, + Tokens, + ]), + ConfigModule, + ], + providers: [ + AccountWriterService, + BlockWriterService, + CollectionWriterService, + EventWriterService, + ExtrinsicWriterService, + TokenWriterService, + ], + exports: [ + AccountWriterService, + BlockWriterService, + CollectionWriterService, + EventWriterService, + ExtrinsicWriterService, + TokenWriterService, + ], +}) +export class WritersModule {} diff --git a/apps/scraper/src/main.ts b/apps/scraper/src/main.ts deleted file mode 100644 index 6f2e581c..00000000 --- a/apps/scraper/src/main.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { NestFactory } from '@nestjs/core'; -import { ScraperModule } from './scraper.module'; - -async function bootstrap() { - await NestFactory.create(ScraperModule); -} -bootstrap(); diff --git a/apps/scraper/src/scraper.controller.spec.ts b/apps/scraper/src/scraper.controller.spec.ts deleted file mode 100644 index 47596413..00000000 --- a/apps/scraper/src/scraper.controller.spec.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { ScraperController } from './scraper.controller'; -import { ScraperService } from './scraper.service'; - -describe('ScraperController', () => { - let scraperController: ScraperController; - - beforeEach(async () => { - const app: TestingModule = await Test.createTestingModule({ - controllers: [ScraperController], - providers: [ScraperService], - }).compile(); - - scraperController = app.get(ScraperController); - }); - - describe('root', () => { - it('should return "Hello World!"', () => { - expect(scraperController.getHello()).toBe('Hello World!'); - }); - }); -}); diff --git a/apps/scraper/src/scraper.controller.ts b/apps/scraper/src/scraper.controller.ts deleted file mode 100644 index 90be5804..00000000 --- a/apps/scraper/src/scraper.controller.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Controller, Get } from '@nestjs/common'; -import { ScraperService } from './scraper.service'; - -@Controller() -export class ScraperController { - constructor(private readonly scraperService: ScraperService) {} - - @Get() - getHello(): string { - return this.scraperService.getHello(); - } -} diff --git a/apps/scraper/src/scraper.module.ts b/apps/scraper/src/scraper.module.ts deleted file mode 100644 index 976793b5..00000000 --- a/apps/scraper/src/scraper.module.ts +++ /dev/null @@ -1,13 +0,0 @@ -import typeormConfig from '@common/typeorm.config'; -import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { ScraperController } from './scraper.controller'; -import { ScraperService } from './scraper.service'; - -@Module({ - imports: [ConfigModule.forRoot(), TypeOrmModule.forRoot(typeormConfig)], - controllers: [ScraperController], - providers: [ScraperService], -}) -export class ScraperModule {} diff --git a/apps/scraper/src/scraper.service.ts b/apps/scraper/src/scraper.service.ts deleted file mode 100644 index 57ff092d..00000000 --- a/apps/scraper/src/scraper.service.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -@Injectable() -export class ScraperService { - getHello(): string { - return 'Hello World!'; - } -} diff --git a/apps/scraper/test/jest-e2e.json b/apps/scraper/test/jest-e2e.json deleted file mode 100644 index e9d912f3..00000000 --- a/apps/scraper/test/jest-e2e.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "moduleFileExtensions": ["js", "json", "ts"], - "rootDir": ".", - "testEnvironment": "node", - "testRegex": ".e2e-spec.ts$", - "transform": { - "^.+\\.(t|j)s$": "ts-jest" - } -} diff --git a/apps/scraper/tsconfig.app.json b/apps/scraper/tsconfig.app.json deleted file mode 100644 index a022a8ac..00000000 --- a/apps/scraper/tsconfig.app.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "declaration": false, - "outDir": "../../dist/apps/scraper" - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "test", "**/*spec.ts"] -} diff --git a/apps/web-api/src/account/account.service.ts b/apps/web-api/src/account/account.service.ts index c1bdc0d7..dd8ccec4 100644 --- a/apps/web-api/src/account/account.service.ts +++ b/apps/web-api/src/account/account.service.ts @@ -34,9 +34,8 @@ export class AccountService extends BaseService { this.applyLimitOffset(qb, queryArgs); this.applyWhereCondition(qb, queryArgs); this.applyOrderCondition(qb, queryArgs); - const data = await qb.getRawMany(); - const count = await qb.getCount(); - return { data, count }; + + return this.getDataAndCount(qb, queryArgs); } public async statistic({ diff --git a/apps/web-api/src/block/block.dto.ts b/apps/web-api/src/block/block.dto.ts index 669210ad..611a5cb1 100644 --- a/apps/web-api/src/block/block.dto.ts +++ b/apps/web-api/src/block/block.dto.ts @@ -18,9 +18,6 @@ export class BlockDto implements Partial { @Field(() => String, { nullable: true }) state_root?: string | null; - @Field(() => Int, { nullable: true }) - session_length?: string | null; - @Field(() => String, { nullable: true }) spec_name?: string; @@ -36,15 +33,9 @@ export class BlockDto implements Partial { @Field(() => Int, { nullable: true }) new_accounts?: number; - @Field(() => String, { nullable: true }) - total_issuance?: string; - @Field(() => Int, { nullable: true }) timestamp?: string; - @Field(() => Boolean, { nullable: true }) - need_rescan?: boolean; - @Field(() => Int, { nullable: true }) total_extrinsics?: number; } diff --git a/apps/web-api/src/block/block.service.ts b/apps/web-api/src/block/block.service.ts index 853fc8d0..38cf82af 100644 --- a/apps/web-api/src/block/block.service.ts +++ b/apps/web-api/src/block/block.service.ts @@ -26,24 +26,19 @@ export class BlockService extends BaseService { 'parent_hash', 'extrinsics_root', 'state_root', - 'session_length', 'spec_name', 'spec_version', 'total_events', 'num_transfers', 'new_accounts', - 'total_issuance', 'timestamp', 'total_extrinsics', - 'need_rescan', ]); this.applyLimitOffset(qb, queryArgs); this.applyWhereCondition(qb, queryArgs); this.applyOrderCondition(qb, queryArgs); - const data = await qb.getRawMany(); - const count = await qb.getCount(); - return { data, count }; + return this.getDataAndCount(qb, queryArgs); } } diff --git a/apps/web-api/src/collection/collection.dto.ts b/apps/web-api/src/collection/collection.dto.ts index 749eed0c..fd09001d 100644 --- a/apps/web-api/src/collection/collection.dto.ts +++ b/apps/web-api/src/collection/collection.dto.ts @@ -89,4 +89,10 @@ export class CollectionDTO implements Partial { @Field(() => GraphQLJSON, { nullable: true }) attributes_schema?: object; + + @Field(() => GraphQLJSON, { nullable: true }) + properties?: object; + + @Field(() => GraphQLJSON, { nullable: true }) + token_property_permissions?: object; } diff --git a/apps/web-api/src/collection/collection.service.ts b/apps/web-api/src/collection/collection.service.ts index 018b8cbe..b04dbd1a 100644 --- a/apps/web-api/src/collection/collection.service.ts +++ b/apps/web-api/src/collection/collection.service.ts @@ -40,10 +40,7 @@ export class CollectionService extends BaseService { const qb = this.repo.createQueryBuilder(); this.applyFilters(qb, queryArgs); - const data = await qb.getRawMany(); - const count = await this.getCountByFilters(qb, queryArgs); - - return { data, count }; + return this.getDataAndCount(qb, queryArgs); } public async findOne( @@ -141,6 +138,11 @@ export class CollectionService extends BaseService { qb.addSelect('Collections.schema_version', 'schema_version'); qb.addSelect('Collections.sponsorship', 'sponsorship'); qb.addSelect('Collections.const_chain_schema', 'const_chain_schema'); + qb.addSelect('Collections.properties', 'properties'); + qb.addSelect( + 'Collections.token_property_permissions', + 'token_property_permissions', + ); qb.addSelect( `COALESCE("Statistics".tokens_count, 0::bigint)`, 'tokens_count', diff --git a/apps/web-api/src/event/event.service.ts b/apps/web-api/src/event/event.service.ts index 4c0be34c..ad61b256 100644 --- a/apps/web-api/src/event/event.service.ts +++ b/apps/web-api/src/event/event.service.ts @@ -47,8 +47,7 @@ export class EventService extends BaseService { this.applyLimitOffset(qb, queryArgs); this.applyWhereCondition(qb, queryArgs); this.applyOrderCondition(qb, queryArgs); - const data = await qb.getRawMany(); - const count = await qb.getCount(); - return { data, count }; + + return this.getDataAndCount(qb, queryArgs); } } diff --git a/apps/web-api/src/extrinsic/extrinsic.service.ts b/apps/web-api/src/extrinsic/extrinsic.service.ts index d3a5bcda..35c0f815 100644 --- a/apps/web-api/src/extrinsic/extrinsic.service.ts +++ b/apps/web-api/src/extrinsic/extrinsic.service.ts @@ -51,9 +51,7 @@ export class ExtrinsicService extends BaseService { this.applyWhereCondition(qb, queryArgs); this.applyOrderCondition(qb, queryArgs); - const data = await qb.getRawMany(); - const count = await qb.getCount(); - return { data, count }; + return this.getDataAndCount(qb, queryArgs); } public async statistic({ diff --git a/apps/web-api/src/holder/holder.service.ts b/apps/web-api/src/holder/holder.service.ts index a3a870a2..5e01a26e 100644 --- a/apps/web-api/src/holder/holder.service.ts +++ b/apps/web-api/src/holder/holder.service.ts @@ -1,7 +1,7 @@ import { Tokens } from '@entities/Tokens'; import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; +import { Repository, SelectQueryBuilder } from 'typeorm'; import { BaseService } from '../utils/base.service'; import { IDataListResponse, IGQLQueryArgs } from '../utils/gql-query-args'; import { HolderDTO } from './holder.dto'; @@ -29,28 +29,30 @@ export class HolderService extends BaseService { qb.addGroupBy('owner_normalized'); this.applyWhereCondition(qb, queryArgs); this.applyOrderCondition(qb, queryArgs); - const { count } = await this.getCount(qb.getQuery(), qb.getParameters()); this.applyLimitOffset(qb, queryArgs); + const { count } = await this.getHandleCount(qb); const data = await qb.getRawMany(); return { data, count }; } - private async getCount( - queryString: string, - params: IQueryParameters, - ): Promise<{ count: number }> { - const query = this.replaceQueryParams(queryString, params); - const countQueryResult: [{ count?: string }] = await this.repo.query( - `select count(1) as "count" from (${query}) "t1"`, - ); + private async getHandleCount(qb: SelectQueryBuilder) { + const query = qb + .clone() + .distinctOn([]) + .orderBy() + .offset(undefined) + .limit(undefined) + .skip(undefined) + .take(undefined); - if (countQueryResult.length === 1) { - const count = Number(countQueryResult[0].count); - return { count }; - } + const qs = this.replaceQueryParams(query.getQuery(), query.getParameters()); + + const result: [{ count?: number }] = await this.repo.query( + `select count(1) as "count" from (${qs}) "t1"`, + ); - return { count: 0 }; + return { count: result.length ? Number(result[0].count ?? 0) : 0 }; } private replaceQueryParams(queryString: string, params: IQueryParameters) { diff --git a/apps/web-api/src/statistics/statistics.service.ts b/apps/web-api/src/statistics/statistics.service.ts index e22b038f..9aafe11a 100644 --- a/apps/web-api/src/statistics/statistics.service.ts +++ b/apps/web-api/src/statistics/statistics.service.ts @@ -25,9 +25,6 @@ export class StatisticsService extends BaseService { this.applyWhereCondition(qb, queryArgs); this.applyOrderCondition(qb, queryArgs); - const data = await qb.getRawMany(); - const count = await qb.getCount(); - - return { data, count }; + return this.getDataAndCount(qb, queryArgs); } } diff --git a/apps/web-api/src/tokens/token.service.ts b/apps/web-api/src/tokens/token.service.ts index be0ee843..8d3f3394 100644 --- a/apps/web-api/src/tokens/token.service.ts +++ b/apps/web-api/src/tokens/token.service.ts @@ -40,10 +40,7 @@ export class TokenService extends BaseService { const qb = this.repo.createQueryBuilder(); this.applyFilters(qb, queryArgs); - const data = await qb.getRawMany(); - const count = await this.getCountByFilters(qb, queryArgs); - - return { data, count }; + return this.getDataAndCount(qb, queryArgs); } public getByCollectionId(id: number, queryArgs: IGQLQueryArgs) { @@ -96,10 +93,10 @@ export class TokenService extends BaseService { queryArgs: IGQLQueryArgs, ): void { this.select(qb); + this.applyDistinctOn(qb, queryArgs); this.applyLimitOffset(qb, queryArgs); this.applyWhereCondition(qb, queryArgs); this.applyOrderCondition(qb, queryArgs); - this.applyDistinctOn(qb, queryArgs); } private select(qb: SelectQueryBuilder): void { diff --git a/apps/web-api/src/transaction/transaction.service.ts b/apps/web-api/src/transaction/transaction.service.ts index e77588ca..990e1d10 100644 --- a/apps/web-api/src/transaction/transaction.service.ts +++ b/apps/web-api/src/transaction/transaction.service.ts @@ -77,9 +77,6 @@ export class TransactionService extends BaseService { method: EventMethod.TRANSFER, }); - const count = await qb.getCount(); - const data = await qb.getRawMany(); - - return { data, count }; + return this.getDataAndCount(qb, queryArgs); } } diff --git a/apps/web-api/src/transfer/transfer.service.ts b/apps/web-api/src/transfer/transfer.service.ts index 5d9cd8ba..051e4d8e 100644 --- a/apps/web-api/src/transfer/transfer.service.ts +++ b/apps/web-api/src/transfer/transfer.service.ts @@ -25,8 +25,6 @@ export class TransferService extends BaseService { this.applyOrderCondition(qb, queryArgs); qb.andWhere({ method: 'Transfer' }); - const data = await qb.getRawMany(); - const count = await qb.getCount(); - return { data, count }; + return this.getDataAndCount(qb, queryArgs); } } diff --git a/apps/web-api/src/utils/base.service.ts b/apps/web-api/src/utils/base.service.ts index 57460433..b2cf707e 100644 --- a/apps/web-api/src/utils/base.service.ts +++ b/apps/web-api/src/utils/base.service.ts @@ -48,20 +48,44 @@ export class BaseService { args: IGQLQueryArgs, ): void { if (args.distinct_on) { - qb.distinctOn([args.distinct_on]); + qb.distinctOn([this.getConditionField(qb, args.distinct_on)]); + + // if order_by: {[args.distinct_on]: undefined | null} condition + // order_by required args.distinct_on in condition + // and he should be first order + if ( + args.order_by && + !args.order_by[args.distinct_on] && + Object.keys(args.order_by).length + ) { + const { order } = GQLToORMOrderByOperatorsMap.desc; + qb.orderBy(); + qb.addOrderBy(this.getConditionField(qb, args.distinct_on), order); + this.applyOrderCondition(qb, args); + } } } - protected async getCountByFilters( + protected async getCount( qb: SelectQueryBuilder, args: IGQLQueryArgs, ): Promise { if (args.distinct_on) { - qb.distinctOn([]); - const { count } = (await qb - .select(`COUNT(DISTINCT(${qb.alias}.${args.distinct_on}))`, 'count') - .getRawOne()) as { count: number }; + const query = qb.clone(); + + query + .distinctOn([]) + .orderBy() + .offset(undefined) + .limit(undefined) + .skip(undefined) + .take(undefined) + .select( + `COUNT(DISTINCT(${this.getConditionField(qb, args.distinct_on)}))`, + 'count', + ); + const { count } = (await query.getRawOne()) as { count: number }; return count; } @@ -106,6 +130,19 @@ export class BaseService { } } + protected async getDataAndCount( + qb: SelectQueryBuilder, + args: IGQLQueryArgs, + ) { + const data = await qb.getRawMany(); + let count = 0; + if (data?.length) { + count = await this.getCount(qb, args); + } + + return { data, count }; + } + private applyConditionTree( qb: SelectQueryBuilder, where: TWhere, diff --git a/apps/web-api/test/account.e2e-spec.ts b/apps/web-api/test/account.e2e-spec.ts index aac21ce5..7e73ff6e 100644 --- a/apps/web-api/test/account.e2e-spec.ts +++ b/apps/web-api/test/account.e2e-spec.ts @@ -85,7 +85,6 @@ describe('Account (e2e)', () => { '226.064588', ); expect(res.body.data.accounts.data[0].locked_balance).toBe('0'); - expect(res.body.data.accounts.data[0].nonce).toBe('132'); expect(res.body.data.accounts.data[0].timestamp).toBe(1653962324); expect(res.body.data.accounts.data[3].account_id).toBe( @@ -104,7 +103,6 @@ describe('Account (e2e)', () => { expect(res.body.data.accounts.data[3].locked_balance).toBe( '56675997171744160000', ); - expect(res.body.data.accounts.data[3].nonce).toBe('8'); expect(res.body.data.accounts.data[3].timestamp).toBe(1653962322); }); }); @@ -138,7 +136,6 @@ describe('Account (e2e)', () => { '226.064588', ); expect(res.body.data.accounts.data[0].locked_balance).toBe('0'); - expect(res.body.data.accounts.data[0].nonce).toBe('132'); expect(res.body.data.accounts.data[0].timestamp).toBe(1653962324); }); }); @@ -169,7 +166,6 @@ describe('Account (e2e)', () => { expect(res.body.data.accounts.data[0].block_height).toBe(123); expect(res.body.data.accounts.data[0].free_balance).toBe('1'); expect(res.body.data.accounts.data[0].locked_balance).toBe('1'); - expect(res.body.data.accounts.data[0].nonce).toBe('test_text'); expect(res.body.data.accounts.data[0].timestamp).toBe(1653962321); }); }); @@ -216,7 +212,6 @@ describe('Account (e2e)', () => { '226.064588', ); expect(res.body.data.accounts.data[0].locked_balance).toBe('0'); - expect(res.body.data.accounts.data[0].nonce).toBe('132'); expect(res.body.data.accounts.data[0].timestamp).toBe(1653962324); expect(res.body.data.accounts.data[1].account_id).toBe( @@ -229,7 +224,6 @@ describe('Account (e2e)', () => { expect(res.body.data.accounts.data[1].block_height).toBe(123); expect(res.body.data.accounts.data[1].free_balance).toBe('1'); expect(res.body.data.accounts.data[1].locked_balance).toBe('1'); - expect(res.body.data.accounts.data[1].nonce).toBe('test_text'); expect(res.body.data.accounts.data[1].timestamp).toBe(1653962321); }); }); diff --git a/apps/web-api/test/block.e2e-spec.ts b/apps/web-api/test/block.e2e-spec.ts index 430244af..419c1031 100644 --- a/apps/web-api/test/block.e2e-spec.ts +++ b/apps/web-api/test/block.e2e-spec.ts @@ -49,15 +49,12 @@ describe('Block (e2e)', () => { parent_hash extrinsics_root state_root - session_length spec_name spec_version total_events num_transfers new_accounts - total_issuance timestamp - need_rescan total_extrinsics } } @@ -80,7 +77,6 @@ describe('Block (e2e)', () => { expect(res.body.data.block.data.length).toBe(10); expect(res.body.data.block.data[0].block_number).toBe(214569); - expect(res.body.data.block.data[0].session_length).toBe(0); expect(res.body.data.block.data[0].spec_name).toBe('quartz'); expect(res.body.data.block.data[0].spec_version).toBe(914000); expect(res.body.data.block.data[0].total_events).toBe(2); @@ -88,29 +84,24 @@ describe('Block (e2e)', () => { expect(res.body.data.block.data[0].new_accounts).toBe(0); expect(res.body.data.block.data[0].timestamp).toBe(1640875392); - expect(res.body.data.block.data[0].need_rescan).toBe(true); expect(res.body.data.block.data[0].total_extrinsics).toBe(2); expect(res.body.data.block.data[3].block_number).toBe(12393); - expect(res.body.data.block.data[3].session_length).toBe(0); expect(res.body.data.block.data[3].spec_name).toBe('quartz'); expect(res.body.data.block.data[3].spec_version).toBe(1); expect(res.body.data.block.data[3].total_events).toBe(2); expect(res.body.data.block.data[3].num_transfers).toBe(0); expect(res.body.data.block.data[3].new_accounts).toBe(0); expect(res.body.data.block.data[3].timestamp).toBe(1638217818); - expect(res.body.data.block.data[3].need_rescan).toBe(true); expect(res.body.data.block.data[3].total_extrinsics).toBe(2); expect(res.body.data.block.data[7].block_number).toBe(12385); - expect(res.body.data.block.data[7].session_length).toBe(0); expect(res.body.data.block.data[7].spec_name).toBe('quartz'); expect(res.body.data.block.data[7].spec_version).toBe(1); expect(res.body.data.block.data[7].total_events).toBe(2); expect(res.body.data.block.data[7].num_transfers).toBe(0); expect(res.body.data.block.data[7].new_accounts).toBe(0); expect(res.body.data.block.data[7].timestamp).toBe(1638217722); - expect(res.body.data.block.data[7].need_rescan).toBe(true); expect(res.body.data.block.data[7].total_extrinsics).toBe(2); }); }); @@ -131,14 +122,12 @@ describe('Block (e2e)', () => { expect(res.body.data.block.data.length).toBe(1); expect(res.body.data.block.data[0].block_number).toBe(214569); - expect(res.body.data.block.data[0].session_length).toBe(0); expect(res.body.data.block.data[0].spec_name).toBe('quartz'); expect(res.body.data.block.data[0].spec_version).toBe(914000); expect(res.body.data.block.data[0].total_events).toBe(2); expect(res.body.data.block.data[0].num_transfers).toBe(0); expect(res.body.data.block.data[0].new_accounts).toBe(0); expect(res.body.data.block.data[0].timestamp).toBe(1640875392); - expect(res.body.data.block.data[0].need_rescan).toBe(true); expect(res.body.data.block.data[0].total_extrinsics).toBe(2); }); }); @@ -159,14 +148,12 @@ describe('Block (e2e)', () => { expect(res.body.data.block.data.length).toBe(1); expect(res.body.data.block.data[0].block_number).toBe(12222); - expect(res.body.data.block.data[0].session_length).toBe(0); expect(res.body.data.block.data[0].spec_name).toBe('quartz'); expect(res.body.data.block.data[0].spec_version).toBe(1); expect(res.body.data.block.data[0].total_events).toBe(2); expect(res.body.data.block.data[0].num_transfers).toBe(0); expect(res.body.data.block.data[0].new_accounts).toBe(0); expect(res.body.data.block.data[0].timestamp).toBe(1638215562); - expect(res.body.data.block.data[0].need_rescan).toBe(true); expect(res.body.data.block.data[0].total_extrinsics).toBe(2); }); }); @@ -243,14 +230,12 @@ describe('Block (e2e)', () => { expect(res.body.data.block.count).toBe(1); expect(res.body.data.block.data[0].block_number).toBe(12231); - expect(res.body.data.block.data[0].session_length).toBe(0); expect(res.body.data.block.data[0].spec_name).toBe('quartz'); expect(res.body.data.block.data[0].spec_version).toBe(1); expect(res.body.data.block.data[0].total_events).toBe(2); expect(res.body.data.block.data[0].num_transfers).toBe(0); expect(res.body.data.block.data[0].new_accounts).toBe(0); expect(res.body.data.block.data[0].timestamp).toBe(1638215670); - expect(res.body.data.block.data[0].need_rescan).toBe(true); expect(res.body.data.block.data[0].total_extrinsics).toBe(2); }); }); diff --git a/apps/web-api/test/fixtures/block.yml b/apps/web-api/test/fixtures/block.yml index 25d89a62..2031006c 100644 --- a/apps/web-api/test/fixtures/block.yml +++ b/apps/web-api/test/fixtures/block.yml @@ -6,15 +6,12 @@ items: parent_hash: '0x62e16486d85b7ad15e1170f43385b1f73a6bbae68913ea74cbe8af5d2baf2d5d' extrinsics_root: '0xcd0374e0bb41df6197b654c4400f76a1ef0c0015fcd1f8d2b85a667d6fddd148' state_root: '0xa13654c58f26cdf6b2d05695fc50720c4ce7859416e56506b358613cb7284884' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215562 - need_rescan: true total_extrinsics: 2 block12224: block_number: 12224 @@ -22,15 +19,12 @@ items: parent_hash: '0xc0c9c0aa5653117ccf8dc6c6e09199a0ee5b1f53f64760d4f578616c1ecb30ae' extrinsics_root: '0x35926abf8488dfbcf8352dd6e7a4e7a1f829b13726c9cd617f79cf8309f991e3' state_root: '0x58f2dd38619b8567fb7b95795764b4c11d79f35d295a7c1b65ec05e777d47694' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215586 - need_rescan: true total_extrinsics: 2 block12226: block_number: 12226 @@ -38,15 +32,12 @@ items: parent_hash: '0xe72b0d06f02b8fdeade988083e529b9f64e95f5f872cbb6189cf0d1ba942a4a4' extrinsics_root: '0x89682c7a7569389050867708bd3e456a5e80ebccd7e13545eb1c2ddc9097625d' state_root: '0xac90f915b82aa3a612db21a69e10b5983e994ee867f8ec8aa7bf35f85e7c2bfc' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215610 - need_rescan: true total_extrinsics: 2 block12228: block_number: 12228 @@ -54,15 +45,12 @@ items: parent_hash: '0xf77f4e265dac39ed4edca8efe2d6f83f24f30a67a7448a747faa1da015c5c727' extrinsics_root: '0x22d1e0a245bc38e2066353a9c075875d920e9e436221bab8eca19ad66dec7789' state_root: '0x8e89f73ae8944cb279053e7a1aebb37c04dc63160d39f6a927458e708c300dc6' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215634 - need_rescan: true total_extrinsics: 2 block12230: block_number: 12230 @@ -70,15 +58,12 @@ items: parent_hash: '0x82f60cd8e9ca9ac23bce1eab3759ccec815e731c0c1a3cc5a7914fed3cee6df5' extrinsics_root: '0x7f570bad69079c9cea2902de4142c22b91208b839e0c1469e38ecfde78ee315d' state_root: '0x118cb0a3128639e21326810f72a70e29cd68ed5e56c9c892c7c59de16938e48b' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215658 - need_rescan: true total_extrinsics: 2 block12231: block_number: 12231 @@ -86,15 +71,12 @@ items: parent_hash: '0xbe3f99dadf968db4d7294884e2a3577b6df2e2d6040f9e4bdcd0df236e43c7c9' extrinsics_root: '0x53fb8f46e85bf860316f18e82f6136eb260f8fd78cb7e92bacecaacd8fc5bf24' state_root: '0xe19d69be30cda8e8a22a49471fcba033068c40fffdf4461ce2b64e8839e95094' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215670 - need_rescan: true total_extrinsics: 2 block12233: block_number: 12233 @@ -102,15 +84,12 @@ items: parent_hash: '0xc4795eb68ed5659012f2b3d18136ecdb0bc449d6a32eeb63333d69fb8c317b5e' extrinsics_root: '0x7c471abfd1523b3ae4b4e45dbf3cd929ca9a31420cc89cafefcc778e8e5ba7f5' state_root: '0xb1c90d844931a5d468e9d13ab00fbcd73fcade1ab4673fa4c1b67192fce971be' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215700 - need_rescan: true total_extrinsics: 2 block12235: block_number: 12235 @@ -118,15 +97,12 @@ items: parent_hash: '0x37d2f8cbac0740bb9e027248b1f435194a442ba7dcc0ffe4bb012449b76b7c5c' extrinsics_root: '0x0eb802f555ad15fa908b76140dfc71c579d79cf0ee8610e531c99c2cfd7f444f' state_root: '0xa8e50dfe54c8430834fd2d8bea7267cecee429435aa8218b160828df16e881b0' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215724 - need_rescan: true total_extrinsics: 2 block12237: block_number: 12237 @@ -134,15 +110,12 @@ items: parent_hash: '0x08e6df24ea4f6c65f0d8979ca61041d3b714d90a5a09b94eefadeb536eca6431' extrinsics_root: '0x367da8cc1f26a498358f8aefc5d187a499c2944592f69df98b10900c98e44b2e' state_root: '0xa33650158b88f1337a1b62f361cf628c0e7f4303da570c46e22f7362aea66af5' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215754 - need_rescan: true total_extrinsics: 2 block12239: block_number: 12239 @@ -150,15 +123,12 @@ items: parent_hash: '0x256b98186c82421397f70652d650e8de4a1459459228310796c8ea748c82496a' extrinsics_root: '0xcfc98d552cb597f830d4a7bfa2f350f3e604d5ea05d011210796fc0d913500e6' state_root: '0x12e5afcc7bdb67772a9e971642d14515f2375c1d4dca20acd7a350a4ba26db7c' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215778 - need_rescan: true total_extrinsics: 2 block12241: block_number: 12241 @@ -166,15 +136,12 @@ items: parent_hash: '0x6e600811657dcfb0b5a03c37532434d41a90916ed2154e90e1d54d12988a38dc' extrinsics_root: '0xdc204555137eed83369ecb5652c12d0e6d84730cdc5bbb8ecb6d5ca85d72c41d' state_root: '0x39d94e62429b14429b15e0699a71f777d0b0740b801d06e2bf2b8e328a9eb115' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215802 - need_rescan: true total_extrinsics: 2 block12243: block_number: 12243 @@ -182,15 +149,12 @@ items: parent_hash: '0x4637eede6329c4284c02070e07bf1e5291ae9326e460926bf5f53e802c715224' extrinsics_root: '0x3fc5d01a1eea82a287227192685b8842f1bea01d020e3071443bbced7e971938' state_root: '0x5575bbedce089562b4c69fcdb7b9e495a3bdfabdc2838f8e75a5226aba4ec783' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215826 - need_rescan: true total_extrinsics: 2 block12245: block_number: 12245 @@ -198,15 +162,12 @@ items: parent_hash: '0x1601ba89114e8f906b6cc5271480c6506fdc3b8ecef742026fc5d2a829af9169' extrinsics_root: '0xa8636d64b46f18a680e0f0166c2a4c52e9fe2fa5c0e58a862b03eedc6e3e4cd2' state_root: '0x39ec056603cc4141b575ba6fb6818f50e91d5013aaba35475c8dab4145764226' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215850 - need_rescan: true total_extrinsics: 2 block214568: block_number: 214568 @@ -214,15 +175,12 @@ items: parent_hash: '0xd6849bb2b9d0b3434c983f0ce7dc8e0cac066fea67572b0a932436ef846bbc35' extrinsics_root: '0x5ee4c80c7076aebeb727b3260bff5356493a998e43b9f12d9413d980999f508d' state_root: '0xffae8288dbffb1e66eb815b6efcbce1a1cff3f7419c5459b7b7be8d0395d0d47' - session_length: 0 spec_name: quartz spec_version: 914000 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1005482112360290544921600000' timestamp: 1640875380 - need_rescan: true total_extrinsics: 2 block12248: block_number: 12248 @@ -230,15 +188,12 @@ items: parent_hash: '0xff64613eb8db741ddf298d5ac467066478b278ba40c14fbb916fd6da37387100' extrinsics_root: '0xca0d17ce0bcbf8b1e6c6ce1e06a838935b3982a426d615f38fc023af2f4db866' state_root: '0x03e42630311bfdd2ae21945787f105c2cff91e2ca00b9c586a35db88332156db' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215886 - need_rescan: true total_extrinsics: 2 block12236: block_number: 12236 @@ -246,15 +201,12 @@ items: parent_hash: '0x04fd53fe0edf666395c0eaaf61065ebfdff8a765ab11beaae0d31af06b37dc0a' extrinsics_root: '0x984632d404e6c5f80674de2b023c106d17bae1e338e4d348bca4d4c84ac42a3f' state_root: '0x63985b13f510a25ed60c0fb9fd5512fb525d9c82bf46c022cb13010bf62f5c37' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215742 - need_rescan: true total_extrinsics: 2 block12238: block_number: 12238 @@ -262,15 +214,12 @@ items: parent_hash: '0x8799bc7b09b959ba8151f179d21b2c91f49b54fe6bbd3640ab8f3fe0ecc16e62' extrinsics_root: '0x9cb7d329d257767e2fb667c658a5831f7b1bd953eecf7c612a03a037a3be16da' state_root: '0xb45c5a3f8084c44a11a7e0e75fe556fdec1b2baf12d8ff3eaddf887835b42326' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215766 - need_rescan: true total_extrinsics: 2 block12240: block_number: 12240 @@ -278,15 +227,12 @@ items: parent_hash: '0x8670b9b0e36293db5e6d8cb315ad8189236ba3ea6ddd6288002afb9039b2c490' extrinsics_root: '0x6cf049296bb81a5cc115ecd7a43e4245769d80f368cccb04cf2928e2b0206210' state_root: '0x7cd26c02a1be957c260a37f9373bca3ec3adc0ab70fdfccaf663ea67817b9b73' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215790 - need_rescan: true total_extrinsics: 2 block12242: block_number: 12242 @@ -294,15 +240,12 @@ items: parent_hash: '0x1888d06c85a929783e219b10980a6cdd16688c3af9db51f8bd88b5bab0aaaf25' extrinsics_root: '0x8314768c0e88023b02c1ccedcc0097d43171f54c970d046ed81111c312c0748e' state_root: '0xf272e2cb364d087f6e5c52d91fcc0ee8f1df2e374f8b6c2fdef58447c59b5917' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215814 - need_rescan: true total_extrinsics: 2 block12244: block_number: 12244 @@ -310,15 +253,12 @@ items: parent_hash: '0x6c1b250b0be18d63b25405f742771f808b5991a645e43f1e3a61adcb5e9c19dc' extrinsics_root: '0x587d93d07f15ae733da2127524d31b4a9fa4c8c246f93de2e57395b458897cff' state_root: '0xd206100628836b2cdda4892d49371ff8f55bfb35581ad1337755f6b01b6a392a' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215838 - need_rescan: true total_extrinsics: 2 block12246: block_number: 12246 @@ -326,15 +266,12 @@ items: parent_hash: '0xa61801f84db57418c72a97cb3c686202bcdec7c0d6b460c794acd429b0d980ce' extrinsics_root: '0x845e00208ecb3c9f9dd96c207c217da23f4809b7e05ffa33a9d1a117709e96f2' state_root: '0x6fbdb1dee2783a7b47fff588b3a6ede06a1b77e81bb1a822d358f7db789858bc' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215862 - need_rescan: true total_extrinsics: 2 block12247: block_number: 12247 @@ -342,15 +279,12 @@ items: parent_hash: '0x6a8f3d45723987e744e70b7d0124b8c0bb0eaf2e0d2ecbcb23112c98a6adcec7' extrinsics_root: '0x032ef50d1e8d42af7f8136af716ecb5448deef96e370a8ecf2d40f6726669810' state_root: '0x82258ee5fef3cd8eaca2b65ea2c1de554ea1d0350c6894641b96c56c2ad94cb1' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215874 - need_rescan: true total_extrinsics: 2 block12249: block_number: 12249 @@ -358,15 +292,12 @@ items: parent_hash: '0xe00db4738cf23f9b884ed7bcc4f2839285e4ffff893b52cbdb9839c9e2a734cd' extrinsics_root: '0xa3daf1ef4606a18cbd7f1c6f674d03aa12137bfb68db80102f689db3f9bbc5d8' state_root: '0xb0c1c800904fdd43443ea2bf5e5668815befd0fa2298c476c5a43934ac510efb' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215898 - need_rescan: true total_extrinsics: 2 block12251: block_number: 12251 @@ -374,15 +305,12 @@ items: parent_hash: '0xbef9559ee48a5aea2e5730dadc26d0b91e123460a085220dd69808fc7f56a323' extrinsics_root: '0x65a6c76bbc9edbd14dafe61a27bc72fa5b220bf15f8a0260a9591aaacb7d424b' state_root: '0x1f8844a347712cd78204fbf778a6030e1d9474dc3f5297ed48f4e6e35757f720' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215922 - need_rescan: true total_extrinsics: 2 block12253: block_number: 12253 @@ -390,15 +318,12 @@ items: parent_hash: '0x40f7cd66c61f95557de28ab965f611b55849e7edccea545ecd44c66392685bae' extrinsics_root: '0xa04c77cc062a6c3eeb138fd662dfe6f4e2b64fafe9099bb239720a52162f5fb2' state_root: '0xc3f5f19d06e4ca10d788688bc7a23aab216ae22e8b5fb72ab6f84ad405d10a3a' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215958 - need_rescan: true total_extrinsics: 2 block12255: block_number: 12255 @@ -406,15 +331,12 @@ items: parent_hash: '0x31744c0eda0f8414ba5ea76936d5ab745f370671cc008a7d99c228d90a7ea9df' extrinsics_root: '0x5192d627478c40e6ab63f071a282ccf8459a93bb017fc85ceb2221b509e79492' state_root: '0xa54d9652736aa2db74616d73d0741ec8cb5c522908fd87da1ca7b288118dc912' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215982 - need_rescan: true total_extrinsics: 2 block12257: block_number: 12257 @@ -422,15 +344,12 @@ items: parent_hash: '0xc4c29c9c79dafde09cae1f9f2190c36ec570bf289e59db9ff8edc248f4e7622b' extrinsics_root: '0x8e77835ec6d1441493f3cb36a125f4df8926c5e5d8896da6aefefac694816f57' state_root: '0x0dc9a02e93fe20a180aeb091760c027240a6e5da58266e99112dee41f43f29ee' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638216006 - need_rescan: true total_extrinsics: 2 block12259: block_number: 12259 @@ -438,15 +357,12 @@ items: parent_hash: '0x961bef4e28fd6ba44318c86a7dcd407717577d75f5b4500d821d279f909feba6' extrinsics_root: '0x7fb174adaadc4d29943216f2f0b806dd6fc0af30ba3d024a34e123ee8f343fb3' state_root: '0x9f6b17fdff2a686efea1020ee80f05e58c4c9376610c0786830d11acc18ef48f' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638216042 - need_rescan: true total_extrinsics: 2 block12261: block_number: 12261 @@ -454,15 +370,12 @@ items: parent_hash: '0x9a64047b10c53aa55fa87272422600b824a83159bdd744d8bdac656f1144c5e2' extrinsics_root: '0xfdcfe5b6245c60683cff7732909cfc1edafa1e2786ff22e2262504231b8eef8f' state_root: '0xb4a72ed87be03ee6fb657cebba1ef362e41b55b45fe915fc44806b311da867b0' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638216066 - need_rescan: true total_extrinsics: 2 block214569: block_number: 214569 @@ -470,15 +383,12 @@ items: parent_hash: '0xd57f0ce2e6cd50867713ba73c0806d0a704d14682a92f2e49eb900d7b9f0dc61' extrinsics_root: '0x94caf8510aea1565359444c6e9365d683e957c0fe733821856f141f9396cfc72' state_root: '0x98c775547c57c05a5f76881a4e2702e1de7dcfc2a1525fa12c51c6ec41b79799' - session_length: 0 spec_name: quartz spec_version: 914000 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1005482112360290544921600000' timestamp: 1640875392 - need_rescan: true total_extrinsics: 2 block12264: block_number: 12264 @@ -486,15 +396,12 @@ items: parent_hash: '0x6f9edb44b269a1e56d2597294ae438489c1e2c83eca0cd8bf427c81fec3ccbab' extrinsics_root: '0xab330f18f666bc115f2d038f7ae444f7429096e0043a307ce8bdec8a87e97205' state_root: '0x0d8bc3a82a2aa1bc33910e6ac18185572a648e775d8a38c8219abfaef6371312' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638216102 - need_rescan: true total_extrinsics: 2 block12266: block_number: 12266 @@ -502,15 +409,12 @@ items: parent_hash: '0x04a71fdfe0c8b8d178a00930b9e79ad00147419f8ad8cc3b19f7b33c6ef76b26' extrinsics_root: '0x8b2e927c86cef68102fae88f329244de03afdf37b04c2f786c1a347cf2317a90' state_root: '0x9e19c08f8bb13d37acaaa399c044b8b00c9f44531b79930f5f973accc18a5d89' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638216126 - need_rescan: true total_extrinsics: 2 block12268: block_number: 12268 @@ -518,15 +422,12 @@ items: parent_hash: '0x95b9b80778f904585b36b7e8b3854405daa0055c4058bb24b445d62d900e8c5a' extrinsics_root: '0x6b17c0a388d085e32a455b6d79a3283fbc213a46e73245e042e07c15681ebad8' state_root: '0xdf838de0409acdfa5d001f3def856f661d107ff87541401b4aee8777deda5b31' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638216150 - need_rescan: true total_extrinsics: 2 block12250: block_number: 12250 @@ -534,15 +435,12 @@ items: parent_hash: '0xa294c9b1676edf92bfcecd04958fd055b3aa0d32ff52133fb0b5e39676d94091' extrinsics_root: '0x83f9df9b1d4bc6fe01561ce031b4805da799e778644372642ca3628bd828585b' state_root: '0xb69aa497cdf6a104903f2f053a14943be418925e135c36009ccba370d0df1e1c' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215910 - need_rescan: true total_extrinsics: 2 block12252: block_number: 12252 @@ -550,15 +448,12 @@ items: parent_hash: '0x873aba13517fe225a08ab77ab5db4620805c4ae3ac702eb5d1a37e1f1eca2d99' extrinsics_root: '0xdba8fe0cd55c313f96585e7a51df5b3b55a69aa62af52d855379bffa3bdd12cc' state_root: '0x75322d9ca54e7bb605fd47762323f88e785eb614fa33928ce717f34d22db9c0d' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215946 - need_rescan: true total_extrinsics: 2 block12254: block_number: 12254 @@ -566,15 +461,12 @@ items: parent_hash: '0x02dd4da448d1291391f7e52b25d45dd8abfc494333c8537587a878b68930fcc5' extrinsics_root: '0x60c24798f33e0bd047cfca6a2d64467d97c4e87667a01e8d83f18729e93be6fb' state_root: '0x391c3716d665d8ff860ae2f58e8e23f39f1c5a7c19cdc081d89b01516b3ca56d' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215970 - need_rescan: true total_extrinsics: 2 block12256: block_number: 12256 @@ -582,15 +474,12 @@ items: parent_hash: '0xe23485b2bb08af218339d80949f256457e17f9a1c8e9e34204e88e7cadf76ed4' extrinsics_root: '0x57e814374766cc7a9d910d0caeb9f3615232592aebabba8a19c306bbb611fad3' state_root: '0x78cb93bb35f933a929ffbbbaa1105efb02a62f1789f9b726b623378f4b303c5e' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638215994 - need_rescan: true total_extrinsics: 2 block12379: block_number: 12379 @@ -598,15 +487,12 @@ items: parent_hash: '0x3e49dfab46f5dc27e49e619e1ff641fe552b189f312828aee3e3e0a0ec5ee922' extrinsics_root: '0x41b00fdef6b7eef08af446dcdfc735abc66c0852ded57f3bf3c690b0d7821642' state_root: '0xea331a72a0f7c093c25164e5c5cfdca2afcc70af828ecf7cde528f532a2741ae' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638217626 - need_rescan: true total_extrinsics: 2 block12381: block_number: 12381 @@ -614,15 +500,12 @@ items: parent_hash: '0xb7cae7032ddee6e0883f37bdcf2af85c144a5a9ff9b3502a2f457375af90ddba' extrinsics_root: '0xef080d3933ddc5911ec864986f07917c7be3139ab68347d97331d195af505b93' state_root: '0xae4ca4cdefc64b3e1244b094364e1ce8747c5e809f8e4fcf13632de3332c47ae' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638217650 - need_rescan: true total_extrinsics: 2 block12383: block_number: 12383 @@ -630,15 +513,12 @@ items: parent_hash: '0x24181024222f7d0f525de5025705c662a09e033805522ebc24c7510f34e57fe5' extrinsics_root: '0xcb020a6399d5a9996296b0a7ec04daec5079a1b0de8a96e2b5c85799af054b8f' state_root: '0x88a28c8689619575b66739bec86c2dbdda7f5aebdf79ff076898754d11a88d83' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638217674 - need_rescan: true total_extrinsics: 2 block12385: block_number: 12385 @@ -646,15 +526,12 @@ items: parent_hash: '0x7141d1d5506bac5c1690fc047ee508d41603b20dbd5151af4535254dd363855b' extrinsics_root: '0x5c91990beffbefc2edb9006232705b1fbfebbce0fb5d340b709f610271fa3658' state_root: '0xf8190557e92fb55ef5e72864ab652a4d15354837f8d355112ed3719f0dde8e76' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638217722 - need_rescan: true total_extrinsics: 2 block12387: block_number: 12387 @@ -662,15 +539,12 @@ items: parent_hash: '0x20c98a2873a7e6aa4b8c19c7195ceb406365e0feaa8d2d1d8277bf9544bf9da8' extrinsics_root: '0x61c716766bdd84bf3a548b5b9019ad95a8273cd5908ba7c873ce75f265b11061' state_root: '0x76fc160ab924c6ea83c41f249db9de86a3faf4b096060be1ada314c331bd2b12' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638217746 - need_rescan: true total_extrinsics: 2 block12389: block_number: 12389 @@ -678,15 +552,12 @@ items: parent_hash: '0xde55e169de8114ecd28aeace626d3cd55b713d398911fd3504eb1db7837c3ab8' extrinsics_root: '0xfc299020bdead491bc2cc3101cd435f5c9b1835ff4e04403ac3cf769ee14b8c5' state_root: '0x2b5adbe869dcfcc34ffa19162feeabb2148d318738bf88ee1a09be360dae7ed2' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638217770 - need_rescan: true total_extrinsics: 2 block12391: block_number: 12391 @@ -694,15 +565,12 @@ items: parent_hash: '0xb685ab1edcd7768f360c73b732a3a8fd55fcbcb8842e7e37951bbef04ca9fb02' extrinsics_root: '0x4f9bda80377b049007e54f9472cad9fc7955877d1d4bc39a14d1a4d5d45c58d9' state_root: '0x44974d3296c68849d2b20596a323f6e02174c2313f406cbfbafbdb3187ec0ce0' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638217794 - need_rescan: true total_extrinsics: 2 block12393: block_number: 12393 @@ -710,15 +578,12 @@ items: parent_hash: '0x5a9a42bd5662ba20748bf30ede058eace207bf2953adeb9f03b9741a2cc84de8' extrinsics_root: '0x6283ae7a32a3ed0e2b4ac499ebea54bb8613ee3a8f26fd8d0bb31e0145efb19d' state_root: '0xcd474517d540a418790a949cc428cf2850cd1810551f8b9b2f32b165498e5b5f' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638217818 - need_rescan: true total_extrinsics: 2 block12394: block_number: 12394 @@ -726,13 +591,10 @@ items: parent_hash: '0x47a316a36a13a05b00aacdd74506b14f36caaed3a48a05abcac946d323bead52' extrinsics_root: '0xf584ebd543ee6c1d7788457427112ca7cc066d60ddd6a55c9cbe0a3dd75d76da' state_root: '0xe425e0c650150d09dd6e4ee3e49d20d3541ffdbd412dd515af2675a43ae3720d' - session_length: 0 spec_name: quartz spec_version: 1 total_events: 2 num_transfers: 0 new_accounts: 0 - total_issuance: '1000000000000000000000000' timestamp: 1638217830 - need_rescan: true total_extrinsics: 2 diff --git a/apps/web-api/test/fixtures/event.yml b/apps/web-api/test/fixtures/event.yml index 98b5d806..e7cee46a 100644 --- a/apps/web-api/test/fixtures/event.yml +++ b/apps/web-api/test/fixtures/event.yml @@ -3,7 +3,7 @@ items: event{1..55}: block_number: 213826($current) event_index: ($current) - section: 'balances' + section: 'Balances' method: 'Transfer' phase: 'Initialization($current)' data: '' diff --git a/apps/web-api/test/fixtures/tokens.yml b/apps/web-api/test/fixtures/tokens.yml index f5368232..052c1d97 100644 --- a/apps/web-api/test/fixtures/tokens.yml +++ b/apps/web-api/test/fixtures/tokens.yml @@ -9,7 +9,6 @@ items: owner_normalized: '0x2303410dcc766995e70b47beedda828b4486320b($current)' image: { ipfsCid: '($current)' } # attributes: "{}" - token_prefix: 'token_prefix_($current)' token_name: 'token_name($current)' collection: '@collections($current)' tokens46: @@ -19,7 +18,6 @@ items: collection_id: 46 date_of_creation: 1650433764 owner_normalized: '0x2303410dcc766995e70b47beedda828b4486320b46' - token_prefix: 'token_prefix_46' token_name: 'token_name46' collection: '@collections46' tokens47: @@ -29,7 +27,6 @@ items: collection_id: 47 date_of_creation: 1650433764 owner_normalized: '0x2303410dcc766995e70b47beedda828b4486320b47' - token_prefix: 'token_prefix_47' token_name: 'token_name47' collection: '@collections47' parent_id: '46_46' @@ -40,7 +37,6 @@ items: collection_id: 48 date_of_creation: 1650433764 owner_normalized: '0x2303410dcc766995e70b47beedda828b4486320b48' - token_prefix: 'token_prefix_48' token_name: 'token_name48' collection: '@collections48' parent_id: '46_46' @@ -51,7 +47,6 @@ items: collection_id: 49 date_of_creation: 1650433764 owner_normalized: '0x2303410dcc766995e70b47beedda828b4486320b49' - token_prefix: 'token_prefix_49' token_name: 'token_name49' collection: '@collections49' parent_id: '46_46' @@ -62,7 +57,6 @@ items: collection_id: 50 date_of_creation: 1650433764 owner_normalized: '0x2303410dcc766995e70b47beedda828b4486320b50' - token_prefix: 'token_prefix_50' token_name: 'token_name50' collection: '@collections50' parent_id: '47_47' diff --git a/apps/web-api/test/tokens.e2e-spec.ts b/apps/web-api/test/tokens.e2e-spec.ts index 2594ea9b..aa877957 100644 --- a/apps/web-api/test/tokens.e2e-spec.ts +++ b/apps/web-api/test/tokens.e2e-spec.ts @@ -120,7 +120,7 @@ describe('Tokens (e2e)', () => { '0x2303410dcc766995e70b47beedda828b4486320b1', ); expect(res.body.data.tokens.data[0].token_id).toBe(1); - expect(res.body.data.tokens.data[0].token_name).toBe('testSTa1 #1'); + expect(res.body.data.tokens.data[0].token_name).toBe('token_name1'); expect(res.body.data.tokens.data[0].token_prefix).toBe('testSTa1'); expect(res.body.data.tokens.data[3].collection.collection_id).toBe(4); @@ -154,7 +154,7 @@ describe('Tokens (e2e)', () => { '0x2303410dcc766995e70b47beedda828b4486320b4', ); expect(res.body.data.tokens.data[3].token_id).toBe(4); - expect(res.body.data.tokens.data[3].token_name).toBe('testSTa4 #4'); + expect(res.body.data.tokens.data[3].token_name).toBe('token_name4'); expect(res.body.data.tokens.data[3].token_prefix).toBe('testSTa4'); }); }); diff --git a/common/constants.ts b/common/constants.ts index 473c3d72..521bf7e6 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -66,6 +66,9 @@ export const EventName = { BALANCES_ENDOWED: `${EventSection.BALANCES}.${EventMethod.ENDOWED}`, BALANCES_WITHDRAW: `${EventSection.BALANCES}.${EventMethod.WITHDRAW}`, BALANCES_TRANSFER: `${EventSection.BALANCES}.${EventMethod.TRANSFER}`, + + // Treasury + TREASURY_DEPOSIT: `${EventSection.TREASURY}.${EventMethod.DEPOSIT}`, }; export enum ExtrinsicSection { @@ -82,9 +85,20 @@ export enum ExtrinsicMethod { VESTED_TRANSFER = 'vested_transfer', } -export const ETHEREUM_ADDRESS_MAX_LENGTH = 42; - export const STATE_SCHEMA_NAME_BY_MODE = { SCAN: 'scan_status', RESCAN: 'rescan_status', }; + +export enum SubscriberName { + ACCOUNTS = 'account', + BLOCKS = 'blocks', + COLLECTIONS = 'collections', + TOKENS = 'tokens', +} + +export enum SubscriberAction { + UPSERT = 'UPSERT', + DELETE = 'DELETE', + DELETE_NOT_FOUND = 'DELETE: NOT FOUND', +} diff --git a/common/entities/Block.ts b/common/entities/Block.ts index 55a96f04..24a83788 100644 --- a/common/entities/Block.ts +++ b/common/entities/Block.ts @@ -2,7 +2,6 @@ import { Field, Int, ObjectType } from '@nestjs/graphql'; import { Column, Entity, Index } from 'typeorm'; @Index('block_pkey', ['block_number'], { unique: true }) -@Index('block_need_rescan_idx', ['need_rescan'], {}) @Entity('block', { schema: 'public' }) @ObjectType() export class Block { @@ -27,9 +26,6 @@ export class Block { @Column('text', { name: 'state_root', nullable: true }) state_root: string | null; - @Column('bigint', { name: 'session_length', nullable: true }) - session_length: string | null; - @Column('text', { name: 'spec_name' }) spec_name: string; @@ -45,15 +41,9 @@ export class Block { @Column('integer', { name: 'new_accounts' }) new_accounts: number; - @Column('text', { name: 'total_issuance' }) - total_issuance: string; - @Column('bigint', { name: 'timestamp' }) timestamp: string; - @Column('boolean', { name: 'need_rescan', default: false }) - need_rescan: boolean; - @Column('integer', { name: 'total_extrinsics' }) total_extrinsics: number; } diff --git a/common/entities/Collections.ts b/common/entities/Collections.ts index a2924e9d..cb8c4147 100644 --- a/common/entities/Collections.ts +++ b/common/entities/Collections.ts @@ -25,6 +25,9 @@ export class Collections { @Column('jsonb', { name: 'properties', default: [] }) properties: object | null; + @Column('jsonb', { name: 'token_property_permissions', default: [] }) + token_property_permissions: object; + @Column('jsonb', { name: 'attributes_schema', default: {} }) attributes_schema: object | null; diff --git a/common/typeorm.config.ts b/common/typeorm.config.ts index 0457d67e..d475830f 100644 --- a/common/typeorm.config.ts +++ b/common/typeorm.config.ts @@ -11,11 +11,12 @@ import { Tokens } from './entities/Tokens'; import { Total } from './entities/Total'; import { TokensStats } from './entities/TokensStats'; import { DataSourceOptions } from 'typeorm'; -import dotenv = require('dotenv'); -import path = require('path'); +import * as dotenv from 'dotenv'; +import * as path from 'path'; dotenv.config(); const migrationsDir = path.join(__dirname, '..', 'migrations'); +const isTestMode = process.env.NODE_ENV === 'test'; const typeormConfig: DataSourceOptions = { type: 'postgres', @@ -40,7 +41,7 @@ const typeormConfig: DataSourceOptions = { ], synchronize: false, migrationsRun: false, - migrations: [path.join(migrationsDir, '/**/*{.ts,.js}')], + migrations: isTestMode ? [] : [path.join(migrationsDir, '/**/*{.ts,.js}')], logging: process.env.LOGGING === '1', }; diff --git a/common/utils.ts b/common/utils.ts index 15e054c2..eade8ce7 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -1,14 +1,15 @@ import BigNumber from 'bignumber.js'; -import { encodeAddress, decodeAddress } from '@polkadot/util-crypto'; -import { ETHEREUM_ADDRESS_MAX_LENGTH } from './constants'; +import { + encodeAddress, + decodeAddress, + isEthereumAddress, +} from '@polkadot/util-crypto'; import { Prefix } from '@polkadot/util-crypto/types'; export function normalizeSubstrateAddress(address, ss58Format?: Prefix) { - if (address?.length <= ETHEREUM_ADDRESS_MAX_LENGTH) { - return address; - } - - return encodeAddress(decodeAddress(address), ss58Format); + return isEthereumAddress(address) + ? address + : encodeAddress(decodeAddress(address), ss58Format); } export function normalizeTimestamp(timestamp: number) { diff --git a/migrations/1662471315191-collections-properties-permissions.ts b/migrations/1662471315191-collections-properties-permissions.ts new file mode 100644 index 00000000..859a3c8d --- /dev/null +++ b/migrations/1662471315191-collections-properties-permissions.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class collectionsPropertiesPermissions1662471315191 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "collections" ADD "token_property_permissions" jsonb NOT NULL DEFAULT '[]'`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "collections" DROP COLUMN "token_property_permissions"`, + ); + } +} diff --git a/migrations/1662629432041-fix-holders-count-trigger.ts b/migrations/1662629432041-fix-holders-count-trigger.ts new file mode 100644 index 00000000..e56b8652 --- /dev/null +++ b/migrations/1662629432041-fix-holders-count-trigger.ts @@ -0,0 +1,109 @@ +/* eslint-disable max-len */ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +const calcStatsQuery = ` + update collections_stats + set holders_count = sub.holders_count + from ( + select stat.collection_id, count(1) as holders_count + from ( + select t.collection_id, count(1) + from tokens t + group by t."owner", t.collection_id + ) as stat + group by stat.collection_id + ) as sub + where collections_stats.collection_id = sub.collection_id + ; +`; + +const fnName = 'update_collections_stats_holders'; +const triggerName = 'collection_holders_stats'; + +const tokensUpdateHoldersStatsFn = ` +create or replace function ${fnName}() returns trigger as $$ + declare + hasAnotherTokens int; + newOwnerHasAnotherTokens int; + oldOwnerHasTokens int; + begin + if (TG_OP = 'INSERT') then + --Check owner already has token in collection + select token_id from tokens where collection_id = NEW.collection_id and token_id != NEW.token_id and "owner" = NEW.owner limit 1 into hasAnotherTokens; + if (hasAnotherTokens is null) then + insert into collections_stats (collection_id, tokens_count, holders_count, actions_count) + values (NEW.collection_id, 0, 1, 0) + ON CONFLICT (collection_id) + DO UPDATE SET holders_count = collections_stats.holders_count + 1; + end if; + end if; + + if (TG_OP = 'UPDATE' and NEW.owner != OLD.owner) then + select token_id from tokens where collection_id = NEW.collection_id and token_id != NEW.token_id and "owner" = NEW.owner limit 1 into newOwnerHasAnotherTokens; + select token_id from tokens where collection_id = OLD.collection_id and token_id != OLD.token_id and "owner" = OLD.owner limit 1 into oldOwnerHasTokens; + --Previous owner has another token in this collection but new owner not. + if (newOwnerHasAnotherTokens is null and oldOwnerHasTokens is not null) then + insert into collections_stats(collection_id, tokens_count, holders_count, actions_count) + values (NEW.collection_id, 0, 1, 0) + ON CONFLICT (collection_id) + DO UPDATE SET holders_count = collections_stats.holders_count + 1; + end if; + + --Previous owner hasn't another tokens any more + if (oldOwnerHasTokens is null and newOwnerHasAnotherTokens is not null) then + insert into collections_stats(collection_id, tokens_count, holders_count, actions_count) + values (NEW.collection_id, 0, 0, 0) + ON CONFLICT (collection_id) + DO UPDATE SET holders_count = collections_stats.holders_count - 1; + end if; + + end if; + + + if (TG_OP = 'DELETE') then + --Check owner has another token in this collection + select token_id from tokens where collection_id = OLD.collection_id and token_id != OLD.token_id and "owner" = OLD.owner limit 1 into hasAnotherTokens; + if (hasAnotherTokens is null) then + insert into collections_stats(collection_id, tokens_count, holders_count, actions_count) + values (OLD.collection_id, 0, 0, 0) + ON CONFLICT (collection_id) + DO UPDATE SET holders_count = collections_stats.holders_count - 1; + end if; + end if; + + return null; + end; +$$ LANGUAGE plpgsql; +`; + +const deleteHoldersStatsTrigger = `drop trigger if exists ${triggerName} on tokens;`; +const tokensHoldersStatsTrigger = ` + Create trigger ${triggerName} after insert or update or delete on tokens + FOR EACH row + execute function ${fnName}(); +`; + +export class fixHoldersCountTrigger1662629432041 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.startTransaction(); + + try { + await queryRunner.query(calcStatsQuery); + + await queryRunner.query(deleteHoldersStatsTrigger); + await queryRunner.query(tokensUpdateHoldersStatsFn); + await queryRunner.query(tokensHoldersStatsTrigger); + + await queryRunner.commitTransaction(); + } catch (err) { + await queryRunner.rollbackTransaction(); + throw err; + } + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('delete from collections_stats'); + await queryRunner.query(`drop function ${fnName};`); + await queryRunner.query(`drop trigger ${triggerName} on tokens;`); + } +} diff --git a/migrations/1663046775910-remove-block-deprecated-fields.ts b/migrations/1663046775910-remove-block-deprecated-fields.ts new file mode 100644 index 00000000..074f82b6 --- /dev/null +++ b/migrations/1663046775910-remove-block-deprecated-fields.ts @@ -0,0 +1,27 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class removeBlockDeprecatedFields1663046775910 + implements MigrationInterface +{ + name = 'removeBlockDeprecatedFields1663046775910'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."block_need_rescan_idx"`); + await queryRunner.query(`ALTER TABLE "block" DROP COLUMN "session_length"`); + await queryRunner.query(`ALTER TABLE "block" DROP COLUMN "total_issuance"`); + await queryRunner.query(`ALTER TABLE "block" DROP COLUMN "need_rescan"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "block" ADD "need_rescan" boolean NOT NULL DEFAULT false`, + ); + await queryRunner.query( + `ALTER TABLE "block" ADD "total_issuance" text NOT NULL`, + ); + await queryRunner.query(`ALTER TABLE "block" ADD "session_length" bigint`); + await queryRunner.query( + `CREATE INDEX "block_need_rescan_idx" ON "block" ("need_rescan") `, + ); + } +} diff --git a/nest-cli.json b/nest-cli.json index 7f6227ab..bcb81ec3 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -17,15 +17,6 @@ "tsConfigPath": "apps/web-api/tsconfig.app.json" } }, - "scraper": { - "type": "application", - "root": "apps/scraper", - "entryFile": "main", - "sourceRoot": "apps/scraper/src", - "compilerOptions": { - "tsConfigPath": "apps/scraper/tsconfig.app.json" - } - }, "crawler": { "type": "application", "root": "apps/crawler", @@ -36,4 +27,4 @@ } } } -} \ No newline at end of file +} diff --git a/package-lock.json b/package-lock.json index 3992a75e..16ef7402 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,13 +19,15 @@ "@ntegral/nestjs-sentry": "^3.0.7", "@polkadot/util-crypto": "^9.7.2", "@sentry/node": "^6.19.7", - "@subsquid/substrate-processor": "^1.5.1", - "@subsquid/typeorm-store": "^0.1.1", + "@subsquid/substrate-processor": "^1.8.0", + "@subsquid/typeorm-store": "^0.1.3", "@unique-nft/opal-testnet-types": "^0.5.2", "@unique-nft/quartz-mainnet-types": "^0.6.0", - "@unique-nft/substrate-client": "^0.5.1", + "@unique-nft/substrate-client": "^0.5.5", "apollo-server-express": "^3.6.2", "bignumber.js": "^9.0.2", + "cache-manager": "^4.1.0", + "cache-manager-redis-store": "^2.0.0", "graphql": "^15.8.0", "graphql-type-json": "^0.3.2", "lodash": "^4.17.21", @@ -40,6 +42,7 @@ "@nestjs/cli": "^8.0.0", "@nestjs/schematics": "^8.0.0", "@nestjs/testing": "^8.0.0", + "@types/cache-manager": "^4.0.1", "@types/chai": "^4.3.0", "@types/express": "^4.17.13", "@types/jest": "27.0.2", @@ -3685,9 +3688,9 @@ "integrity": "sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg==" }, "node_modules/@subsquid/logger": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/@subsquid/logger/-/logger-0.0.2.tgz", - "integrity": "sha512-Z7M2D1No25ZraTzuCibwC3TdnxiCD/UI/tLYxY5T5hXfRX9QKiqJ2Sj+EOBwx1wOD9bVreNFFF7LE15gQzAsfg==", + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@subsquid/logger/-/logger-0.1.0.tgz", + "integrity": "sha512-tiM9gUyaB9RJupfD8CskKSMyDb8xXFl6JhQm0r/xy00RhFO3EKOu2Jc7UvvCkCVuJImVwWLh/xdwqZ+d4SXY2g==", "dependencies": { "@subsquid/util-internal-hex": "^0.0.1", "@subsquid/util-internal-json": "^0.1.1", @@ -3718,47 +3721,47 @@ } }, "node_modules/@subsquid/scale-codec": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@subsquid/scale-codec/-/scale-codec-1.0.2.tgz", - "integrity": "sha512-beWD0WFjJBK9JmBSJrfHK3kzae55s4itcJlmZelo70rknHgsRW1oTXHihDvRdG5fKAfrw5dkc1ZmEz8d3Pr20Q==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@subsquid/scale-codec/-/scale-codec-1.0.3.tgz", + "integrity": "sha512-wUPtNrO9i6doOqoxH7IeoZ8wS/jSgH/VZraBCzno+Cmv/6pRf6svWiCuooPzEXFZ1BI1XDHoZ2NLBf7DFkdMkA==", "dependencies": { "@subsquid/util-internal-hex": "^0.0.1", "@subsquid/util-internal-json": "^0.1.1" } }, "node_modules/@subsquid/substrate-metadata": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@subsquid/substrate-metadata/-/substrate-metadata-1.0.2.tgz", - "integrity": "sha512-WAcWDsA/T5vYZzkNL7KSZI1K+0Bj3JBsxrL/y2vfw58p/AxuhoXsdfe0GTAylvOryo7MUPNsDMbw6v98PEb/Cw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@subsquid/substrate-metadata/-/substrate-metadata-1.1.0.tgz", + "integrity": "sha512-X19/x8SFcQ3Yyupld8bm2DsoukOUbhCUqEnwCb+a4NqutmoZhiKMnK8lsUyzND21SGEt3CQ7QRO7VB0nw3P5tA==", "dependencies": { - "@subsquid/scale-codec": "^1.0.2", + "@subsquid/scale-codec": "^1.0.3", "@subsquid/util-internal": "^0.0.1", "@subsquid/util-naming": "^0.0.1" } }, "node_modules/@subsquid/substrate-processor": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@subsquid/substrate-processor/-/substrate-processor-1.5.1.tgz", - "integrity": "sha512-rKHQVodVKy3GyeBQ6vFt/puC0xPPHieJwTAylXvQI4CE3mDIeOHagu0mQjj3yspTZ8jmk+dPmDNaXA/kRFdGwQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@subsquid/substrate-processor/-/substrate-processor-1.8.0.tgz", + "integrity": "sha512-8pXbXLyeX3ItxpGYRkNc8q7lH9Lj5B4Y79xmb81ky+hzUzSGG8qREoU5w6xnacLi19p004jveJ/vKu2YHkgfkg==", "dependencies": { - "@subsquid/logger": "^0.0.2", + "@subsquid/logger": "^0.1.0", "@subsquid/rpc-client": "^1.0.2", - "@subsquid/scale-codec": "^1.0.2", - "@subsquid/substrate-metadata": "^1.0.2", - "@subsquid/typeorm-config": "^1.1.0", + "@subsquid/scale-codec": "^1.0.3", + "@subsquid/substrate-metadata": "^1.1.0", + "@subsquid/typeorm-config": "^2.0.0", "@subsquid/util-internal": "^0.0.1", "@subsquid/util-internal-binary-heap": "^0.0.0", "@subsquid/util-internal-code-printer": "^0.0.2", "@subsquid/util-internal-counters": "^0.0.1", "@subsquid/util-internal-gql-request": "^0.1.0", "@subsquid/util-internal-hex": "^0.0.1", - "@subsquid/util-internal-prometheus-server": "^0.0.1", + "@subsquid/util-internal-prometheus-server": "^0.0.2", "@subsquid/util-xxhash": "^0.1.1", "blake2b": "^2.1.4", "prom-client": "^14.0.1" }, "peerDependencies": { - "@subsquid/typeorm-store": "^0.1.1" + "@subsquid/typeorm-store": "^0.1.3" }, "peerDependenciesMeta": { "@subsquid/typeorm-store": { @@ -3767,9 +3770,9 @@ } }, "node_modules/@subsquid/typeorm-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@subsquid/typeorm-config/-/typeorm-config-1.1.0.tgz", - "integrity": "sha512-7qIOlpodIvIhrm3pudu2jpzMXx3aCxj39FPMV38NPzbft5AEKTmfRJPPSzBDbsYnz+WFJECDkLEb2vnFUsfuDw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@subsquid/typeorm-config/-/typeorm-config-2.0.0.tgz", + "integrity": "sha512-TJ/ksxTLTEHqY6NiYM5S9JXFdUEROTjTY+PKQv3ixoQuuidNzIGRwwGvzMnSKNl9s1V5ADcwWIEbmcPBZjHURg==", "dependencies": { "@subsquid/util-naming": "^0.0.1" }, @@ -3783,11 +3786,11 @@ } }, "node_modules/@subsquid/typeorm-store": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@subsquid/typeorm-store/-/typeorm-store-0.1.1.tgz", - "integrity": "sha512-DOx6L/NFDj4XkTNOjcgcQQjHbMPKR7vmO/XHXUHQ27ikxUZrZ8w8y2pPMfFXN5aJUfWoHmGeLOO9BvZ0ZeN/vQ==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@subsquid/typeorm-store/-/typeorm-store-0.1.3.tgz", + "integrity": "sha512-agu6G4p9mNB0FmLi/uRx8+8k6AT9FQB12jqIRMzRtmobS2IzH88d3+SuTz86vHhuR4t6VHM7aarByTTHKZBX9w==", "dependencies": { - "@subsquid/typeorm-config": "^1.1.0", + "@subsquid/typeorm-config": "^2.0.0", "@subsquid/util-internal": "^0.0.1" }, "peerDependencies": { @@ -3828,9 +3831,9 @@ "integrity": "sha512-sNok0jQV6+OpAl3QKaH2VFh8PKZyZ6XHZhZ71LeirOhgfVprKFmEvFG9yQIp7qKe7JGXmolX54zu150OMP9f5w==" }, "node_modules/@subsquid/util-internal-http-server": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@subsquid/util-internal-http-server/-/util-internal-http-server-0.0.1.tgz", - "integrity": "sha512-12zcFZLeI6ss0XJeR+5p6oJZLjyHSGh3Lrs80T9YuQzverE5dP0zIvnEpx62iPYNSJGesP2Z5olAy9VduUsUzA==", + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@subsquid/util-internal-http-server/-/util-internal-http-server-0.1.0.tgz", + "integrity": "sha512-7+lQlGMpKg5Qeavkas2/uBgF4cQQKG5gblhsuPwt/cyDGfIQcwPX9KxJUVQDBE8SL2P62qg8TkzeEYSG89iykg==", "dependencies": { "stoppable": "^1.1.0" } @@ -3844,11 +3847,11 @@ } }, "node_modules/@subsquid/util-internal-prometheus-server": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@subsquid/util-internal-prometheus-server/-/util-internal-prometheus-server-0.0.1.tgz", - "integrity": "sha512-kEQ50rWVr56bMsiVYP6LgeQVtkpw+TW0o2rhs2M3E4U8Rt2P7XnUIJLLRMHN+AEco0OJ3m6PlQ0jlQylI7ZZSg==", + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@subsquid/util-internal-prometheus-server/-/util-internal-prometheus-server-0.0.2.tgz", + "integrity": "sha512-ODLBH03RkDm34AHkInQ6M5gcm5LCtI9wBAoOlKtqCcgTSIflELJKfmvqGCd3hhPyw0t6PJ2LQayrX3rth/nOqw==", "dependencies": { - "@subsquid/util-internal-http-server": "^0.0.1" + "@subsquid/util-internal-http-server": "^0.1.0" }, "peerDependencies": { "prom-client": "^14.0.1" @@ -4071,6 +4074,12 @@ "@types/node": "*" } }, + "node_modules/@types/cache-manager": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/cache-manager/-/cache-manager-4.0.1.tgz", + "integrity": "sha512-w4Gm7qg4ohvk0k4CLhOoqnMohWEyeyAOTovPgkguhuDCfVEV1wN/HWEd1XzB1S9/NV9pUcZcc498qU4E15ck6A==", + "dev": true + }, "node_modules/@types/chai": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.0.tgz", @@ -4498,9 +4507,9 @@ "dev": true }, "node_modules/@unique-nft/api": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/@unique-nft/api/-/api-0.1.12.tgz", - "integrity": "sha512-M52g/cjk/PRo33YnpeFALeHlfBVXiGs79+VdY4Bg4C/yHzGfQ/zJ2Amhoga7hCUoFY5zJg0o/3Qoo74+eCEq9Q==", + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@unique-nft/api/-/api-0.1.13.tgz", + "integrity": "sha512-Asee4NywIYOQ9PUzc5o2VOf2nKxG7iAAqcd5nH9CuyWZR+NV2HxO+u//DAUFugE0qCh5APAh7pkDJ1G5gFO/GA==", "dependencies": { "@noble/hashes": "^1.1.2", "@polkadot/api": "8.7.1", @@ -4938,12 +4947,12 @@ } }, "node_modules/@unique-nft/substrate-client": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@unique-nft/substrate-client/-/substrate-client-0.5.1.tgz", - "integrity": "sha512-Nj2qWxGNNBXkvuNVSZ1EHhTpP3lYh9sSWMMDjf4o2Ecl9lgz6m2zq1RBHYnzWaTUIBxgx2N+UBs/oYJ1ynFIWA==", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@unique-nft/substrate-client/-/substrate-client-0.5.5.tgz", + "integrity": "sha512-qiMPV5Z634QHCAQG/TCTnE8J0RETSenOOQgocfJoKRyRvJdSRwC9mjEpzIGjR2G4SIFFoWv+F9FR6J3EnrclOQ==", "dependencies": { "@polkadot/api": "^8.6.2", - "@unique-nft/api": "0.1.12", + "@unique-nft/api": "0.1.13", "protobufjs": "^6.11.2", "rxjs": "^7.0.0" } @@ -5976,6 +5985,40 @@ "node": ">= 0.8" } }, + "node_modules/cache-manager": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-4.1.0.tgz", + "integrity": "sha512-ZGM6dLxrP65bfOZmcviWMadUOCICqpLs92+P/S5tj8onz+k+tB7Gr+SAgOUHCQtfm2gYEQDHiKeul4+tYPOJ8A==", + "dependencies": { + "async": "3.2.3", + "lodash.clonedeep": "^4.5.0", + "lru-cache": "^7.10.1" + } + }, + "node_modules/cache-manager-redis-store": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cache-manager-redis-store/-/cache-manager-redis-store-2.0.0.tgz", + "integrity": "sha512-bWLWlUg6nCYHiJLCCYxY2MgvwvKnvlWwrbuynrzpjEIhfArD2GC9LtutIHFEPeyGVQN6C+WEw+P3r+BFBwhswg==", + "dependencies": { + "redis": "^3.0.2" + }, + "engines": { + "node": ">= 8.3" + } + }, + "node_modules/cache-manager/node_modules/async": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", + "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==" + }, + "node_modules/cache-manager/node_modules/lru-cache": { + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.14.0.tgz", + "integrity": "sha512-EIRtP1GrSJny0dqb50QXRUNBxHJhcpxHC++M5tD7RYbvLLn5KVWKsbyswSSqDuU15UFi3bgTQIY8nhDMeF6aDQ==", + "engines": { + "node": ">=12" + } + }, "node_modules/call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -6605,6 +6648,14 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", + "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -9769,6 +9820,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -10955,9 +11011,9 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "node_modules/prom-client": { - "version": "14.0.1", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.0.1.tgz", - "integrity": "sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.1.0.tgz", + "integrity": "sha512-iFWCchQmi4170omLpFXbzz62SQTmPhtBL35v0qGEVRHKcqIeiexaoYeP0vfZTujxEq3tA87iqOdRbC9svS1B9A==", "dependencies": { "tdigest": "^0.1.1" }, @@ -11160,6 +11216,48 @@ "node": ">= 0.10" } }, + "node_modules/redis": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz", + "integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==", + "dependencies": { + "denque": "^1.5.0", + "redis-commands": "^1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-redis" + } + }, + "node_modules/redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -15983,9 +16081,9 @@ "integrity": "sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg==" }, "@subsquid/logger": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/@subsquid/logger/-/logger-0.0.2.tgz", - "integrity": "sha512-Z7M2D1No25ZraTzuCibwC3TdnxiCD/UI/tLYxY5T5hXfRX9QKiqJ2Sj+EOBwx1wOD9bVreNFFF7LE15gQzAsfg==", + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@subsquid/logger/-/logger-0.1.0.tgz", + "integrity": "sha512-tiM9gUyaB9RJupfD8CskKSMyDb8xXFl6JhQm0r/xy00RhFO3EKOu2Jc7UvvCkCVuJImVwWLh/xdwqZ+d4SXY2g==", "requires": { "@subsquid/util-internal-hex": "^0.0.1", "@subsquid/util-internal-json": "^0.1.1", @@ -16012,60 +16110,60 @@ } }, "@subsquid/scale-codec": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@subsquid/scale-codec/-/scale-codec-1.0.2.tgz", - "integrity": "sha512-beWD0WFjJBK9JmBSJrfHK3kzae55s4itcJlmZelo70rknHgsRW1oTXHihDvRdG5fKAfrw5dkc1ZmEz8d3Pr20Q==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@subsquid/scale-codec/-/scale-codec-1.0.3.tgz", + "integrity": "sha512-wUPtNrO9i6doOqoxH7IeoZ8wS/jSgH/VZraBCzno+Cmv/6pRf6svWiCuooPzEXFZ1BI1XDHoZ2NLBf7DFkdMkA==", "requires": { "@subsquid/util-internal-hex": "^0.0.1", "@subsquid/util-internal-json": "^0.1.1" } }, "@subsquid/substrate-metadata": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@subsquid/substrate-metadata/-/substrate-metadata-1.0.2.tgz", - "integrity": "sha512-WAcWDsA/T5vYZzkNL7KSZI1K+0Bj3JBsxrL/y2vfw58p/AxuhoXsdfe0GTAylvOryo7MUPNsDMbw6v98PEb/Cw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@subsquid/substrate-metadata/-/substrate-metadata-1.1.0.tgz", + "integrity": "sha512-X19/x8SFcQ3Yyupld8bm2DsoukOUbhCUqEnwCb+a4NqutmoZhiKMnK8lsUyzND21SGEt3CQ7QRO7VB0nw3P5tA==", "requires": { - "@subsquid/scale-codec": "^1.0.2", + "@subsquid/scale-codec": "^1.0.3", "@subsquid/util-internal": "^0.0.1", "@subsquid/util-naming": "^0.0.1" } }, "@subsquid/substrate-processor": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@subsquid/substrate-processor/-/substrate-processor-1.5.1.tgz", - "integrity": "sha512-rKHQVodVKy3GyeBQ6vFt/puC0xPPHieJwTAylXvQI4CE3mDIeOHagu0mQjj3yspTZ8jmk+dPmDNaXA/kRFdGwQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@subsquid/substrate-processor/-/substrate-processor-1.8.0.tgz", + "integrity": "sha512-8pXbXLyeX3ItxpGYRkNc8q7lH9Lj5B4Y79xmb81ky+hzUzSGG8qREoU5w6xnacLi19p004jveJ/vKu2YHkgfkg==", "requires": { - "@subsquid/logger": "^0.0.2", + "@subsquid/logger": "^0.1.0", "@subsquid/rpc-client": "^1.0.2", - "@subsquid/scale-codec": "^1.0.2", - "@subsquid/substrate-metadata": "^1.0.2", - "@subsquid/typeorm-config": "^1.1.0", + "@subsquid/scale-codec": "^1.0.3", + "@subsquid/substrate-metadata": "^1.1.0", + "@subsquid/typeorm-config": "^2.0.0", "@subsquid/util-internal": "^0.0.1", "@subsquid/util-internal-binary-heap": "^0.0.0", "@subsquid/util-internal-code-printer": "^0.0.2", "@subsquid/util-internal-counters": "^0.0.1", "@subsquid/util-internal-gql-request": "^0.1.0", "@subsquid/util-internal-hex": "^0.0.1", - "@subsquid/util-internal-prometheus-server": "^0.0.1", + "@subsquid/util-internal-prometheus-server": "^0.0.2", "@subsquid/util-xxhash": "^0.1.1", "blake2b": "^2.1.4", "prom-client": "^14.0.1" } }, "@subsquid/typeorm-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@subsquid/typeorm-config/-/typeorm-config-1.1.0.tgz", - "integrity": "sha512-7qIOlpodIvIhrm3pudu2jpzMXx3aCxj39FPMV38NPzbft5AEKTmfRJPPSzBDbsYnz+WFJECDkLEb2vnFUsfuDw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@subsquid/typeorm-config/-/typeorm-config-2.0.0.tgz", + "integrity": "sha512-TJ/ksxTLTEHqY6NiYM5S9JXFdUEROTjTY+PKQv3ixoQuuidNzIGRwwGvzMnSKNl9s1V5ADcwWIEbmcPBZjHURg==", "requires": { "@subsquid/util-naming": "^0.0.1" } }, "@subsquid/typeorm-store": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@subsquid/typeorm-store/-/typeorm-store-0.1.1.tgz", - "integrity": "sha512-DOx6L/NFDj4XkTNOjcgcQQjHbMPKR7vmO/XHXUHQ27ikxUZrZ8w8y2pPMfFXN5aJUfWoHmGeLOO9BvZ0ZeN/vQ==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@subsquid/typeorm-store/-/typeorm-store-0.1.3.tgz", + "integrity": "sha512-agu6G4p9mNB0FmLi/uRx8+8k6AT9FQB12jqIRMzRtmobS2IzH88d3+SuTz86vHhuR4t6VHM7aarByTTHKZBX9w==", "requires": { - "@subsquid/typeorm-config": "^1.1.0", + "@subsquid/typeorm-config": "^2.0.0", "@subsquid/util-internal": "^0.0.1" } }, @@ -16103,9 +16201,9 @@ "integrity": "sha512-sNok0jQV6+OpAl3QKaH2VFh8PKZyZ6XHZhZ71LeirOhgfVprKFmEvFG9yQIp7qKe7JGXmolX54zu150OMP9f5w==" }, "@subsquid/util-internal-http-server": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@subsquid/util-internal-http-server/-/util-internal-http-server-0.0.1.tgz", - "integrity": "sha512-12zcFZLeI6ss0XJeR+5p6oJZLjyHSGh3Lrs80T9YuQzverE5dP0zIvnEpx62iPYNSJGesP2Z5olAy9VduUsUzA==", + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@subsquid/util-internal-http-server/-/util-internal-http-server-0.1.0.tgz", + "integrity": "sha512-7+lQlGMpKg5Qeavkas2/uBgF4cQQKG5gblhsuPwt/cyDGfIQcwPX9KxJUVQDBE8SL2P62qg8TkzeEYSG89iykg==", "requires": { "stoppable": "^1.1.0" } @@ -16119,11 +16217,11 @@ } }, "@subsquid/util-internal-prometheus-server": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@subsquid/util-internal-prometheus-server/-/util-internal-prometheus-server-0.0.1.tgz", - "integrity": "sha512-kEQ50rWVr56bMsiVYP6LgeQVtkpw+TW0o2rhs2M3E4U8Rt2P7XnUIJLLRMHN+AEco0OJ3m6PlQ0jlQylI7ZZSg==", + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@subsquid/util-internal-prometheus-server/-/util-internal-prometheus-server-0.0.2.tgz", + "integrity": "sha512-ODLBH03RkDm34AHkInQ6M5gcm5LCtI9wBAoOlKtqCcgTSIflELJKfmvqGCd3hhPyw0t6PJ2LQayrX3rth/nOqw==", "requires": { - "@subsquid/util-internal-http-server": "^0.0.1" + "@subsquid/util-internal-http-server": "^0.1.0" } }, "@subsquid/util-naming": { @@ -16325,6 +16423,12 @@ "@types/node": "*" } }, + "@types/cache-manager": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/cache-manager/-/cache-manager-4.0.1.tgz", + "integrity": "sha512-w4Gm7qg4ohvk0k4CLhOoqnMohWEyeyAOTovPgkguhuDCfVEV1wN/HWEd1XzB1S9/NV9pUcZcc498qU4E15ck6A==", + "dev": true + }, "@types/chai": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.0.tgz", @@ -16663,9 +16767,9 @@ "dev": true }, "@unique-nft/api": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/@unique-nft/api/-/api-0.1.12.tgz", - "integrity": "sha512-M52g/cjk/PRo33YnpeFALeHlfBVXiGs79+VdY4Bg4C/yHzGfQ/zJ2Amhoga7hCUoFY5zJg0o/3Qoo74+eCEq9Q==", + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@unique-nft/api/-/api-0.1.13.tgz", + "integrity": "sha512-Asee4NywIYOQ9PUzc5o2VOf2nKxG7iAAqcd5nH9CuyWZR+NV2HxO+u//DAUFugE0qCh5APAh7pkDJ1G5gFO/GA==", "requires": { "@noble/hashes": "^1.1.2", "@polkadot/api": "8.7.1", @@ -17014,12 +17118,12 @@ "requires": {} }, "@unique-nft/substrate-client": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@unique-nft/substrate-client/-/substrate-client-0.5.1.tgz", - "integrity": "sha512-Nj2qWxGNNBXkvuNVSZ1EHhTpP3lYh9sSWMMDjf4o2Ecl9lgz6m2zq1RBHYnzWaTUIBxgx2N+UBs/oYJ1ynFIWA==", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@unique-nft/substrate-client/-/substrate-client-0.5.5.tgz", + "integrity": "sha512-qiMPV5Z634QHCAQG/TCTnE8J0RETSenOOQgocfJoKRyRvJdSRwC9mjEpzIGjR2G4SIFFoWv+F9FR6J3EnrclOQ==", "requires": { "@polkadot/api": "^8.6.2", - "@unique-nft/api": "0.1.12", + "@unique-nft/api": "0.1.13", "protobufjs": "^6.11.2", "rxjs": "^7.0.0" } @@ -17846,6 +17950,36 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" }, + "cache-manager": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-4.1.0.tgz", + "integrity": "sha512-ZGM6dLxrP65bfOZmcviWMadUOCICqpLs92+P/S5tj8onz+k+tB7Gr+SAgOUHCQtfm2gYEQDHiKeul4+tYPOJ8A==", + "requires": { + "async": "3.2.3", + "lodash.clonedeep": "^4.5.0", + "lru-cache": "^7.10.1" + }, + "dependencies": { + "async": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", + "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==" + }, + "lru-cache": { + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.14.0.tgz", + "integrity": "sha512-EIRtP1GrSJny0dqb50QXRUNBxHJhcpxHC++M5tD7RYbvLLn5KVWKsbyswSSqDuU15UFi3bgTQIY8nhDMeF6aDQ==" + } + } + }, + "cache-manager-redis-store": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cache-manager-redis-store/-/cache-manager-redis-store-2.0.0.tgz", + "integrity": "sha512-bWLWlUg6nCYHiJLCCYxY2MgvwvKnvlWwrbuynrzpjEIhfArD2GC9LtutIHFEPeyGVQN6C+WEw+P3r+BFBwhswg==", + "requires": { + "redis": "^3.0.2" + } + }, "call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -18336,6 +18470,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "denque": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", + "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -20747,6 +20886,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" + }, "lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -21634,9 +21778,9 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "prom-client": { - "version": "14.0.1", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.0.1.tgz", - "integrity": "sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.1.0.tgz", + "integrity": "sha512-iFWCchQmi4170omLpFXbzz62SQTmPhtBL35v0qGEVRHKcqIeiexaoYeP0vfZTujxEq3tA87iqOdRbC9svS1B9A==", "requires": { "tdigest": "^0.1.1" } @@ -21789,6 +21933,35 @@ "resolve": "^1.1.6" } }, + "redis": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz", + "integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==", + "requires": { + "denque": "^1.5.0", + "redis-commands": "^1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0" + } + }, + "redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "requires": { + "redis-errors": "^1.0.0" + } + }, "reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", diff --git a/package.json b/package.json index e8318a0e..bc4a9151 100644 --- a/package.json +++ b/package.json @@ -50,13 +50,15 @@ "@ntegral/nestjs-sentry": "^3.0.7", "@polkadot/util-crypto": "^9.7.2", "@sentry/node": "^6.19.7", - "@subsquid/substrate-processor": "^1.5.1", - "@subsquid/typeorm-store": "^0.1.1", + "@subsquid/substrate-processor": "^1.8.0", + "@subsquid/typeorm-store": "^0.1.3", "@unique-nft/opal-testnet-types": "^0.5.2", "@unique-nft/quartz-mainnet-types": "^0.6.0", - "@unique-nft/substrate-client": "^0.5.1", + "@unique-nft/substrate-client": "^0.5.5", "apollo-server-express": "^3.6.2", "bignumber.js": "^9.0.2", + "cache-manager": "^4.1.0", + "cache-manager-redis-store": "^2.0.0", "graphql": "^15.8.0", "graphql-type-json": "^0.3.2", "lodash": "^4.17.21", @@ -71,6 +73,7 @@ "@nestjs/cli": "^8.0.0", "@nestjs/schematics": "^8.0.0", "@nestjs/testing": "^8.0.0", + "@types/cache-manager": "^4.0.1", "@types/chai": "^4.3.0", "@types/express": "^4.17.13", "@types/jest": "27.0.2",