Merge pull request #231 from UniqueNetwork/release/v1.7.2

Release/v1.7.2

.env.default

@@ -13,6 +13,8 @@ SCAN_TYPES_BUNDLE=quartz
 #SCAN_RANGE_FROM=111000
 #SCAN_RANGE_TO=1000000
 #SCAN_FORCE_RESCAN=true
+SCAN_COLLECTIONS_BATCH_SIZE=50
+SCAN_TOKENS_BATCH_SIZE=100
 PROMETHEUS_PORT=3003
 BATCH_SIZE=10
 

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npm --no-git-tag-version version patch && git add package.json

apps/crawler/src/config/config.module.ts

@@ -26,11 +26,17 @@ export type Config = {
 
   scanRangeTo?: number;
 
+  scanCollectionsBatchSize?: number;
+
+  scanTokensBatchSize?: number;
+
   rescan: boolean;
 
   prometheusPort: number;
 
   batchSize: number;
+
+  rpcProviderUrl: string;
 };
 
 const loadConfig = (): Config => ({
@@ -54,11 +60,18 @@ const loadConfig = (): Config => ({
 
   scanRangeTo: +process.env.SCAN_RANGE_TO || undefined,
 
+  scanCollectionsBatchSize: +process.env.SCAN_COLLECTIONS_BATCH_SIZE || 50,
+
+  scanTokensBatchSize: +process.env.SCAN_TOKENS_BATCH_SIZE || 100,
+
   rescan: process.env.SCAN_FORCE_RESCAN === 'true',
 
   prometheusPort: +process.env.PROMETHEUS_PORT || 9090,
 
   batchSize: +process.env.BATCH_SIZE || 10,
+
+  rpcProviderUrl:
+    process.env.RPC_PROVIDER_URL || 'https://rpc-opal.unique.network',
 });
 
 export const GlobalConfigModule = ConfigModule.forRoot({

apps/crawler/src/crawler.module.ts

@@ -13,6 +13,14 @@ import { CacheProviderModule } from './cache/cache-provider.module';
     GlobalConfigModule,
     CacheProviderModule,
     TypeOrmModule.forRoot(typeormConfig),
+    // HarvesterModule.registerAsync({
+    //   useFactory: (config: ConfigService<Config>) =>
+    //     ({
+    //       chainWsUrl: config.get('chainWsUrl'),
+    //       database: typeormConfig,
+    //     } as HarvesterModuleOptions),
+    //   inject: [ConfigService],
+    // }),
     SentryModule.forRootAsync({
       useFactory: async (configService: ConfigService<Config>) => {
         return configService.get('sentry');

apps/crawler/src/main.ts

@@ -14,6 +14,8 @@ async function bootstrap() {
 
   Logger.overrideLogger(logLevels);
 
+  await app.init();
+
   try {
     const crawlerService = app.get(CrawlerService);
 

apps/crawler/src/sdk/sdk.module.ts

@@ -4,7 +4,6 @@ import { Client } from '@unique-nft/substrate-client';
 import { sdkFactory } from './sdk-factory';
 import { SdkService } from './sdk.service';
 import { Config } from '../config/config.module';
-import '@unique-nft/substrate-client/extrinsics';
 import '@unique-nft/substrate-client/tokens';
 import '@unique-nft/substrate-client/balance';
 

apps/crawler/src/sdk/sdk.service.ts

@@ -1,5 +1,6 @@
 import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
+import { Cache } from 'cache-manager';
 import { Client } from '@unique-nft/substrate-client';
 import {
   CollectionInfoWithSchema,
@@ -9,6 +10,7 @@ import {
 } from '@unique-nft/substrate-client/tokens';
 import { Config } from '../config/config.module';
 import { SdkCache } from './sdk-cache.decorator';
+import { TokenBalanceRequest } from '@unique-nft/substrate-client/refungible';
 
 @Injectable()
 export class SdkService {
@@ -18,6 +20,12 @@ export class SdkService {
     @Inject(CACHE_MANAGER) private cacheManager: Cache,
   ) {}
 
+  @SdkCache('getApi')
+  async getApi(hash) {
+    const optionUpgrade = await this.sdk.api.query.system.events.at(hash);
+    return optionUpgrade.toJSON();
+  }
+
   @SdkCache('getCollection')
   getCollection(
     collectionId: number,
@@ -47,12 +55,16 @@ export class SdkService {
   }
 
   @SdkCache('getToken')
-  getToken(
+  async getToken(
     collectionId: number,
     tokenId: number,
     at?: string,
   ): Promise<TokenByIdResult | null> {
-    return this.sdk.tokens.get({ collectionId, tokenId, at });
+    if (at) {
+      return await this.sdk.tokens.get({ collectionId, tokenId, at });
+    } else {
+      return await this.sdk.tokens.get({ collectionId, tokenId });
+    }
   }
 
   @SdkCache('isTokenBundle')
@@ -84,4 +96,55 @@ export class SdkService {
   async getBalances(rawAddress: string) {
     return this.sdk.balance.get({ address: rawAddress });
   }
+
+  @SdkCache('getRFTBalances')
+  async getRFTBalances(
+    tokenBalance: TokenBalanceRequest,
+    at?: string,
+  ): Promise<any> {
+    const collection = await this.getCollection(tokenBalance.collectionId);
+    if (collection.mode === 'NFT') {
+      return {
+        amount: 0, // todo tak ne nado
+      };
+    }
+    if (collection.mode === 'ReFungible') {
+      let dataCheckNalance;
+      if (at) {
+        dataCheckNalance = {
+          address: `${tokenBalance.address}`,
+          collectionId: tokenBalance.collectionId,
+          tokenId: tokenBalance.tokenId,
+          at,
+        };
+      } else {
+        dataCheckNalance = {
+          address: `${tokenBalance.address}`,
+          collectionId: tokenBalance.collectionId,
+          tokenId: tokenBalance.tokenId,
+        };
+      }
+      return await this.sdk.refungible.getBalance(dataCheckNalance);
+    }
+  }
+
+  @SdkCache('getTotalPieces')
+  async getTotalPieces(
+    tokenId: number,
+    collectionId: number,
+    at?: string,
+  ): Promise<any> {
+    if (at) {
+      return await this.sdk.refungible.totalPieces({
+        tokenId,
+        collectionId,
+        at,
+      });
+    } else {
+      return await this.sdk.refungible.totalPieces({
+        tokenId,
+        collectionId,
+      });
+    }
+  }
 }

apps/crawler/src/services/account/account.service.ts

@@ -2,7 +2,7 @@ import { normalizeSubstrateAddress, normalizeTimestamp } from '@common/utils';
 import { Account } from '@entities/Account';
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
-import { isEthereumAddress } from '@polkadot/util-crypto';
+import { Address as AddressUtils } from '@unique-nft/utils';
 import { Address } from '@unique-nft/substrate-client/types';
 import { AllBalances } from '@unique-nft/substrate-client/balance';
 import { Repository } from 'typeorm';
@@ -70,7 +70,9 @@ export class AccountService {
   ): BalancesExtended {
     return {
       ...balances,
-      etheriumAddress: isEthereumAddress(address) ? address : null,
+      etheriumAddress: AddressUtils.is.ethereumAddress(address)
+        ? address
+        : null,
     };
   }
 

apps/crawler/src/services/collection.service.ts

@@ -1,4 +1,10 @@
-import { EventName, SubscriberAction } from '@common/constants';
+import { chunk } from 'lodash';
+import {
+  COLLECTION_BURN_EVENTS,
+  COLLECTION_UPDATE_EVENTS,
+  EventName,
+  SubscriberAction,
+} from '@common/constants';
 import {
   normalizeSubstrateAddress,
   normalizeTimestamp,
@@ -12,12 +18,18 @@ import {
   CollectionInfoWithSchema,
   CollectionLimits,
   CollectionProperty,
-  UniqueCollectionSchemaDecoded,
   PropertyKeyPermission,
-  CollectionMode,
+  UniqueCollectionSchemaDecoded,
 } from '@unique-nft/substrate-client/tokens';
 import { Repository } from 'typeorm';
 import { SdkService } from '../sdk/sdk.service';
+import {
+  IBlockCommonData,
+  ItemsBatchProcessingResult,
+} from '../subscribers/blocks.subscriber.service';
+import { ConfigService } from '@nestjs/config';
+import { Config } from '../config/config.module';
+import { Event } from '@entities/Event';
 
 type ParsedSchemaFields = {
   collectionCover?: string;
@@ -39,8 +51,12 @@ export class CollectionService {
 
   constructor(
     private sdkService: SdkService,
+
+    private configService: ConfigService<Config>,
+
     @InjectRepository(Collections)
     private collectionsRepository: Repository<Collections>,
+
     @InjectRepository(Tokens)
     private tokensRepository: Repository<Tokens>,
   ) {}
@@ -65,9 +81,9 @@ export class CollectionService {
     }
 
     // TODO: delete after rft support
-    if (collectionDecoded.mode === CollectionMode.ReFungible) {
-      return null;
-    }
+    // if (collectionDecoded.mode === CollectionMode.ReFungible) {
+    //   return null;
+    // }
 
     const [collectionLimits, tokenPropertyPermissions] = await Promise.all([
       this.sdkService.getCollectionLimits(
@@ -183,7 +199,6 @@ export class CollectionService {
   ): Promise<Collections> {
     const { collectionDecoded, collectionLimits, tokenPropertyPermissions } =
       collectionData;
-
     const {
       id: collection_id,
       owner,
@@ -252,7 +267,7 @@ export class CollectionService {
       sponsorship: sponsorship?.isConfirmed ? sponsorship.address : null,
       schema_version: schemaVersion,
       token_prefix,
-      mode,
+      mode: mode === 'ReFungible' ? 'RFT' : mode,
       mint_mode: mintMode,
       nesting_enabled: nesting?.collectionAdmin || nesting?.tokenOwner,
       owner_normalized: normalizeSubstrateAddress(owner),
@@ -261,6 +276,58 @@ export class CollectionService {
     };
   }
 
+  async batchProcess({
+    events,
+    blockCommonData,
+  }: {
+    events: Event[];
+    blockCommonData: IBlockCommonData;
+  }): Promise<ItemsBatchProcessingResult> {
+    const collectionEvents = this.extractCollectionEvents(events);
+
+    const eventChunks = chunk(
+      collectionEvents,
+      this.configService.get('scanCollectionsBatchSize'),
+    );
+
+    let rejected = [];
+    for (const chunk of eventChunks) {
+      const result = await Promise.allSettled(
+        chunk.map((event) => {
+          const { section, method, values } = event;
+          const { collectionId } = values as unknown as {
+            collectionId: number;
+          };
+
+          const { blockHash, blockTimestamp } = blockCommonData;
+          const eventName = `${section}.${method}`;
+
+          if (COLLECTION_UPDATE_EVENTS.includes(eventName)) {
+            return this.update({
+              collectionId,
+              eventName,
+              blockTimestamp,
+              blockHash,
+            });
+          } else {
+            return this.burn(collectionId);
+          }
+        }),
+      );
+
+      // todo: Process rejected tokens again or maybe process sdk disconnect
+      rejected = [
+        ...rejected,
+        ...result.filter(({ status }) => status === 'rejected'),
+      ];
+    }
+
+    return {
+      totalEvents: collectionEvents.length,
+      rejected,
+    };
+  }
+
   async update({
     collectionId,
     eventName,
@@ -316,4 +383,14 @@ export class CollectionService {
       ),
     ]);
   }
+
+  private extractCollectionEvents(events: Event[]) {
+    return events.filter(({ section, method }) => {
+      const eventName = `${section}.${method}`;
+      return (
+        COLLECTION_UPDATE_EVENTS.includes(eventName) ||
+        COLLECTION_BURN_EVENTS.includes(eventName)
+      );
+    });
+  }
 }