Merge pull request #270 from UniqueNetwork/release/v2.0.0

Release/v2.0.0
UniqueNetwork · Jul 14, 2023 · 5997f22 · 5997f22
2 parents c154abb + afbb567
commit 5997f22
Show file tree

Hide file tree

Showing 65 changed files with 6,526 additions and 4,351 deletions.
diff --git a/apps/crawler/src/config/processor.config.service.ts b/apps/crawler/src/config/processor.config.service.ts
@@ -1,14 +1,17 @@
 import { Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
-import { DataSource as SubscquidDataSource } from '@subsquid/substrate-processor';
-import { Range } from '@subsquid/substrate-processor/lib/util/range';
 import { Config } from './config.module';
 
+export type Range = {
+  from: number;
+  to: number;
+};
+
 @Injectable()
 export class ProcessorConfigService {
   constructor(private configService: ConfigService<Config>) {}
 
-  public getDataSource(): SubscquidDataSource {
+  public getDataSource() {
     return {
       archive: this.configService.get('archiveGqlUrl'),
       chain: this.configService.get('chainWsUrl'),
@@ -27,12 +30,12 @@ export class ProcessorConfigService {
   }
 
   public getAllParams(): {
-    dataSource: SubscquidDataSource;
+    // dataSource: SubscquidDataSource;
     range: Range;
     typesBundle: string;
   } {
     return {
-      dataSource: this.getDataSource(),
+      //dataSource: this.getDataSource(),
       range: this.getRange(),
       typesBundle: this.getTypesBundle(),
     };

diff --git a/apps/crawler/src/crawler.module.ts b/apps/crawler/src/crawler.module.ts
@@ -7,20 +7,23 @@ import { CrawlerService } from './crawler.service';
 import { SubscribersModule } from './subscribers/subscribers.module';
 import { Config, GlobalConfigModule } from './config/config.module';
 import { CacheProviderModule } from './cache/cache-provider.module';
+import { HarvesterModule, HarvesterModuleOptions } from '@unique-nft/harvester';
+import { MonitoringModule } from '@common/monitoring';
 
 @Module({
   imports: [
     GlobalConfigModule,
     CacheProviderModule,
+    MonitoringModule,
     TypeOrmModule.forRoot(typeormConfig),
-    // HarvesterModule.registerAsync({
-    //   useFactory: (config: ConfigService<Config>) =>
-    //     ({
-    //       chainWsUrl: config.get('chainWsUrl'),
-    //       database: typeormConfig,
-    //     } as HarvesterModuleOptions),
-    //   inject: [ConfigService],
-    // }),
+    HarvesterModule.registerAsync({
+      useFactory: (config: ConfigService<Config>) =>
+        ({
+          chainWsUrl: config.get('chainWsUrl'),
+          databaseConfig: typeormConfig,
+        } as HarvesterModuleOptions),
+      inject: [ConfigService],
+    }),
     SentryModule.forRootAsync({
       useFactory: async (configService: ConfigService<Config>) => {
         return configService.get('sentry');

diff --git a/apps/crawler/src/crawler.service.ts b/apps/crawler/src/crawler.service.ts
@@ -1,4 +1,4 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Scope } from '@nestjs/common';
 import { SubscribersService } from './subscribers/subscribers.service';
 
 @Injectable()

diff --git a/apps/crawler/src/main.ts b/apps/crawler/src/main.ts
@@ -1,3 +1,4 @@
+import { startMetricsServer } from '@common/monitoring';
 import { Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { NestFactory } from '@nestjs/core';
@@ -13,7 +14,7 @@ async function bootstrap() {
   const logLevels = configService.get('logLevels');
 
   Logger.overrideLogger(logLevels);
-
+  await startMetricsServer(app);
   await app.init();
 
   try {

diff --git a/apps/crawler/src/services/account/account.service.ts b/apps/crawler/src/services/account/account.service.ts
@@ -6,8 +6,9 @@ import { Address as AddressUtils } from '@unique-nft/utils';
 import { Address } from '@unique-nft/substrate-client/types';
 import { AllBalances } from '@unique-nft/substrate-client/balance';
 import { Repository } from 'typeorm';
-import { SdkService } from '../../sdk/sdk.service';
+
 import { AccountRecord } from './account.types';
+import { SdkService } from '@common/sdk/sdk.service';
 
 type BalancesExtended = AllBalances & {
   etheriumAddress?: string;
@@ -41,10 +42,8 @@ export class AccountService {
     );
 
     const normalizedAddress = await this.upsert({
-      // todo: Решить с пустыми blockTimestamp & blockNumber
       blockTimestamp,
       blockNumber,
-
       balances: balancesExtended,
     });
 

diff --git a/apps/crawler/src/services/account/account.types.ts b/apps/crawler/src/services/account/account.types.ts
@@ -2,4 +2,8 @@ import { Address } from '@unique-nft/substrate-client/types';
 
 export type AccountRecord =
   | Address
-  | { value: Address; __kind: 'Substrate' | 'Etherium' };
+  | { value: Address; __kind: 'Substrate' | 'Ethereum' };
+
+export type AccountAddressData =
+  | { substrate?: Address; ethereum?: Address }
+  | Address;
diff --git a/apps/crawler/src/services/block.service.ts b/apps/crawler/src/services/block.service.ts
@@ -1,14 +1,9 @@
-import { EventName } from '@common/constants';
-import { normalizeTimestamp } from '@common/utils';
 import { Block } from '@entities/Block';
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
-import { SubstrateBlock } from '@subsquid/substrate-processor';
+
 import { Repository } from 'typeorm';
-import {
-  IBlockItem,
-  IItemCounts,
-} from '../subscribers/blocks.subscriber.service';
+import { IBlockDataContainer } from '../subscribers/blocks.subscriber.service';
 
 @Injectable()
 export class BlockService {
@@ -17,92 +12,8 @@ export class BlockService {
     private blocksRepository: Repository<Block>,
   ) {}
 
-  private collectItemCounts(blockItems: IBlockItem[]) {
-    const itemCounts = {
-      totalEvents: 0,
-      totalExtrinsics: 0,
-      numTransfers: 0,
-      newAccounts: 0,
-    };
-
-    blockItems.forEach((item) => {
-      const { kind } = item;
-
-      if (kind === 'event') {
-        // Event
-        itemCounts.totalEvents += 1;
-
-        const {
-          event: { name },
-        } = item;
-
-        if (name === EventName.BALANCES_TRANSFER) {
-          itemCounts.numTransfers += 1;
-        } else if (name === EventName.BALANCES_ENDOWED) {
-          itemCounts.newAccounts += 1;
-        }
-      } else {
-        // Extrinsic
-        itemCounts.totalExtrinsics += 1;
-      }
-    });
-
-    return itemCounts;
-  }
-
-  private prepareDataForDb({
-    block,
-    itemCounts,
-  }: {
-    block: SubstrateBlock;
-    itemCounts: IItemCounts;
-  }): Block {
-    const {
-      specId,
-      parentHash,
-      stateRoot,
-      extrinsicsRoot,
-      hash: blockHash,
-      height: blockNumber,
-      timestamp: blockTimestamp,
-    } = block;
-
-    const [specName, specVersion] = specId.split('@') as [string, number];
-
-    const { totalEvents, totalExtrinsics, numTransfers, newAccounts } =
-      itemCounts;
-
-    return {
-      block_number: blockNumber,
-      block_hash: blockHash,
-      parent_hash: parentHash,
-      extrinsics_root: extrinsicsRoot,
-      state_root: stateRoot,
-      spec_name: specName,
-      spec_version: specVersion,
-      timestamp: String(normalizeTimestamp(blockTimestamp)),
-
-      // Item counts
-      total_events: totalEvents,
-      num_transfers: numTransfers,
-      new_accounts: newAccounts,
-      total_extrinsics: totalExtrinsics,
-    };
-  }
-
-  async upsert({
-    block,
-    blockItems,
-  }: {
-    block: SubstrateBlock;
-    blockItems: IBlockItem[];
-  }): Promise<IItemCounts> {
-    const itemCounts = this.collectItemCounts(blockItems);
-
-    const blockData = this.prepareDataForDb({ block, itemCounts });
-
+  async upsert(blockData): Promise<IBlockDataContainer> {
     await this.blocksRepository.upsert(blockData, ['block_number']);
-
-    return itemCounts;
+    return blockData;
   }
 }
diff --git a/apps/crawler/src/services/collection.service.ts b/apps/crawler/src/services/collection.service.ts
@@ -22,14 +22,11 @@ import {
   UniqueCollectionSchemaDecoded,
 } from '@unique-nft/substrate-client/tokens';
 import { Repository } from 'typeorm';
-import { SdkService } from '../sdk/sdk.service';
-import {
-  IBlockCommonData,
-  ItemsBatchProcessingResult,
-} from '../subscribers/blocks.subscriber.service';
+
 import { ConfigService } from '@nestjs/config';
 import { Config } from '../config/config.module';
 import { Event } from '@entities/Event';
+import { SdkService } from '@common/sdk/sdk.service';
 
 type ParsedSchemaFields = {
   collectionCover?: string;
@@ -68,14 +65,18 @@ export class CollectionService {
     collectionId: number,
     at: string,
   ): Promise<CollectionData | null> {
-    let collectionDecoded = await this.sdkService.getCollection(collectionId);
+    debugger;
+    let collectionDecoded = await this.sdkService.getCollection(
+      collectionId,
+      at,
+    );
     let checkAt = false; // for burned collections
 
     if (!collectionDecoded) {
       collectionDecoded = await this.sdkService.getCollection(collectionId, at);
       checkAt = true;
     }
-
+    debugger;
     if (!collectionDecoded) {
       return null;
     }
@@ -84,7 +85,7 @@ export class CollectionService {
     // if (collectionDecoded.mode === CollectionMode.ReFungible) {
     //   return null;
     // }
-
+    // debugger;
     const [collectionLimits, tokenPropertyPermissions] = await Promise.all([
       this.sdkService.getCollectionLimits(
         collectionId,
@@ -98,8 +99,9 @@ export class CollectionService {
 
     return {
       collectionDecoded,
-      collectionLimits,
-      tokenPropertyPermissions,
+      collectionLimits: collectionLimits || collectionDecoded.limits,
+      tokenPropertyPermissions:
+        tokenPropertyPermissions || collectionDecoded.tokenPropertyPermissions,
     };
   }
 
@@ -281,10 +283,9 @@ export class CollectionService {
     blockCommonData,
   }: {
     events: Event[];
-    blockCommonData: IBlockCommonData;
-  }): Promise<ItemsBatchProcessingResult> {
+    blockCommonData: any;
+  }): Promise<any> {
     const collectionEvents = this.extractCollectionEvents(events);
-
     const eventChunks = chunk(
       collectionEvents,
       this.configService.get('scanCollectionsBatchSize'),
@@ -299,15 +300,15 @@ export class CollectionService {
             collectionId: number;
           };
 
-          const { blockHash, blockTimestamp } = blockCommonData;
+          const { block_hash, timestamp } = blockCommonData;
           const eventName = `${section}.${method}`;
 
           if (COLLECTION_UPDATE_EVENTS.includes(eventName)) {
             return this.update({
               collectionId,
               eventName,
-              blockTimestamp,
-              blockHash,
+              blockTimestamp: timestamp,
+              blockHash: block_hash,
             });
           } else {
             return this.burn(collectionId);
@@ -324,6 +325,8 @@ export class CollectionService {
 
     return {
       totalEvents: collectionEvents.length,
+      collection:
+        collectionEvents.length === 1 ? collectionEvents[0].values : null,
       rejected,
     };
   }