Merge pull request #167 from UniqueNetwork/release/v1.1.0

Release/v1.1.0

.env-e2e

@@ -0,0 +1,47 @@
+PORT=3000
+POSTGRES_USER=polkastats
+POSTGRES_PASSWORD=polkastats
+POSTGRES_DATABASE=polkastats
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+
+LOGGING=0
+LOG_LEVELS='log,error,warn,verbose'
+ARCHIVE_GQL_URL=https://archive.dev.uniquenetwork.dev/graphql
+#ARCHIVE_GQL_URL=https://archive.opal.uniquenetwork.dev/graphql
+CHAIN_WS_URL=ws://localhost:9944
+SCAN_TYPES_BUNDLE=quartz
+#SCAN_RANGE_FROM=111000
+#SCAN_RANGE_TO=1000000
+#SCAN_FORCE_RESCAN=true
+PROMETHEUS_PORT=3003
+BATCH_SIZE=10
+
+## Subscquid based subscribers
+
+#ACCOUNTS_SUBSCRIBER_DISABLE=true
+#BLOCKS_SUBSCRIBER_DISABLE=true
+#COLLECTIONS_SUBSCRIBER_DISABLE=true
+#TOKENS_SUBSCRIBER_DISABLE=true
+
+## Environment variables for tests
+
+TESTS_UNIQUE_WS_ENDPOINT=ws://localhost:9944
+TESTS_GRAPHQL_URL='http://localhost:3031/v1/graphql'
+TESTS_IPFS_URL=https://ipfs.unique.network/ipfs/
+
+## Sentry variables
+
+#SENTRY_DSN=SENTRY_DSN
+SENTRY_DEBUG=0
+SENTRY_LOG_LEVELS='error,warning'
+
+## Cache variables
+
+### Cache ttl in seconds.
+CACHE_TTL=60
+
+### Redis cache connection. If not set, memory cache is in use.
+#REDIS_HOST=
+#REDIS_PORT=
+#REDIS_DB=

.gitignore

@@ -35,3 +35,4 @@ lerna-debug.log*
 .env*
 !.env.default
 docker-compose.local.yml
+.fleet

README.md

@@ -63,13 +63,13 @@ $ npm run test:cov
 
 ```bash
 # start environment
-$ npm run tests:start-environment
+$ npm run integration:tests:start-environment
 
 # run tests
-$ npm run tests
+$ npm run integration:tests
 
 # stop environment
-$ npm run tests:stop-environment
+$ npm run integration:tests:stop-environment
 ```
 
 ## Support

apps/crawler/src/sdk/sdk-factory.ts

@@ -1,7 +1,7 @@
-import { Sdk } from '@unique-nft/substrate-client';
+import { Client } from '@unique-nft/substrate-client';
 
-export async function sdkFactory(chainWsUrl: string): Promise<Sdk> {
-  const sdk = new Sdk({
+export async function sdkFactory(chainWsUrl: string) {
+  const sdk = await Client.create({
     chainWsUrl,
   });
 

apps/crawler/src/sdk/sdk.module.ts

@@ -1,6 +1,6 @@
 import { Global, Module } from '@nestjs/common';
 import { ConfigModule, ConfigService } from '@nestjs/config';
-import { Sdk } from '@unique-nft/substrate-client';
+import { Client } from '@unique-nft/substrate-client';
 import { sdkFactory } from './sdk-factory';
 import { SdkService } from './sdk.service';
 import { Config } from '../config/config.module';
@@ -13,7 +13,7 @@ import '@unique-nft/substrate-client/balance';
   imports: [ConfigModule],
   providers: [
     {
-      provide: Sdk,
+      provide: Client,
       useFactory: async (configService: ConfigService<Config>) =>
         sdkFactory(configService.get('chainWsUrl')),
       inject: [ConfigService],

apps/crawler/src/sdk/sdk.service.ts

@@ -1,50 +1,75 @@
 import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
-import { Sdk } from '@unique-nft/substrate-client';
+import { Client } from '@unique-nft/substrate-client';
 import {
   CollectionInfoWithSchema,
+  PropertyKeyPermission,
   TokenByIdResult,
   TokenPropertiesResult,
 } from '@unique-nft/substrate-client/tokens';
-import { AllBalances } from '@unique-nft/substrate-client/types';
 import { Config } from '../config/config.module';
 import { SdkCache } from './sdk-cache.decorator';
 
 @Injectable()
 export class SdkService {
   constructor(
-    private sdk: Sdk,
+    private sdk: Client,
     private configService: ConfigService<Config>,
     @Inject(CACHE_MANAGER) private cacheManager: Cache,
   ) {}
 
   @SdkCache('getCollection')
   getCollection(
     collectionId: number,
+    at?: string,
   ): Promise<CollectionInfoWithSchema | null> {
-    return this.sdk.collections.get({ collectionId });
+    return this.sdk.collections.get({ collectionId, at });
   }
 
   @SdkCache('getCollectionLimits')
-  async getCollectionLimits(collectionId: number) {
-    const result = await this.sdk.collections.getLimits({ collectionId });
+  async getCollectionLimits(collectionId: number, at?: string) {
+    const result = await this.sdk.collections.getLimits({ collectionId, at });
     return result?.limits;
   }
 
   @SdkCache('getTokenPropertyPermissions')
-  async getTokenPropertyPermissions(collectionId: number) {
-    const result = await this.sdk.collections.propertyPermissions({
-      collectionId,
-    });
-    return result?.propertyPermissions ?? [];
+  async getTokenPropertyPermissions(collectionId: number, at?: string) {
+    try {
+      const property = await this.sdk.collections.propertyPermissions({
+        collectionId,
+        at: at ? `0x${at}` : undefined,
+      });
+
+      return property?.propertyPermissions;
+    } catch {
+      return [] as PropertyKeyPermission[];
+    }
   }
 
   @SdkCache('getToken')
   getToken(
     collectionId: number,
     tokenId: number,
+    at?: string,
   ): Promise<TokenByIdResult | null> {
-    return this.sdk.tokens.get({ collectionId, tokenId });
+    return this.sdk.tokens.get({ collectionId, tokenId, at });
+  }
+
+  @SdkCache('isTokenBundle')
+  isTokenBundle(collectionId: number, tokenId: number, at?: string) {
+    return this.sdk.tokens.isBundle({ collectionId, tokenId, at });
+  }
+
+  getTokenBundle(collectionId: number, tokenId: number, at?: string) {
+    return this.sdk.tokens.getBundle({
+      collectionId,
+      tokenId,
+      at,
+    });
+  }
+
+  getTokenParents(collectionId: number, tokenId: number, at?: string) {
+    return this.sdk.tokens.parent({ collectionId, tokenId, at });
   }
 
   @SdkCache('getTokenProperties')
@@ -56,7 +81,7 @@ export class SdkService {
   }
 
   @SdkCache('getBalances')
-  async getBalances(rawAddress: string): Promise<AllBalances> {
+  async getBalances(rawAddress: string) {
     return this.sdk.balance.get({ address: rawAddress });
   }
 }

apps/crawler/src/services/account/account.service.ts

@@ -3,7 +3,8 @@ import { Account } from '@entities/Account';
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { isEthereumAddress } from '@polkadot/util-crypto';
-import { Address, AllBalances } from '@unique-nft/substrate-client/types';
+import { Address } from '@unique-nft/substrate-client/types';
+import { AllBalances } from '@unique-nft/substrate-client/balance';
 import { Repository } from 'typeorm';
 import { SdkService } from '../../sdk/sdk.service';
 import { AccountRecord } from './account.types';
@@ -16,7 +17,6 @@ type BalancesExtended = AllBalances & {
 export class AccountService {
   constructor(
     private sdkService: SdkService,
-
     @InjectRepository(Account)
     private accountsRepository: Repository<Account>,
   ) {}

apps/crawler/src/services/collection.service.ts

@@ -14,6 +14,7 @@ import {
   CollectionProperty,
   UniqueCollectionSchemaDecoded,
   PropertyKeyPermission,
+  CollectionMode,
 } from '@unique-nft/substrate-client/tokens';
 import { Repository } from 'typeorm';
 import { SdkService } from '../sdk/sdk.service';
@@ -38,10 +39,8 @@ export class CollectionService {
 
   constructor(
     private sdkService: SdkService,
-
     @InjectRepository(Collections)
     private collectionsRepository: Repository<Collections>,
-
     @InjectRepository(Tokens)
     private tokensRepository: Repository<Tokens>,
   ) {}
@@ -51,16 +50,34 @@ export class CollectionService {
    */
   private async getCollectionData(
     collectionId: number,
+    at: string,
   ): Promise<CollectionData | null> {
-    const collectionDecoded = await this.sdkService.getCollection(collectionId);
+    let collectionDecoded = await this.sdkService.getCollection(collectionId);
+    let checkAt = false; // for burned collections
+
+    if (!collectionDecoded) {
+      collectionDecoded = await this.sdkService.getCollection(collectionId, at);
+      checkAt = true;
+    }
 
     if (!collectionDecoded) {
       return null;
     }
 
+    // TODO: delete after rft support
+    if (collectionDecoded.mode === CollectionMode.ReFungible) {
+      return null;
+    }
+
     const [collectionLimits, tokenPropertyPermissions] = await Promise.all([
-      this.sdkService.getCollectionLimits(collectionId),
-      this.sdkService.getTokenPropertyPermissions(collectionId),
+      this.sdkService.getCollectionLimits(
+        collectionId,
+        checkAt ? at : undefined,
+      ),
+      this.sdkService.getTokenPropertyPermissions(
+        collectionId,
+        checkAt ? at : undefined,
+      ),
     ]);
 
     return {
@@ -161,7 +178,9 @@ export class CollectionService {
     return result as UniqueCollectionSchemaDecoded;
   }
 
-  private prepareDataForDb(collectionData: CollectionData): Collections {
+  private async prepareDataForDb(
+    collectionData: CollectionData,
+  ): Promise<Collections> {
     const { collectionDecoded, collectionLimits, tokenPropertyPermissions } =
       collectionData;
 
@@ -208,6 +227,10 @@ export class CollectionService {
       ownerCanDestroy: owner_can_destroy,
     } = collectionLimits;
 
+    const collection = await this.collectionsRepository.findOneBy({
+      collection_id,
+    });
+
     return {
       collection_id,
       owner,
@@ -234,24 +257,30 @@ export class CollectionService {
       nesting_enabled: nesting?.collectionAdmin || nesting?.tokenOwner,
       owner_normalized: normalizeSubstrateAddress(owner),
       collection_cover: collectionCover,
+      burned: collection?.burned ?? false,
     };
   }
 
   async update({
     collectionId,
     eventName,
     blockTimestamp,
+    blockHash,
   }: {
     collectionId: number;
     eventName: string;
     blockTimestamp: number;
+    blockHash: string;
   }): Promise<SubscriberAction> {
-    const collectionData = await this.getCollectionData(collectionId);
+    const collectionData = await this.getCollectionData(
+      collectionId,
+      blockHash,
+    );
 
     let result;
 
     if (collectionData) {
-      const preparedData = this.prepareDataForDb(collectionData);
+      const preparedData = await this.prepareDataForDb(collectionData);
 
       await this.collectionsRepository.upsert(
         {
@@ -267,23 +296,24 @@ export class CollectionService {
       result = SubscriberAction.UPSERT;
     } else {
       // No entity returned from sdk. Most likely it was destroyed in a future block.
-      await this.delete(collectionId);
+      await this.burn(collectionId);
 
       result = SubscriberAction.DELETE_NOT_FOUND;
     }
 
     return result;
   }
 
-  async delete(collectionId: number) {
-    return this.deleteCollectionWithTokens(collectionId);
-  }
-
-  // Delete db collection record and related tokens
-  private async deleteCollectionWithTokens(collectionId: number) {
-    return Promise.all([
-      this.collectionsRepository.delete(collectionId),
-      this.tokensRepository.delete({ collection_id: collectionId }),
+  async burn(collectionId: number) {
+    return Promise.allSettled([
+      this.collectionsRepository.update(
+        { collection_id: collectionId },
+        { burned: true },
+      ),
+      this.tokensRepository.update(
+        { collection_id: collectionId },
+        { burned: true },
+      ),
     ]);
   }
 }

apps/crawler/src/services/services.module.ts

@@ -9,12 +9,13 @@ import { Extrinsic } from '@entities/Extrinsic';
 import { Account } from '@entities/Account';
 import { CollectionService } from './collection.service';
 import { AccountService } from './account/account.service';
-import { TokenService } from './token.service';
+import { TokenService } from './token/token.service';
 import { BlockService } from './block.service';
 import { EventService } from './event/event.service';
 import { ExtrinsicService } from './extrinsic.service';
 import { EventArgumentsService } from './event/event.arguments.service';
 import { SdkModule } from '../sdk/sdk.module';
+import { TokenNestingService } from './token/nesting.service';
 
 @Module({
   imports: [
@@ -37,6 +38,7 @@ import { SdkModule } from '../sdk/sdk.module';
     EventArgumentsService,
     ExtrinsicService,
     TokenService,
+    TokenNestingService,
   ],
   exports: [
     AccountService,
@@ -45,6 +47,7 @@ import { SdkModule } from '../sdk/sdk.module';
     EventService,
     ExtrinsicService,
     TokenService,
+    TokenNestingService,
   ],
 })
 export class ServicesModule {}