Merge pull request #140 from UniqueNetwork/release/v1.0.0

Release/v1.0.0

.env.default

@@ -5,22 +5,28 @@ POSTGRES_DATABASE=polkastats
 POSTGRES_HOST=localhost
 POSTGRES_PORT=5432
 LOGGING=0
-ARCHIVE_GQL_URL=https://quartz.indexer.gc.subsquid.io/v4/graphql
-# ARCHIVE_GQL_URL=https://quartz.subsquid.fatcat.ventures/v1/graphql
-CHAIN_WS_URL=wss://ws-quartz.unique.network
+LOG_LEVELS='log,error,warn,verbose'
+ARCHIVE_GQL_URL=https://archive.dev.uniquenetwork.dev/graphql
+#ARCHIVE_GQL_URL=https://archive.opal.uniquenetwork.dev/graphql
+CHAIN_WS_URL=wss://ws-rc.unique.network
 SCAN_TYPES_BUNDLE=quartz
-SCAN_RANGE_FROM_DEFAULT=1110000
-SCAN_RANGE_FROM=1110000
-# tokens 573323
-# SCAN_RANGE_TO=730000
+SCAN_RANGE_FROM_DEFAULT=0
+SCAN_RANGE_FROM=111000
+# SCAN_RANGE_TO=1000000
 SCAN_FORCE_RESCAN=true
+PROMETHEUS_PORT=3003
 
-ACCOUNTS_SUBSCRIBER_DISABLE=false
-BLOCKS_SUBSCRIBER_DISABLE=false
-COLLECTIONS_SUBSCRIBER_DISABLE=false
-TOKENS_SUBSCRIBER_DISABLE=false
+## Subscquid based subscribers
+#ACCOUNTS_SUBSCRIBER_DISABLE=true
+#BLOCKS_SUBSCRIBER_DISABLE=true
+#COLLECTIONS_SUBSCRIBER_DISABLE=true
+#TOKENS_SUBSCRIBER_DISABLE=true
 
 #Environment variables for tests
 TESTS_UNIQUE_WS_ENDPOINT=wss://ws-rc.unique.network
 TESTS_GRAPHQL_URL='http://localhost:3031/v1/graphql'
 TESTS_IPFS_URL=https://ipfs.unique.network/ipfs/
+
+SENTRY_DSN=SENTRY_DSN
+SENTRY_DEBUG=0
+SENTRY_LOG_LEVELS='error,warning'

.github/workflows/deploy-to-demo.yml

@@ -0,0 +1,17 @@
+name: Deploy to Demo VM
+
+on:
+  push:
+    branches:
+      - demo
+  workflow_dispatch:
+
+jobs:
+  run-gitlab-pipeline:
+    name: Run gitlab pipeline
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger gitlab
+        uses: wei/curl@v1
+        with:
+          args: -X POST -F token=${{ secrets.UNIQUE_GITLAB_TOKEN_DEMO }} -F ref=master -F variables[REMOTE_COMMIT]=${GITHUB_SHA} -F variables[DEPLOY_TO]=stage https://gitlab.uniquenetwork.dev/api/v4/projects/57/trigger/pipeline

.gitignore

@@ -31,6 +31,7 @@ lerna-debug.log*
 .vscode/*
 !.vscode/settings.json
 !.vscode/tasks.json
-!.vscode/launch.json
 !.vscode/extensions.json
-.env
+.env*
+!.env.default
+docker-compose.local.yml

Dockerfile.crawler

@@ -0,0 +1,8 @@
+FROM node:16.14-alpine3.14
+RUN mkdir /app
+WORKDIR /app
+COPY . /app/
+RUN npm install
+RUN npm run build:crawler
+
+CMD ["node", "-r", "source-map-support/register", "dist/apps/crawler/main.js"]

apps/crawler/src/crawler.module.ts

@@ -1,15 +1,29 @@
 import typeormConfig from '@common/typeorm.config';
 import { Logger, Module } from '@nestjs/common';
-import { ConfigModule } from '@nestjs/config';
+import { ConfigModule, ConfigService } from '@nestjs/config';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { ProcessorConfigService } from './processor.config.service';
 import { CrawlerService } from './crawler.service';
 import { SubscribersModule } from './subscribers/subscribers.module';
+import { SentryModule } from '@ntegral/nestjs-sentry';
 
 @Module({
   imports: [
     ConfigModule.forRoot(),
     TypeOrmModule.forRoot(typeormConfig),
+    SentryModule.forRootAsync({
+      imports: [ConfigModule],
+      useFactory: async (config: ConfigService) => ({
+        dsn: config.get('SENTRY_DSN'),
+        debug: config.get('SENTRY_DEBUG') === '1',
+        environment: process.env.NODE_ENV ?? 'development',
+        logLevels: config.get('SENTRY_LOG_LEVELS')
+          ? config.get('SENTRY_LOG_LEVELS').split(',')
+          : ['error'],
+        enabled: !!config.get('SENTRY_DSN'),
+      }),
+      inject: [ConfigService],
+    }),
     SubscribersModule,
   ],
   controllers: [],

apps/crawler/src/crawler.service.ts

@@ -17,7 +17,7 @@ export class CrawlerService {
     private tokensSubscriberService: TokensSubscriberService,
   ) {}
 
-  async subscribe(forceRescan = false) {
+  run(forceRescan = false) {
     if (this.configService.get('ACCOUNTS_SUBSCRIBER_DISABLE') !== 'true') {
       this.accountsSubscriberService.subscribe();
     }

apps/crawler/src/main.ts

@@ -1,25 +1,30 @@
+import { Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
 import { NestFactory } from '@nestjs/core';
 import { CrawlerModule } from './crawler.module';
 import { CrawlerService } from './crawler.service';
 
 async function bootstrap() {
-  const app = await NestFactory.create(CrawlerModule, {
-    logger: [
-      'log',
-      'error',
-      'warn',
-      'verbose',
-      //
-    ],
-  });
+  const app = await NestFactory.create(CrawlerModule);
+
+  const configService = app.get(ConfigService);
+
+  const logLevels = configService.get('LOG_LEVELS')
+    ? configService.get('LOG_LEVELS').split(',')
+    : ['log', 'error', 'warn'];
+
+  Logger.overrideLogger(logLevels);
 
   try {
     const crawlerService = app.get(CrawlerService);
 
-    await crawlerService.subscribe(process.env.SCAN_FORCE_RESCAN === 'true');
+    await crawlerService.run(process.env.SCAN_FORCE_RESCAN === 'true');
   } catch (err) {
     // eslint-disable-next-line no-console
     console.error(err);
   }
 }
-bootstrap();
+
+(async function () {
+  await bootstrap();
+})();

apps/crawler/src/processor.config.service.ts

@@ -37,4 +37,16 @@ export class ProcessorConfigService {
       typesBundle: this.getTypesBundle(),
     };
   }
+
+  public getForceMode() {
+    return this.configService.get('SCAN_FORCE_RESCAN');
+  }
+
+  public getPrometheusPort(): number {
+    return this.configService.get('PROMETHEUS_PORT', 9090);
+  }
+
+  public getBatchSize(): number {
+    return Number(this.configService.get('BATCH_SIZE', 10));
+  }
 }

apps/crawler/src/sdk/factory.ts

@@ -0,0 +1,11 @@
+import { Sdk } from '@unique-nft/substrate-client';
+
+export async function sdkFactory(chainWsUrl: string): Promise<Sdk> {
+  const sdk = new Sdk({
+    chainWsUrl,
+  });
+
+  await sdk.connect();
+
+  return sdk;
+}

apps/crawler/src/sdk/sdk.module.ts

@@ -0,0 +1,24 @@
+import { Global, Module } from '@nestjs/common';
+import { ConfigModule, ConfigService } from '@nestjs/config';
+import { sdkFactory } from './factory';
+import { SdkService } from './sdk.service';
+import { Sdk } from '@unique-nft/substrate-client';
+import '@unique-nft/substrate-client/extrinsics';
+import '@unique-nft/substrate-client/tokens';
+import '@unique-nft/substrate-client/balance';
+
+@Global()
+@Module({
+  imports: [ConfigModule],
+  providers: [
+    {
+      provide: Sdk,
+      useFactory: async (configService: ConfigService) =>
+        sdkFactory(configService.get('CHAIN_WS_URL')),
+      inject: [ConfigService],
+    },
+    SdkService,
+  ],
+  exports: [SdkService],
+})
+export class SdkModule {}

apps/crawler/src/sdk/sdk.service.ts

@@ -0,0 +1,41 @@
+import { Injectable } from '@nestjs/common';
+import { Sdk } from '@unique-nft/substrate-client';
+import {
+  CollectionInfoWithSchema,
+  TokenByIdResult,
+  TokenPropertiesResult,
+} from '@unique-nft/substrate-client/tokens';
+
+@Injectable()
+export class SdkService {
+  constructor(private sdk: Sdk) {}
+
+  getCollection(
+    collectionId: number,
+  ): Promise<CollectionInfoWithSchema | null> {
+    return this.sdk.collections.get_new({ collectionId });
+  }
+
+  async getCollectionLimits(collectionId: number) {
+    const result = await this.sdk.collections.getLimits({ collectionId });
+    return result?.limits;
+  }
+
+  getToken(
+    collectionId: number,
+    tokenId: number,
+  ): Promise<TokenByIdResult | null> {
+    return this.sdk.tokens.get_new({ collectionId, tokenId });
+  }
+
+  getTokenProperties(
+    collectionId: number,
+    tokenId: number,
+  ): Promise<TokenPropertiesResult | null> {
+    return this.sdk.tokens.properties({ collectionId, tokenId });
+  }
+
+  getBalances(rawAddress: string) {
+    return this.sdk.balance.get({ address: rawAddress });
+  }
+}