Merge pull request #142 from UniqueNetwork/release/v1.0.1

Release/v1.0.1

.env.default

@@ -10,23 +10,37 @@ ARCHIVE_GQL_URL=https://archive.dev.uniquenetwork.dev/graphql
 #ARCHIVE_GQL_URL=https://archive.opal.uniquenetwork.dev/graphql
 CHAIN_WS_URL=wss://ws-rc.unique.network
 SCAN_TYPES_BUNDLE=quartz
-SCAN_RANGE_FROM_DEFAULT=0
-SCAN_RANGE_FROM=111000
-# SCAN_RANGE_TO=1000000
-SCAN_FORCE_RESCAN=true
+#SCAN_RANGE_FROM=111000
+#SCAN_RANGE_TO=1000000
+#SCAN_FORCE_RESCAN=true
 PROMETHEUS_PORT=3003
+BATCH_SIZE=10
 
 ## Subscquid based subscribers
+
 #ACCOUNTS_SUBSCRIBER_DISABLE=true
 #BLOCKS_SUBSCRIBER_DISABLE=true
 #COLLECTIONS_SUBSCRIBER_DISABLE=true
 #TOKENS_SUBSCRIBER_DISABLE=true
 
-#Environment variables for tests
+## Environment variables for tests
+
 TESTS_UNIQUE_WS_ENDPOINT=wss://ws-rc.unique.network
 TESTS_GRAPHQL_URL='http://localhost:3031/v1/graphql'
 TESTS_IPFS_URL=https://ipfs.unique.network/ipfs/
 
+## Sentry variables
+
 SENTRY_DSN=SENTRY_DSN
 SENTRY_DEBUG=0
 SENTRY_LOG_LEVELS='error,warning'
+
+## Cache variables
+
+### Cache ttl in seconds.
+CACHE_TTL=60
+
+### Redis cache connection. If not set, memory cache is in use.
+#REDIS_HOST=
+#REDIS_PORT=
+#REDIS_DB=

apps/crawler/src/cache/cache-provider.module.ts

@@ -0,0 +1,40 @@
+import { CACHE_MANAGER, Global, Module, Provider } from '@nestjs/common';
+import * as redisStore from 'cache-manager-redis-store';
+import { ConfigService } from '@nestjs/config';
+import { caching, Cache } from 'cache-manager';
+import { CacheConfig, CacheType } from '../config/cache.config';
+
+const cacheProvider: Provider = {
+  inject: [ConfigService],
+  provide: CACHE_MANAGER,
+  useFactory: (configService: ConfigService): Cache => {
+    const cacheConfig: CacheConfig = configService.get('cache');
+
+    switch (cacheConfig.type) {
+      case CacheType.DEFAULT:
+        return caching({
+          ttl: cacheConfig.ttl,
+
+          store: 'memory',
+        });
+      case CacheType.REDIS:
+        return caching({
+          ttl: cacheConfig.ttl,
+
+          store: redisStore,
+          host: cacheConfig.host,
+          port: cacheConfig.port,
+          db: cacheConfig.db,
+        });
+      default:
+        throw new Error('Invalid cache config');
+    }
+  },
+};
+
+@Global()
+@Module({
+  providers: [cacheProvider],
+  exports: [CACHE_MANAGER],
+})
+export class CacheProviderModule {}

apps/crawler/src/config/cache.config.ts

@@ -0,0 +1,42 @@
+export enum CacheType {
+  DEFAULT = 'Default',
+  REDIS = 'Redis',
+}
+
+interface CacheConfigBase {
+  type: CacheType;
+  ttl: number;
+}
+
+export interface DefaultCacheConfig extends CacheConfigBase {
+  type: CacheType.DEFAULT;
+}
+
+export interface RedisCacheConfig extends CacheConfigBase {
+  type: CacheType.REDIS;
+  host: string;
+  port: number;
+  db: number;
+}
+
+export type CacheConfig = DefaultCacheConfig | RedisCacheConfig;
+
+export function createCacheConfig(env: Record<string, string>): CacheConfig {
+  const { CACHE_TTL, REDIS_HOST, REDIS_PORT, REDIS_DB } = env;
+  const ttl = +CACHE_TTL || 600;
+
+  if (REDIS_HOST) {
+    return {
+      type: CacheType.REDIS,
+      host: REDIS_HOST,
+      port: +REDIS_PORT || 6379,
+      db: +REDIS_DB || 0,
+      ttl,
+    };
+  }
+
+  return {
+    type: CacheType.DEFAULT,
+    ttl,
+  };
+}

apps/crawler/src/config/config.module.ts

@@ -0,0 +1,67 @@
+import * as process from 'process';
+import { ConfigModule } from '@nestjs/config';
+import { CacheConfig, createCacheConfig } from './cache.config';
+import { SentryConfig, createSentryConfig } from './sentry.config';
+import {
+  SubscribersConfig,
+  createSubscribersConfig,
+} from './subscribers.config';
+
+export type Config = {
+  logLevels: Array<string>;
+
+  chainWsUrl: string;
+
+  archiveGqlUrl: string;
+
+  sentry: SentryConfig;
+
+  cache: CacheConfig;
+
+  subscribers: SubscribersConfig;
+
+  scanTypesBundle: string;
+
+  scanRangeFrom: number;
+
+  scanRangeTo?: number;
+
+  rescan: boolean;
+
+  prometheusPort: number;
+
+  batchSize: number;
+};
+
+const loadConfig = (): Config => ({
+  logLevels: process.env.LOG_LEVELS
+    ? process.env.LOG_LEVELS.split(',')
+    : ['log', 'error', 'warn'],
+
+  chainWsUrl: process.env.CHAIN_WS_URL,
+
+  archiveGqlUrl: process.env.ARCHIVE_GQL_URL,
+
+  sentry: createSentryConfig(process.env),
+
+  cache: createCacheConfig(process.env),
+
+  subscribers: createSubscribersConfig(process.env),
+
+  scanTypesBundle: process.env.SCAN_TYPES_BUNDLE || 'quartz',
+
+  scanRangeFrom: +process.env.SCAN_RANGE_FROM || 0,
+
+  scanRangeTo: +process.env.SCAN_RANGE_TO || undefined,
+
+  rescan: process.env.SCAN_FORCE_RESCAN === 'true',
+
+  prometheusPort: +process.env.PROMETHEUS_PORT || 9090,
+
+  batchSize: +process.env.BATCH_SIZE || 10,
+});
+
+export const GlobalConfigModule = ConfigModule.forRoot({
+  isGlobal: true,
+  load: [loadConfig],
+});

apps/crawler/src/processor.config.service.ts → apps/crawler/src/config/processor.config.service.ts

@@ -2,28 +2,28 @@ import { Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { DataSource as SubscquidDataSource } from '@subsquid/substrate-processor';
 import { Range } from '@subsquid/substrate-processor/lib/util/range';
+import { Config } from './config.module';
 
 @Injectable()
 export class ProcessorConfigService {
-  constructor(private configService: ConfigService) {}
+  constructor(private configService: ConfigService<Config>) {}
 
   public getDataSource(): SubscquidDataSource {
     return {
-      archive: this.configService.get('ARCHIVE_GQL_URL'),
-      chain: this.configService.get('CHAIN_WS_URL'),
+      archive: this.configService.get('archiveGqlUrl'),
+      chain: this.configService.get('chainWsUrl'),
     };
   }
 
   public getRange(): Range {
-    const to = this.configService.get('SCAN_RANGE_TO');
     return {
-      from: this.configService.get('SCAN_RANGE_FROM', 0),
-      to,
+      from: this.configService.get('scanRangeFrom'),
+      to: this.configService.get('scanRangeTo'),
     };
   }
 
   public getTypesBundle(): string {
-    return this.configService.get('SCAN_TYPES_BUNDLE');
+    return this.configService.get('scanTypesBundle');
   }
 
   public getAllParams(): {
@@ -38,15 +38,15 @@ export class ProcessorConfigService {
     };
   }
 
-  public getForceMode() {
-    return this.configService.get('SCAN_FORCE_RESCAN');
+  public isRescan() {
+    return this.configService.get('rescan');
   }
 
   public getPrometheusPort(): number {
-    return this.configService.get('PROMETHEUS_PORT', 9090);
+    return this.configService.get('prometheusPort');
   }
 
   public getBatchSize(): number {
-    return Number(this.configService.get('BATCH_SIZE', 10));
+    return this.configService.get('batchSize');
   }
 }

apps/crawler/src/config/sentry.config.ts

@@ -0,0 +1,19 @@
+export type SentryConfig = {
+  dsn: string;
+  debug: boolean;
+  environment: string;
+  logLevels: string[];
+  enabled: boolean;
+};
+
+export function createSentryConfig(env: Record<string, string>): SentryConfig {
+  const { NODE_ENV, SENTRY_DSN, SENTRY_DEBUG, SENTRY_LOG_LEVELS } = env;
+
+  return {
+    dsn: SENTRY_DSN,
+    debug: SENTRY_DEBUG === '1',
+    environment: NODE_ENV ?? 'development',
+    logLevels: SENTRY_LOG_LEVELS ? SENTRY_LOG_LEVELS.split(',') : ['error'],
+    enabled: !!SENTRY_DSN,
+  };
+}

apps/crawler/src/config/subscribers.config.ts

@@ -0,0 +1,21 @@
+import { SubscriberName } from '@common/constants';
+
+export type SubscribersConfig = { [key in SubscriberName]: boolean };
+
+export function createSubscribersConfig(
+  env: Record<string, string>,
+): SubscribersConfig {
+  const {
+    ACCOUNTS_SUBSCRIBER_DISABLE,
+    BLOCKS_SUBSCRIBER_DISABLE,
+    COLLECTIONS_SUBSCRIBER_DISABLE,
+    TOKENS_SUBSCRIBER_DISABLE,
+  } = env;
+
+  return {
+    [SubscriberName.ACCOUNTS]: ACCOUNTS_SUBSCRIBER_DISABLE !== 'true',
+    [SubscriberName.BLOCKS]: BLOCKS_SUBSCRIBER_DISABLE !== 'true',
+    [SubscriberName.COLLECTIONS]: COLLECTIONS_SUBSCRIBER_DISABLE !== 'true',
+    [SubscriberName.TOKENS]: TOKENS_SUBSCRIBER_DISABLE !== 'true',
+  };
+}

apps/crawler/src/crawler.module.ts

@@ -1,32 +1,27 @@
-import typeormConfig from '@common/typeorm.config';
-import { Logger, Module } from '@nestjs/common';
-import { ConfigModule, ConfigService } from '@nestjs/config';
+import { Module } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
 import { TypeOrmModule } from '@nestjs/typeorm';
-import { ProcessorConfigService } from './processor.config.service';
+import { SentryModule } from '@ntegral/nestjs-sentry';
+import typeormConfig from '@common/typeorm.config';
 import { CrawlerService } from './crawler.service';
 import { SubscribersModule } from './subscribers/subscribers.module';
-import { SentryModule } from '@ntegral/nestjs-sentry';
+import { Config, GlobalConfigModule } from './config/config.module';
+import { CacheProviderModule } from './cache/cache-provider.module';
 
 @Module({
   imports: [
-    ConfigModule.forRoot(),
+    GlobalConfigModule,
+    CacheProviderModule,
     TypeOrmModule.forRoot(typeormConfig),
     SentryModule.forRootAsync({
-      imports: [ConfigModule],
-      useFactory: async (config: ConfigService) => ({
-        dsn: config.get('SENTRY_DSN'),
-        debug: config.get('SENTRY_DEBUG') === '1',
-        environment: process.env.NODE_ENV ?? 'development',
-        logLevels: config.get('SENTRY_LOG_LEVELS')
-          ? config.get('SENTRY_LOG_LEVELS').split(',')
-          : ['error'],
-        enabled: !!config.get('SENTRY_DSN'),
-      }),
+      useFactory: async (configService: ConfigService<Config>) => {
+        return configService.get('sentry');
+      },
       inject: [ConfigService],
     }),
     SubscribersModule,
   ],
   controllers: [],
-  providers: [Logger, CrawlerService, ProcessorConfigService],
+  providers: [CrawlerService],
 })
 export class CrawlerModule {}

apps/crawler/src/crawler.service.ts

@@ -1,39 +1,11 @@
 import { Injectable } from '@nestjs/common';
-import { ConfigService } from '@nestjs/config';
-import { ProcessorService } from './subscribers/processor.service';
-import { AccountsSubscriberService } from './subscribers/accounts-subscriber.service';
-import { BlocksSubscriberService } from './subscribers/blocks-subscriber.service';
-import { CollectionsSubscriberService } from './subscribers/collections-subscriber.service';
-import { TokensSubscriberService } from './subscribers/tokens-subscriber.service';
+import { SubscribersService } from './subscribers/subscribers.service';
 
 @Injectable()
 export class CrawlerService {
-  constructor(
-    private configService: ConfigService,
-    private processorService: ProcessorService,
-    private accountsSubscriberService: AccountsSubscriberService,
-    private blocksSubscriberService: BlocksSubscriberService,
-    private collectionsSubscriberService: CollectionsSubscriberService,
-    private tokensSubscriberService: TokensSubscriberService,
-  ) {}
+  constructor(private subscribersService: SubscribersService) {}
 
-  run(forceRescan = false) {
-    if (this.configService.get('ACCOUNTS_SUBSCRIBER_DISABLE') !== 'true') {
-      this.accountsSubscriberService.subscribe();
-    }
-
-    if (this.configService.get('BLOCKS_SUBSCRIBER_DISABLE') !== 'true') {
-      this.blocksSubscriberService.subscribe();
-    }
-
-    if (this.configService.get('COLLECTIONS_SUBSCRIBER_DISABLE') !== 'true') {
-      this.collectionsSubscriberService.subscribe();
-    }
-
-    if (this.configService.get('TOKENS_SUBSCRIBER_DISABLE') !== 'true') {
-      this.tokensSubscriberService.subscribe();
-    }
-
-    return this.processorService.run(forceRescan);
+  run() {
+    return this.subscribersService.run();
   }
 }