Monitor module prometheus metrics

apps/crawler/src/crawler.module.ts

@@ -8,11 +8,13 @@ import { SubscribersModule } from './subscribers/subscribers.module';
 import { Config, GlobalConfigModule } from './config/config.module';
 import { CacheProviderModule } from './cache/cache-provider.module';
 import { HarvesterModule, HarvesterModuleOptions } from '@unique-nft/harvester';
+import { MonitoringModule } from '@common/monitoring';
 
 @Module({
   imports: [
     GlobalConfigModule,
     CacheProviderModule,
+    MonitoringModule,
     TypeOrmModule.forRoot(typeormConfig),
     HarvesterModule.registerAsync({
       useFactory: (config: ConfigService<Config>) =>

apps/crawler/src/main.ts

@@ -1,3 +1,4 @@
+import { startMetricsServer } from '@common/monitoring';
 import { Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { NestFactory } from '@nestjs/core';
@@ -13,7 +14,7 @@ async function bootstrap() {
   const logLevels = configService.get('logLevels');
 
   Logger.overrideLogger(logLevels);
-
+  await startMetricsServer(app);
   await app.init();
 
   try {

apps/crawler/src/services/token/token.service.ts

@@ -557,28 +557,4 @@ export class TokenService {
       });
     }
   }
-
-  private async checkAndSaveOrUpdateTokenOwnerPart(tokenOwner: TokenOwnerData) {
-    const ownerToken = await this.tokensOwnersRepository.findOne({
-      where: {
-        owner: tokenOwner.owner,
-        collection_id: tokenOwner.collection_id,
-        token_id: tokenOwner.token_id,
-      },
-    });
-    if (ownerToken != null) {
-      await this.tokensOwnersRepository.update(
-        {
-          id: ownerToken.id,
-        },
-        {
-          amount: tokenOwner.amount,
-          block_number: tokenOwner.block_number,
-          type: tokenOwner.type,
-        },
-      );
-    } else {
-      await this.tokensOwnersRepository.save(tokenOwner);
-    }
-  }
 }

common/monitoring/controllers/health.controller.ts

@@ -0,0 +1,12 @@
+import { Controller, Get, VERSION_NEUTRAL } from '@nestjs/common';
+import { HealthService } from '../health';
+
+@Controller({ path: 'health', version: VERSION_NEUTRAL })
+export class HealthController {
+  constructor(private readonly health: HealthService) {}
+
+  @Get()
+  check() {
+    return this.health.check();
+  }
+}

common/monitoring/controllers/metrics.controller.ts

@@ -0,0 +1,18 @@
+import { PrometheusController } from '@willsoto/nestjs-prometheus';
+import { Controller, Get, Res, VERSION_NEUTRAL } from '@nestjs/common';
+import { Response } from 'express';
+import { PrometheusHealthService } from '../health';
+
+@Controller({ path: 'metrics', version: VERSION_NEUTRAL })
+export class MetricsController extends PrometheusController {
+  constructor(private prometheusHealthService: PrometheusHealthService) {
+    super();
+  }
+
+  @Get()
+  async index(@Res() response: Response): Promise<string> {
+    await this.prometheusHealthService.refresh();
+
+    return await super.index(response);
+  }
+}

common/monitoring/controllers/noop.metrics.controller.ts

@@ -0,0 +1,4 @@
+import { Controller } from '@nestjs/common';
+
+@Controller()
+export class NoopMetricsController {}

common/monitoring/health/health-indicators/base.health-indicator.ts

@@ -0,0 +1,40 @@
+import {
+  HealthCheckError,
+  HealthIndicator,
+  HealthIndicatorResult,
+} from '@nestjs/terminus';
+import { HealthItem } from '../../types';
+
+export abstract class BaseHealthIndicator extends HealthIndicator {
+  abstract key: string;
+
+  abstract check(): Promise<HealthIndicatorResult>;
+
+  async isHealthy(): Promise<HealthItem> {
+    const value = await this.check()
+      .then(() => true)
+      .catch(() => false);
+
+    return { key: this.key, value };
+  }
+
+  protected async onDisabled(): Promise<HealthIndicatorResult> {
+    return super.getStatus(this.key, false, { message: 'disabled' });
+  }
+
+  protected disable() {
+    this.check = this.onDisabled;
+  }
+
+  protected getStatus(
+    key: string,
+    isHealthy: boolean,
+    data?: Record<string, any>,
+  ): HealthIndicatorResult {
+    const status = super.getStatus(key, isHealthy, data);
+
+    if (isHealthy) return status;
+
+    throw new HealthCheckError(key, status);
+  }
+}

common/monitoring/health/health-indicators/index.ts

@@ -0,0 +1 @@
+export * from './redis.health-indicator';

common/monitoring/health/health-indicators/redis.health-indicator.ts

@@ -0,0 +1,68 @@
+import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
+import { HealthIndicatorResult } from '@nestjs/terminus';
+import { Cache } from 'cache-manager';
+import { ConfigService } from '@nestjs/config';
+
+import { BaseHealthIndicator } from './base.health-indicator';
+
+// todo - rather meaningless indicator, 'cause app crashes on redis connection lost
+
+export enum CacheType {
+  DEFAULT = 'Default',
+  REDIS = 'Redis',
+}
+
+interface CacheConfigBase {
+  type: CacheType;
+  ttl: number;
+}
+
+export interface DefaultCacheConfig extends CacheConfigBase {
+  type: CacheType.DEFAULT;
+}
+
+export interface RedisCacheConfig extends CacheConfigBase {
+  type: CacheType.REDIS;
+  host: string;
+  port: number;
+  db: number;
+}
+
+export type CacheConfig = DefaultCacheConfig | RedisCacheConfig;
+
+@Injectable()
+export class RedisHealthIndicator extends BaseHealthIndicator {
+  key = 'redis';
+
+  private readonly config: RedisCacheConfig;
+
+  constructor(
+    @Inject(CACHE_MANAGER)
+    private readonly cache: Cache,
+    configService: ConfigService,
+  ) {
+    super();
+    const config = configService.get<CacheConfig>('cache');
+
+    if (config.type === CacheType.REDIS) {
+      this.config = config;
+    } else {
+      this.disable();
+    }
+  }
+
+  private tryGet(): Promise<boolean> {
+    return this.cache
+      .get(Math.random().toString())
+      .then(() => true)
+      .catch(() => false);
+  }
+
+  async check(): Promise<HealthIndicatorResult> {
+    const { host, port, db } = this.config;
+
+    const isHealthy = await this.tryGet();
+
+    return this.getStatus(this.key, isHealthy, { host, port, db });
+  }
+}

common/monitoring/health/health.service.ts

@@ -0,0 +1,48 @@
+import { Injectable } from '@nestjs/common';
+import { totalmem } from 'os';
+import {
+  HealthCheck,
+  HealthCheckResult,
+  HealthCheckService,
+  MemoryHealthIndicator,
+} from '@nestjs/terminus';
+import {
+  RedisHealthIndicator,
+} from './health-indicators';
+import { HealthItem } from '../types';
+
+const memoryRSS = 'memory_RSS';
+const rssThreshold = totalmem() * 0.9;
+
+@Injectable()
+export class HealthService {
+  constructor(
+    private health: HealthCheckService,
+    private redis: RedisHealthIndicator,
+    private memory: MemoryHealthIndicator,
+  ) {}
+
+  getStatuses(): Promise<HealthItem[]> {
+    return Promise.all([
+      this.redis.isHealthy(),
+      this.isMemoryHealthy(),
+    ]);
+  }
+
+  private async isMemoryHealthy(): Promise<HealthItem> {
+    const value = await this.memory
+      .checkRSS(memoryRSS, rssThreshold)
+      .then(() => true)
+      .catch(() => true);
+
+    return { key: memoryRSS, value };
+  }
+
+  @HealthCheck()
+  public async check(): Promise<HealthCheckResult> {
+    return this.health.check([
+      () => this.redis.check(),
+      () => this.memory.checkRSS(memoryRSS, rssThreshold),
+    ]);
+  }
+}

common/monitoring/health/index.ts

@@ -0,0 +1,4 @@
+export { HealthService } from './health.service';
+export { PrometheusHealthService } from './prometheus.health.service';
+
+export * from './health-indicators';

common/monitoring/health/prometheus.health.service.ts

@@ -0,0 +1,23 @@
+import { Injectable } from '@nestjs/common';
+import { InjectMetric } from '@willsoto/nestjs-prometheus';
+import { Gauge } from 'prom-client';
+
+import { HealthService } from './health.service';
+import { HEALTH_METRIC } from '../metrics';
+
+@Injectable()
+export class PrometheusHealthService {
+  constructor(
+    private readonly healthService: HealthService,
+    @InjectMetric(HEALTH_METRIC)
+    private readonly healthGauge: Gauge,
+  ) {}
+
+  async refresh(): Promise<void> {
+    const results = await this.healthService.getStatuses();
+
+    results.forEach(({ key, value }) => {
+      this.healthGauge.labels(key).set(value ? 1 : 0);
+    });
+  }
+}

common/monitoring/index.ts

@@ -0,0 +1,3 @@
+export * from './metrics';
+export { startMetricsServer } from './monitoring.server';
+export { MonitoringModule } from './monitoring.module';

common/monitoring/metrics/current-tracking-extrinsics.metric.ts

@@ -0,0 +1,8 @@
+import { makeGaugeProvider } from '@willsoto/nestjs-prometheus';
+
+export const CURRENT_TRACKING_EXTRINSICS_METRIC = 'current_tracking_extrinsics';
+
+export const CurrentTrackingExtrinsicsMetric = makeGaugeProvider({
+  name: CURRENT_TRACKING_EXTRINSICS_METRIC,
+  help: 'Count of current tracking extrinsics',
+});

common/monitoring/metrics/health.metric.ts

@@ -0,0 +1,9 @@
+import { makeGaugeProvider } from '@willsoto/nestjs-prometheus';
+
+export const HEALTH_METRIC = 'health_checks';
+
+export const HealthMetric = makeGaugeProvider({
+  name: HEALTH_METRIC,
+  help: 'Health status - key is service, 1 for ok, 0 for error',
+  labelNames: ['key'],
+});

common/monitoring/metrics/index.ts

@@ -0,0 +1,4 @@
+export * from './current-tracking-extrinsics.metric';
+export * from './total-tracking-extrinsics.metric';
+export * from './requests.metric';
+export * from './health.metric';

common/monitoring/metrics/requests.metric.ts

@@ -0,0 +1,10 @@
+import { makeHistogramProvider } from '@willsoto/nestjs-prometheus';
+
+export const REQUESTS_METRIC = 'http_requests';
+
+export const RequestsMetric = makeHistogramProvider({
+  name: REQUESTS_METRIC,
+  help: 'HTTP requests - Duration in seconds',
+  labelNames: ['method', 'status', 'path', 'use'],
+  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 10],
+});

common/monitoring/metrics/total-tracking-extrinsics.metric.ts

@@ -0,0 +1,8 @@
+import { makeCounterProvider } from '@willsoto/nestjs-prometheus';
+
+export const TOTAL_TRACKING_EXTRINSICS_METRIC = 'total_tracking_extrinsics';
+
+export const TotalTrackingExtrinsicsMetric = makeCounterProvider({
+  name: TOTAL_TRACKING_EXTRINSICS_METRIC,
+  help: 'Count of total tracking extrinsics',
+});

common/monitoring/middleware/requests.middleware.ts

@@ -0,0 +1,61 @@
+import { Injectable, Logger, NestMiddleware } from '@nestjs/common';
+import { InjectMetric } from '@willsoto/nestjs-prometheus';
+import { Histogram } from 'prom-client';
+import responseTime from 'response-time';
+
+import { REQUESTS_METRIC } from '../metrics';
+
+@Injectable()
+export class RequestsMiddleware implements NestMiddleware {
+  private readonly excluded: string[] = ['/favicon.ico', '/metrics'];
+
+  private readonly defaultUse = 'NA';
+
+  private readonly logger = new Logger(RequestsMiddleware.name);
+
+  constructor(
+    @InjectMetric(REQUESTS_METRIC)
+    private readonly requestsHistogram: Histogram,
+  ) {}
+
+  use(req, res, next) {
+    responseTime((request, response, time) => {
+      const { url = 'unknown_url', method } = request;
+
+      const [path, query] = url.split('?');
+      if (this.excluded.includes(path)) return;
+
+      const status = RequestsMiddleware.normalizeStatus(res.statusCode);
+      const use = this.getUse(query);
+
+      const labels = { method, status, path, use };
+
+      this.requestsHistogram.observe(labels, time / 1000);
+    })(req, res, next);
+  }
+
+  private static normalizeStatus(statusCode: number): string {
+    if (statusCode >= 100 && statusCode < 200) return '1xx';
+    if (statusCode >= 200 && statusCode < 300) return '2xx';
+    if (statusCode >= 300 && statusCode < 400) return '3xx';
+    if (statusCode >= 400 && statusCode < 500) return '4xx';
+
+    return '5xx';
+  }
+
+  private getUse(query?: string): string {
+    try {
+      if (!query) return this.defaultUse;
+
+      const params = query.split('&').map((val) => val.split('='));
+
+      const useParam = params.find((param) => param[0] === 'use');
+
+      return useParam ? useParam[1] : this.defaultUse;
+    } catch (error) {
+      this.logger.warn(`Failed to parse query "${query}": ${error.message}`);
+
+      return this.defaultUse;
+    }
+  }
+}