Merge pull request #6 from DmitriySh/feature/monitoring-3

Feature/monitoring 3

.gitignore

@@ -12,4 +12,4 @@
 
 # Project logs
 *.log
-*/build_info.txt
+*build_info.txt

swarm/.env.example

@@ -0,0 +1,26 @@
+# Custom variables for a services in Docker Composed
+MONGO_VERSION=3.2
+NODE_EXPORTER_VERSION=v0.15.0
+POST_VERSION=latest
+COMMENT_VERSION=latest
+UI_VERSION=latest
+CADVISOR_VERSION=latest
+PROMETHEUS_VERSION=latest
+
+BACK_NET=10.0.2.0/24
+FRONT_NET=10.0.1.0/24
+
+UI_HOST_PORT=9292
+UI_CONTAINER_PORT=9292
+CADV_HOST_PORT=8080
+CADV_CONTAINER_PORT=8080
+GF_HOST_PORT=3000
+GF_CONTAINER_PORT=3000
+ALERTM_HOST_PORT=9093
+ALERTM_CONTAINER_PORT=9093
+PROM_HOST_PORT=9090
+PROM_CONTAINER_PORT=9090
+
+# Grafana auth
+GF_SECURITY_ADMIN_USER=admin
+GF_SECURITY_ADMIN_PASSWORD=admin

swarm/comment/Dockerfile

@@ -0,0 +1,17 @@
+FROM ruby:2.2
+
+RUN apt-get update -qq && apt-get install -y build-essential
+
+ENV APP_HOME /app
+RUN mkdir $APP_HOME
+WORKDIR $APP_HOME
+
+ADD Gemfile* $APP_HOME/
+RUN bundle install
+
+ADD . $APP_HOME
+
+ENV COMMENT_DATABASE_HOST comment_db
+ENV COMMENT_DATABASE comments
+
+CMD ["puma"]

swarm/comment/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+gem 'sinatra'
+gem 'bson_ext'
+gem 'mongo'
+gem 'puma'
+gem 'prometheus-client'
+gem 'rack'
+gem 'rufus-scheduler'

swarm/comment/Gemfile.lock

@@ -0,0 +1,43 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    bson (4.2.2)
+    bson_ext (1.5.1)
+    et-orbi (1.0.8)
+      tzinfo
+    mongo (2.4.3)
+      bson (>= 4.2.1, < 5.0.0)
+    mustermann (1.0.0)
+    prometheus-client (0.7.1)
+      quantile (~> 0.2.0)
+    puma (3.10.0)
+    quantile (0.2.0)
+    rack (2.0.3)
+    rack-protection (2.0.0)
+      rack
+    rufus-scheduler (3.4.2)
+      et-orbi (~> 1.0)
+    sinatra (2.0.0)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.0)
+      tilt (~> 2.0)
+    thread_safe (0.3.6)
+    tilt (2.0.8)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bson_ext
+  mongo
+  prometheus-client
+  puma
+  rack
+  rufus-scheduler
+  sinatra
+
+BUNDLED WITH
+   1.15.4

swarm/comment/VERSION

@@ -0,0 +1 @@
+0.0.3

swarm/comment/comment_app.rb

@@ -0,0 +1,63 @@
+require 'sinatra'
+require 'json/ext' # for .to_json
+require 'uri'
+require 'mongo'
+require './helpers'
+require 'prometheus/client'
+require 'rufus-scheduler'
+
+mongo_host = ENV['COMMENT_DATABASE_HOST'] || '127.0.0.1'
+mongo_port = ENV['COMMENT_DATABASE_PORT'] || '27017'
+mongo_database = ENV['COMMENT_DATABASE'] || 'test'
+
+
+# Create and register metrics
+prometheus = Prometheus::Client.registry
+
+comment_health_gauge = Prometheus::Client::Gauge.new(:comment_health, 'Health status of Comment service')
+comment_health_db_gauge = Prometheus::Client::Gauge.new(:comment_health_mongo_availability, 'Check if MongoDB is available to Comment')
+comment_count = Prometheus::Client::Counter.new(:comment_count, 'A counter of new comments')
+prometheus.register(comment_health_gauge)
+prometheus.register(comment_health_db_gauge)
+prometheus.register(comment_count)
+
+## Schedule healthcheck function
+build_info=File.readlines('build_info.txt')
+
+scheduler = Rufus::Scheduler.new
+
+scheduler.every '3s' do
+  check = JSON.parse(healthcheck(mongo_host, mongo_port))
+  comment_health_gauge.set({ version: check['version'].strip, commit_hash: build_info[0].strip, branch: build_info[1].strip }, check['status'])
+  comment_health_db_gauge.set({ version: check['version'].strip, commit_hash: build_info[0].strip, branch: build_info[1].strip }, check['dependent_services']['commentdb'])
+end
+
+
+configure do
+  db = Mongo::Client.new(["#{mongo_host}:#{mongo_port}"], database: mongo_database, heartbeat_frequency: 2)
+  set :mongo_db, db[:comments]
+  set :bind, '0.0.0.0'
+end
+
+get '/:id/comments' do
+  id   = obj_id(params[:id])
+  settings.mongo_db.find(post_id: "#{id}").to_a.to_json
+end
+
+post '/add_comment/?' do
+  content_type :json
+  halt 400, json(error: 'No name provided') if params['name'].nil? or params['name'].empty?
+  halt 400, json(error: 'No comment provided') if params['body'].nil? or params['body'].empty?
+  db = settings.mongo_db
+  result = db.insert_one post_id: params['post_id'], name: params['name'], email: params['email'], body: params['body'], created_at: params['created_at']
+  db.find(_id: result.inserted_id).to_a.first.to_json
+  comment_count.increment
+end
+
+get '/healthcheck' do
+  healthcheck(mongo_host, mongo_port)
+end
+
+get '/*' do |request|
+  halt 404
+end

swarm/comment/config.ru

@@ -0,0 +1,10 @@
+require './comment_app'
+require 'rack'
+require 'prometheus/middleware/collector'
+require 'prometheus/middleware/exporter'
+
+use Rack::Deflater, if: ->(_, _, _, body) { body.any? && body[0].length > 512 }
+use Prometheus::Middleware::Collector
+use Prometheus::Middleware::Exporter
+
+run Sinatra::Application

swarm/comment/docker_build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+echo `git show --format="%h" HEAD | head -1` > build_info.txt
+echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt
+
+docker build -t $USERNAME/comment .

swarm/comment/helpers.rb

@@ -0,0 +1,44 @@
+
+def obj_id(val)
+  begin
+    BSON::ObjectId.from_string(val)
+  rescue BSON::ObjectId::Invalid
+    nil
+  end
+end
+
+def document_by_id(id)
+  id = obj_id(id) if String === id
+  if id.nil?
+    {}.to_json
+  else
+    document = settings.mongo_db.find(:_id => id).to_a.first
+    (document || {}).to_json
+  end
+end
+
+def healthcheck(mongo_host, mongo_port)
+  begin
+    commentdb_test = Mongo::Client.new(["#{mongo_host}:#{mongo_port}"], server_selection_timeout: 2)
+    commentdb_test.database_names
+    commentdb_test.close
+  rescue
+    commentdb_status = 0
+  else
+    commentdb_status = 1
+  end
+
+  status = commentdb_status
+
+  version = File.read('VERSION')
+
+  healthcheck = {
+    status: status,
+    dependent_services: {
+      commentdb: commentdb_status
+    },
+    version: version
+  }
+
+  healthcheck.to_json
+end

swarm/config/alert.rules.yml

@@ -0,0 +1,11 @@
+groups:
+  - name: alert.rules
+    rules:
+    - alert: InstanceDown
+      expr: up == 0
+      for: 1m
+      labels:
+        severity: page
+      annotations:
+        description: '{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minute'
+        summary: 'Instance {{ $labels.instance }} down'

swarm/config/config.yml

@@ -0,0 +1,11 @@
+global:
+  slack_api_url: 'https://hooks.slack.com/services/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'
+
+route:
+  receiver: 'slack-notifications'
+
+receivers:
+- name: 'slack-notifications'
+  slack_configs:
+  - channel: '#dmitriy-shishmakov'
+

swarm/config/prometheus.yml

@@ -0,0 +1,49 @@
+---
+global:
+  scrape_interval: '5s'
+
+rule_files:
+  - "alert.rules.yml"
+
+alerting:
+  alertmanagers:
+    - scheme: http
+      static_configs:
+        - targets:
+          - 'alertmanager:9093'
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets:
+        - 'localhost:9090'
+
+  - job_name: 'ui'
+    static_configs:
+      - targets:
+        - 'ui:9292'
+
+  - job_name: 'comment'
+    static_configs:
+      - targets:
+        - 'comment:9292'
+
+  - job_name: 'node'
+    static_configs:
+      - targets:
+        - 'node-exporter:9100'
+
+  - job_name: 'mongodb-exporter'
+    static_configs:
+      - targets:
+        - 'mongodb-exporter:9001'
+
+  - job_name: 'cadvisor'
+    static_configs:
+      - targets:
+        - 'cadvisor:8080'
+
+  - job_name: 'post'
+    static_configs:
+      - targets:
+        - 'post:5000'