Added Redis sharding and its test case

Gemfile

@@ -31,6 +31,7 @@ group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console
   # gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
   gem 'rspec-rails', '~> 3.5'
+  gem 'rubocop', '~> 0.49.1', require: false
 end
 
 group :development do
@@ -47,5 +48,10 @@ group :test do
   gem 'database_cleaner'
 end
 
+group :coverage do
+  gem 'simplecov', :require => false
+  gem 'simplecov-json', :require => false
+end
+
 # Windows does not include zoneinfo files, so bundle the tzinfo-data gem
 gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]

Gemfile.lock

@@ -39,10 +39,12 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     arel (8.0.0)
+    ast (2.3.0)
     builder (3.2.3)
     concurrent-ruby (1.0.5)
     database_cleaner (1.6.1)
     diff-lcs (1.3)
+    docile (1.1.5)
     erubi (1.6.1)
     factory_girl (4.8.0)
       activesupport (>= 3.0.0)
@@ -55,6 +57,7 @@ GEM
     globalid (0.4.0)
       activesupport (>= 4.2.0)
     i18n (0.8.6)
+    json (2.1.0)
     listen (3.1.5)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
@@ -73,6 +76,10 @@ GEM
     nio4r (2.1.0)
     nokogiri (1.8.0)
       mini_portile2 (~> 2.2.0)
+    parallel (1.12.0)
+    parser (2.4.0.0)
+      ast (~> 2.2)
+    powerpack (0.1.1)
     puma (3.9.1)
     rack (2.0.3)
     rack-test (0.6.3)
@@ -100,6 +107,8 @@ GEM
       method_source
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
+    rainbow (2.2.2)
+      rake
     rake (12.0.0)
     rb-fsevent (0.10.2)
     rb-inotify (0.9.10)
@@ -124,9 +133,25 @@ GEM
       rspec-mocks (~> 3.6.0)
       rspec-support (~> 3.6.0)
     rspec-support (3.6.0)
+    rubocop (0.49.1)
+      parallel (~> 1.10)
+      parser (>= 2.3.3.1, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.8.1)
     ruby_dep (1.5.0)
     shoulda-matchers (3.1.2)
       activesupport (>= 4.0.0)
+    simplecov (0.14.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.1)
+    simplecov-json (0.2)
+      json
+      simplecov
     spring (2.0.2)
       activesupport (>= 4.2)
     spring-watcher-listen (2.0.1)
@@ -143,6 +168,7 @@ GEM
     thread_safe (0.3.6)
     tzinfo (1.2.3)
       thread_safe (~> 0.1)
+    unicode-display_width (1.3.0)
     websocket-driver (0.6.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.2)
@@ -161,7 +187,10 @@ DEPENDENCIES
   redis
   redis-namespace
   rspec-rails (~> 3.5)
+  rubocop (~> 0.49.1)
   shoulda-matchers (~> 3.1)
+  simplecov
+  simplecov-json
   spring
   spring-watcher-listen (~> 2.0.0)
   tzinfo-data

app/controllers/urlinfos_controller.rb

@@ -1,7 +1,7 @@
 class UrlinfosController < ApplicationController
   before_action :set_urlinfo, only: [:show, :update, :destroy]
-  before_action :set_urlinfo_from_cache, only: [:find_by_url]
-  before_action :set_urlinfo_from_db, only: [:delete_by_url]
+  before_action :fetch_urlinfo_from_storage, only: [:find_by_url]
+  before_action :fetch_urlinfo_from_database, only: [:delete_by_url]
 
   def index
     @urlinfos = Urlinfo.all
@@ -23,18 +23,30 @@ def destroy
   end
 
   def find_by_url
-    json_response(@urlinfo)
+    json_response(@result)
   end
 
   def create_by_url
     param_url = generate_url_from_params 
-    @urlinfo = Urlinfo.where(domain_name: params[:domain_name], query_string: params[:query_string]).first
+    Urlinfo.make_empty_response_object(param_url)
+    @urlinfo = Urlinfo.where(domain_name: params[:domain_name],
+                             query_string: params[:query_string]).first
     if @urlinfo.nil?
-      @urlinfo = Urlinfo.create!(:url => param_url, :malware => true, :created_by => params[:requested_by], :domain_name => params[:domain_name], :query_string => params[:query_string])
+      @urlinfo = Urlinfo.create!(:url => param_url,
+                                 :malware => true,
+                                 :created_by => params[:requested_by],
+                                 :domain_name => params[:domain_name],
+                                 :query_string => params[:query_string])
     else
-      @urlinfo.update(:url => param_url, :malware => true, :created_by => params[:requested_by], :domain_name => params[:domain_name], :query_string => params[:query_string])
+      @urlinfo.update(:url => param_url,
+                      :malware => true,
+                      :created_by => params[:requested_by],
+                      :domain_name => params[:domain_name],
+                      :query_string => params[:query_string])
     end
-    json_response(@urlinfo, :created)
+    @result[:data_from] = "database"
+    @result[:malware] = true 
+    json_response(@result, :created)
   end
 
   def delete_by_url
@@ -48,16 +60,47 @@ def set_urlinfo
     @urlinfo = Urlinfo.find(params[:id])
   end
 
-  def set_urlinfo_from_cache
+  def fetch_urlinfo_from_storage
+    # 1. generating full uri from domain_name and query_string
+    #     cache stores full uri as key and malware flag as value
     param_url = generate_url_from_params
-    @urlinfo = Urlinfo.get_url_in_cache(param_url)
-    if @urlinfo.nil?
-      set_urlinfo_from_db
+    # 2. making empty response object with full uri
+    make_empty_response_object(param_url)
+    # 3. looking up the requested uri in cache first
+    cached_value = fetch_urlinfo_from_cache(param_url)
+    if cached_value.nil?
+      # 4. if not found in cache then looking it up in database
+      fetch_urlinfo_from_database(params)
+    end
+  end
+
+  def fetch_urlinfo_from_database(params)
+    @urlinfo = Urlinfo.where(domain_name: params[:domain_name], query_string: params[:query_string]).first
+    malware_flag = @urlinfo.nil? ? false : true
+    @result[:data_from] = "database"
+    @result[:malware] = malware_flag
+    param_url = generate_url_from_params
+    Urlinfo.set_url_in_cache(param_url, malware_flag)
+  end
+
+  def fetch_urlinfo_from_cache(param_url)
+    cached_value = Urlinfo.get_url_in_cache(param_url)
+    if cached_value.nil?
+      return nil
+    else
+      # updating cached url again to prevent the data from being expired (LRU)
+      Urlinfo.set_url_in_cache(param_url, cached_value)
+      @result[:malware] = cached_value == "true" ? true : false
+      @result[:data_from] = "cache"
     end
   end
 
-  def set_urlinfo_from_db
-    @urlinfo = Urlinfo.where(domain_name: params[:domain_name], query_string: params[:query_string])
+  def make_empty_response_object(param_url)
+    @result = {
+      :url => param_url,
+      :malware => nil,
+      :data_from => nil 
+    }
   end
 
   def generate_url_from_params

app/models/urlinfo.rb

@@ -5,22 +5,31 @@ class Urlinfo < ApplicationRecord
   after_save :set_url_as_malware_in_cache
   after_destroy :set_url_as_clean_in_cache
 
-  def self.get_url_in_cache(url)
-    $redis.get(url)
-    $redis[url] = false
-  end
-
   def set_url_as_malware_in_cache
-    redis[self.url] = true
+    cache = Urlinfo.getCacheConnectionByUrl(self.url)
+    cache[self.url] = true
   end
 
   def set_url_as_clean_in_cache
-    redis[self.url] = false
+    cache = Urlinfo.getCacheConnectionByUrl(self.url)
+    cache[self.url] = false
+  end
+
+  def self.get_url_in_cache(url)
+    cache = getCacheConnectionByUrl(url)
+    cache.get(url)
+  end
+
+  def self.set_url_in_cache(url, malware_flag)
+    cache = getCacheConnectionByUrl(url)
+    cache[url] = malware_flag
   end
 
   private
 
-  def redis
-    $redis
+  def self.getCacheConnectionByUrl(url)
+    hashcode = (url.length % UrlLookupCache.redis.length)
+puts "url = #{url.length}, number of redis= #{UrlLookupCache.redis.length} hashcode = #{hashcode}"
+    UrlLookupCache.redis[hashcode][:object]
   end
 end

config/config.yml

@@ -0,0 +1,14 @@
+development: &default
+  redis:
+    - :instance_id: redis_dev
+test:
+  redis:
+    - :instance_id: redis_test1
+    - :instance_id: redis_test2
+production:
+  redis:
+    - :instance_id: redis_cluster1
+    - :instance_id: redis_cluster2
+    - :instance_id: redis_cluster3
+default:
+  <<: *default

config/database.yml

@@ -1,29 +1,15 @@
-# SQLite version 3.x
-#   gem install mysql
-#
-#   Ensure the SQLite 3 gem is defined in your Gemfile
-#   gem 'mysql'
-#
-default: &default
+development: &default
   adapter: mysql2
   encoding: utf8
+  database: urllookup
   username: root
   password:
   port: 3306
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
   timeout: 5000
-
-development:
-  <<: *default
-  database: urllookup
-
-# Warning: The database defined as "test" will be erased and
-# re-generated from your development database when you run "rake".
-# Do not set this db to the same as development or production.
 test:
   <<: *default
   database: urllookup_test
-
 production:
   <<: *default
   database: urllookup

config/environment.rb

@@ -1,5 +1,14 @@
 # Load the Rails application.
 require_relative 'application'
 
+app_config = YAML.load(File.read("#{Rails.root}/config/config.yml"))[Rails.env]
+redis_instances = app_config["redis"]
+
+REDIS_INSTANCE_LIST = {}
+redis_instances.each do |redis_instance|
+  redis_instance_info = YAML.load(File.read("#{Rails.root}/config/redis.yml"))[redis_instance[:instance_id]]
+  REDIS_INSTANCE_LIST[redis_instance_info['hashcode']] = { :object => nil, :url => redis_instance_info['url'] }
+end
+
 # Initialize the Rails application.
 Rails.application.initialize!

config/initializers/redis.rb

@@ -1 +1,10 @@
-$redis = Redis::Namespace.new(:ns, :redis => Redis.new)
+module UrlLookupCache
+  class << self
+    def redis
+      REDIS_INSTANCE_LIST.each do |key, redis_instance|
+        redis_instance[:object] ||= Redis.new(:url => (redis_instance[:url] || 'redis://127.0.0.1:6379'))
+      end
+      REDIS_INSTANCE_LIST
+    end
+  end
+end

config/redis.yml

@@ -0,0 +1,18 @@
+redis_dev:
+  url: redis://localhost:6379
+  hashcode: 0
+redis_test1:
+  url: redis://localhost:6379
+  hashcode: 0
+redis_test2:
+  url: redis://localhost:6379
+  hashcode: 1
+redis_cluster1:
+  url: redis://localhost:6379
+  hashcode: 0
+redis_cluster2:
+  url: redis://localhost:6379
+  hashcode: 1
+redis_cluster3:
+  url: redis://localhost:6379
+  hashcode: 2

coverage/.last_run.json

@@ -0,0 +1,5 @@
+{
+  "result": {
+    "covered_percent": 89.66
+  }
+}