Merge branch 'unstable'

Grokzen · Aug 19, 2019 · 840cb6f · 840cb6f
2 parents 4a499a9 + 9c1b763
commit 840cb6f
Show file tree

Hide file tree

Showing 41 changed files with 2,731 additions and 1,322 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,11 +1,12 @@
 sudo: false
+dist: xenial
 language: python
+cache: pip
 python:
   - "2.7"
-  - "3.3"
   - "3.4"
   - "3.5"
-  - "3.6-dev"
+  - "3.6"
   - "nightly"
 services:
   - redis-server
@@ -14,9 +15,12 @@ install:
   - "if [[ $REDIS_VERSION == '3.2' ]]; then REDIS_VERSION=3.2 make redis-install; fi"
   - "if [[ $REDIS_VERSION == '4.0' ]]; then REDIS_VERSION=4.0 make redis-install; fi"
   - "if [[ $REDIS_VERSION == '5.0' ]]; then REDIS_VERSION=5.0 make redis-install; fi"
+  - "if [[ $TEST_PYCODESTYLE == '1' ]]; then pip install pycodestyle; fi"
   - pip install -r dev-requirements.txt
   - pip install -e .
   - "if [[ $HIREDIS == '1' ]]; then pip install hiredis; fi"
+  - "pip freeze | grep redis"
+  - "pip freeze"
 env:
   # Redis 3.0 & HIREDIS
   - HIREDIS=0 REDIS_VERSION=3.0
@@ -30,14 +34,30 @@ env:
   # Redis 5.0 & HIREDIS
   - HIREDIS=0 REDIS_VERSION=5.0
   - HIREDIS=1 REDIS_VERSION=5.0
-script: 
+script:
   - make start
   - coverage erase
   - coverage run --source rediscluster -p -m py.test
+  - py.test
   - make stop
-after_success: 
+after_success:
   - coverage combine
   - coveralls
+  - "if [[ $TEST_PYCODESTYLE == '1' ]]; then pycodestyle --repeat --show-source --exclude=.venv,.tox,dist,docs,build,*.egg,redis_install .; fi"
 matrix:
   allow_failures:
     - python: "nightly"
+    - python: 2.7
+      env: TEST_PYCODESTYLE=1
+    - python: 3.6
+      env: TEST_PYCODESTYLE=1
+    # python 3.7 has to be specified manually in the matrix
+    # https://github.com/travis-ci/travis-ci/issues/9815
+    - python: 3.7
+      dist: xenial
+      sudo: true
+      env: TEST_HIREDIS=0
+    - python: 3.7
+      dist: xenial
+      sudo: true
+      env: TEST_HIREDIS=1
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,23 @@
+Copyright (c) 2014-2019 Johan Andersson
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+
diff --git a/Makefile b/Makefile
@@ -216,7 +216,7 @@ ifndef REDIS_TRIB_RB
 endif
 
 ifndef REDIS_VERSION
-	REDIS_VERSION=4.0.10
+	REDIS_VERSION=5.0.5
 endif
 
 export REDIS_CLUSTER_NODE1_CONF
@@ -378,16 +378,16 @@ redis-install:
 
 benchmark:
 	@echo ""
-	@echo " -- Running Simple benchmark with StrictRedis lib and non cluster server --"
+	@echo " -- Running Simple benchmark with Redis lib and non cluster server --"
 	python benchmarks/simple.py --port 7007 --timeit --nocluster
 	@echo ""
-	@echo " -- Running Simple benchmark with StrictRedisCluster lib and cluster server --"
+	@echo " -- Running Simple benchmark with RedisCluster lib and cluster server --"
 	python benchmarks/simple.py --port 7001 --timeit
 	@echo ""
-	@echo " -- Running Simple benchmark with pipelines & StrictRedis lib and non cluster server --"
+	@echo " -- Running Simple benchmark with pipelines & Redis lib and non cluster server --"
 	python benchmarks/simple.py --port 7007 --timeit --pipeline --nocluster
 	@echo ""
-	@echo " -- Running Simple benchmark with StrictRedisCluster lib and cluster server"
+	@echo " -- Running Simple benchmark with RedisCluster lib and cluster server"
 	python benchmarks/simple.py --port 7001 --timeit --pipeline
 
 ptp:

diff --git a/README.md b/README.md
@@ -4,8 +4,6 @@ This client provides a client for redis cluster that was added in redis 3.0.
 
 This project is a port of `redis-rb-cluster` by antirez, with alot of added functionality. The original source can be found at https://github.com/antirez/redis-rb-cluster
 
-Gitter chat room: [![Gitter](https://badges.gitter.im/Grokzen/redis-py-cluster.svg)](https://gitter.im/Grokzen/redis-py-cluster?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
-
 [![Build Status](https://travis-ci.org/Grokzen/redis-py-cluster.svg?branch=master)](https://travis-ci.org/Grokzen/redis-py-cluster) [![Coverage Status](https://coveralls.io/repos/Grokzen/redis-py-cluster/badge.png)](https://coveralls.io/r/Grokzen/redis-py-cluster) [![PyPI version](https://badge.fury.io/py/redis-py-cluster.svg)](http://badge.fury.io/py/redis-py-cluster)
 
 
@@ -30,19 +28,21 @@ Latest stable release from pypi
 $ pip install redis-py-cluster
 ```
 
+This major version of `redis-py-cluster` supports `redis-py>=3.0.0,<3.1.0`.
+
 
 
 ## Usage example
 
 Small sample script that shows how to get started with RedisCluster. It can also be found in [examples/basic.py](examples/basic.py)
 
 ```python
->>> from rediscluster import StrictRedisCluster
+>>> from rediscluster import RedisCluster
 
 >>> # Requires at least one node for cluster discovery. Multiple nodes is recommended.
 >>> startup_nodes = [{"host": "127.0.0.1", "port": "7000"}]
 
->>> rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
+>>> rc = RedisCluster(startup_nodes=startup_nodes, decode_responses=True)
 
 >>> rc.set("foo", "bar")
 True
@@ -54,7 +54,7 @@ True
 
 ## License & Authors
 
-Copyright (c) 2013-2018 Johan Andersson
+Copyright (c) 2013-2019 Johan Andersson
 
 MIT (See docs/License.txt file)
 

diff --git a/benchmarks/simple.py b/benchmarks/simple.py
@@ -1,11 +1,26 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+"""
+Usage:
+  redis-cluster-benchmark.py [--host <ip>] [-p <port>] [-n <request>] [-c <concurrent>] [--nocluster] [--timeit] [--pipeline] [--resetlastkey] [-h] [--version]
 
-# python std lib
-import time
+Options:
+  --host <ip>        Redis server to test against [default: 127.0.0.1]
+  -p <port>          Port on redis server [default: 7000]
+  -n <request>       Request number [default: 100000]
+  -c <concurrent>    Concurrent client number [default: 1]
+  --nocluster        If flag is set then Redis will be used instead of cluster lib
+  --timeit           Run a mini benchmark to test performance
+  --pipeline         Only usable with --timeit flag. Runs SET/GET inside pipelines.
+  --resetlastkey     Reset __last__ key
+  -h --help          Output this help and exit
+  --version          Output version and exit
+"""
 
+import time
+from multiprocessing import Process
 # 3rd party imports
 from docopt import docopt
-from redis._compat import xrange
 
 
 def loop(rc, reset_last_key=None):
@@ -25,7 +40,7 @@ def loop(rc, reset_last_key=None):
             print("error {0}".format(e))
             time.sleep(1)
 
-    for i in xrange(last, 1000000000):  # noqa
+    for i in range(last, 1000000000):  # noqa
         try:
             print("SET foo{0} {1}".format(i, i))
             rc.set("foo{0}".format(i), i)
@@ -38,78 +53,54 @@ def loop(rc, reset_last_key=None):
         time.sleep(0.05)
 
 
-def timeit(rc, itterations=50000):
+def timeit(rc, num):
     """
     Time how long it take to run a number of set/get:s
     """
-    t0 = time.time()
-    for i in xrange(0, itterations):  # noqa
+    for i in range(0, num//2):  # noqa
         s = "foo{0}".format(i)
         rc.set(s, i)
         rc.get(s)
 
-    t1 = time.time() - t0
-    print("{0}k SET/GET operations took: {1} seconds... {2} operations per second".format((itterations / 1000) * 2, t1, (itterations / t1) * 2))
 
-
-def timeit_pipeline(rc, itterations=50000):
+def timeit_pipeline(rc, num):
     """
     Time how long it takes to run a number of set/get:s inside a cluster pipeline
     """
-    t0 = time.time()
-    for i in xrange(0, itterations):  # noqa
+    for i in range(0, num//2):  # noqa
         s = "foo{0}".format(i)
-
         p = rc.pipeline()
         p.set(s, i)
         p.get(s)
         p.execute()
 
-    t1 = time.time() - t0
-    print("{0}k SET/GET operations inside pipelines took: {1} seconds... {2} operations per second".format(
-        (itterations / 1000) * 2, t1, (itterations / t1) * 2)
-    )
-
 
 if __name__ == "__main__":
-    __docopt__ = """
-Usage:
-  simple [--host IP] [--port PORT] [--nocluster] [--timeit] [--pipeline] [--resetlastkey] [-h] [--version]
-
-Options:
-  --nocluster        If flag is set then StrictRedis will be used instead of cluster lib
-  --host IP          Redis server to test against [default: 127.0.0.1]
-  --port PORT        Port on redis server [default: 7000]
-  --timeit           run a mini benchmark to test performance
-  --pipeline         Only usable with --timeit flag. Runs SET/GET inside pipelines.
-  --resetlastkey     reset __last__ key
-  -h --help          show this help and exit
-  -v --version       show version and exit
-    """
-
-    args = docopt(__docopt__, version="0.3.0")
-
-    startup_nodes = [{"host": args["--host"], "port": args["--port"]}]
+    args = docopt(__doc__, version="0.3.1")
+    startup_nodes = [{"host": args['--host'], "port": args['-p']}]
 
     if not args["--nocluster"]:
-        from rediscluster import StrictRedisCluster
-        rc = StrictRedisCluster(startup_nodes=startup_nodes, max_connections=32, socket_timeout=0.1, decode_responses=True)
+        from rediscluster import RedisCluster
+        rc = RedisCluster(startup_nodes=startup_nodes, max_connections=32, socket_timeout=0.1, decode_responses=True)
     else:
-        from redis import StrictRedis
-        rc = StrictRedis(host=args["--host"], port=args["--port"], socket_timeout=0.1, decode_responses=True)
-
-    if args["--timeit"]:
-        test_itterstions = [
-            5000,
-            10000,
-            20000,
-        ]
-
-        if args["--pipeline"]:
-            for itterations in test_itterstions:
-                timeit_pipeline(rc, itterations=itterations)
+        from redis import Redis
+        rc = Redis(host=args["--host"], port=args["-p"], socket_timeout=0.1, decode_responses=True)
+    # create specified number processes
+    processes = []
+    single_request = int(args["-n"]) // int(args["-c"])
+    for j in range(int(args["-c"])):
+        if args["--timeit"]:
+            if args["--pipeline"]:
+                p = Process(target=timeit_pipeline, args=(rc, single_request))
+            else:
+                p = Process(target=timeit, args=(rc, single_request))
         else:
-            for itterations in test_itterstions:
-                timeit(rc, itterations=itterations)
-    else:
-        loop(rc, reset_last_key=args["--resetlastkey"])
+            p = Process(target=loop, args=(rc, args["--resetlastkey"]))
+        processes.append(p)
+    t1 = time.time()
+    for p in processes:
+        p.start()
+    for p in processes:
+        p.join()
+    t2 = time.time() - t1
+    print("Tested {0}k SET & GET (each 50%) operations took: {1} seconds... {2} operations per second".format(int(args["-n"]) / 1000, t2, int(args["-n"]) / t2 * 2))
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,11 +1,12 @@
 -r requirements.txt
 
-coverage>=4.0,<5.0
-pytest>=2.8.3,<4.0.0
-testfixtures>=4.5.0,<5.5.0
-mock>=1.3.0,<2.1.0
-docopt>=0.6.2,<1.0.0
-tox>=2.2.0,<3.0.0
-python-coveralls>=2.5.0,<3.0.0
-ptpdb>=0.16,<1.0
-ptpython>=0.31,<1.0
+coverage
+pytest
+testfixtures
+mock
+docopt
+tox
+python-coveralls
+ptpdb
+ptpython
+pysnooper
diff --git a/docs/License.txt b/docs/License.txt
@@ -1,4 +1,4 @@
-Copyright (c) 2014-2016 Johan Andersson
+Copyright (c) 2014-2019 Johan Andersson
 
  Permission is hereby granted, free of charge, to any person
  obtaining a copy of this software and associated documentation

diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst
@@ -8,12 +8,12 @@ These are a few benchmarks that are designed to test specific parts of the code
 Setup benchmarks
 ----------------
 
-Before running any benchmark you should install this lib in editable mode inside a virtualenv so it can import `StrictRedisCluster` lib.
+Before running any benchmark you should install this lib in editable mode inside a virtualenv so it can import `RedisCluster` lib.
 
 Install with
 
 .. code-block:: bash
-    
+
     pip install -e .
 
 You also need a few redis servers to test against. You must have one cluster with at least one node on port `7001` and you must also have a non-clustered server on port `7007`.
@@ -42,26 +42,18 @@ Example output and comparison of different runmodes
 
 .. code-block::
 
-     -- Running Simple benchmark with StrictRedis lib and non cluster server --
-    python benchmarks/simple.py --port 7007 --timeit --nocluster
-    10.0k SET/GET operations took: 0.9711470603942871 seconds... 10297.10165208139 operations per second
-    20.0k SET/GET operations took: 1.9136295318603516 seconds... 10451.343725113202 operations per second
-    40.0k SET/GET operations took: 3.8409764766693115 seconds... 10414.018477584079 operations per second
-
-     -- Running Simple benchmark with StrictRedisCluster lib and cluster server --
-    python benchmarks/simple.py --port 7001 --timeit
-    10.0k SET/GET operations took: 0.760077714920044 seconds... 13156.549394494412 operations per second
-    20.0k SET/GET operations took: 1.5251967906951904 seconds... 13113.061948474155 operations per second
-    40.0k SET/GET operations took: 3.05112361907959 seconds... 13109.924406165655 operations per second
-
-     -- Running Simple benchmark with pipelines & StrictRedis lib and non cluster server --
-    python benchmarks/simple.py --port 7007 --timeit --pipeline --nocluster
-    10.0k SET/GET operations inside pipelines took: 0.8831894397735596 seconds... 11322.599149921782 operations per second
-    20.0k SET/GET operations inside pipelines took: 1.6283915042877197 seconds... 12282.058674058404 operations per second
-    40.0k SET/GET operations inside pipelines took: 3.2882907390594482 seconds... 12164.374495498905 operations per second
-
-     -- Running Simple benchmark with StrictRedisCluster lib and cluster server
-    python benchmarks/simple.py --port 7001 --timeit --pipeline
-    10.0k SET/GET operations inside pipelines took: 0.709221601486206 seconds... 14099.965340937933 operations per second
-    20.0k SET/GET operations inside pipelines took: 1.3776116371154785 seconds... 14517.879684783395 operations per second
-    40.0k SET/GET operations inside pipelines took: 2.794893980026245 seconds... 14311.813001087214 operations per second
+     -- Running Simple benchmark with Redis lib and non cluster server, 50 concurrent processes and total 50000*2 requests --
+    python benchmarks/simple.py --host 127.0.0.1 --timeit --nocluster -c 50 -n 50000
+    50.0k SET/GET operations took: 2.45 seconds... 40799.93 operations per second
+
+     -- Running Simple benchmark with RedisCluster lib and cluster server, 50 concurrent processes and total 50000*2 requests --
+    python benchmarks/simple.py --host 127.0.0.1 --timeit -c 50 -n 50000
+    50.0k SET & GET (each 50%) operations took: 9.51 seconds... 31513.71 operations per second
+
+     -- Running Simple benchmark with pipelines & Redis lib and non cluster server --
+    python benchmarks/simple.py --host 127.0.0.1 --timeit --nocluster -c 50 -n 50000 --pipeline
+    50.0k SET & GET (each 50%) operations took: 2.1728243827819824 seconds... 46023.047602201834 operations per second
+
+     -- Running Simple benchmark with RedisCluster lib and cluster server
+    python benchmarks/simple.py --host 127.0.0.1 --timeit -c 50 -n 50000 --pipeline
+    50.0k SET & GET (each 50%) operations took: 1.7181339263916016 seconds... 58202.68051514381 operations per second