Skip to content

Commit

Permalink
Backport 5.1 changes into master (#3406)
Browse files Browse the repository at this point in the history
* Updated package version (#3389)

* Fix bug with Redis Set commands returns List instead of Set (#3399)

* Fix bug with Redis Set commands returns List instead of Set in RESP2

* Removed fixture, codestyle fixes

* Fixed tests for async

* Fixed asyncio cluster test cases

* Added Sets alignment for RESP2 and RESP3

* Updated doctests

* Fix bug with partial Hiredis availability (#3400)

* Fix bug with partial Hiredis availability

* Added yes flag

* Codestyl fixes

* Removed redundant check

* Removed redundant checks associated with pack command

* Updated condition to check the actual flag

* Removed unused import

* Fix bug with async pipeline and cluster fails with some commands (#3402)

* Fix bug with async pipeline fails with some commands

* Codestyl changes

* Remove keys option in cluster context

* Updated package version (#3403)
  • Loading branch information
vladvildanov authored Oct 7, 2024
1 parent 2e46613 commit 700045c
Show file tree
Hide file tree
Showing 16 changed files with 131 additions and 83 deletions.
33 changes: 33 additions & 0 deletions .github/workflows/integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,21 @@ jobs:
invoke ${{matrix.test-type}}-tests
ls -1
- name: Run tests against hiredis < 3.0.0
if: ${{ matrix.connection-type == 'hiredis' && matrix.python-version == '3.12'}}
run: |
pip uninstall hiredis -y
pip install -U setuptools wheel
pip install -r requirements.txt
pip install -r dev_requirements.txt
if [ "${{matrix.connection-type}}" == "hiredis" ]; then
pip install "hiredis<3.0.0"
fi
invoke devenv
sleep 10 # time to settle
invoke ${{matrix.test-type}}-tests
ls -1
- name: Upload test results and profiling data
uses: actions/upload-artifact@v4
with:
Expand Down Expand Up @@ -145,6 +160,24 @@ jobs:
invoke ${{matrix.test-type}}-tests --protocol=3
fi
- name: Run tests against hiredis < 3.0.0
if: ${{ matrix.connection-type == 'hiredis' && matrix.python-version == '3.12'}}
run: |
pip uninstall hiredis -y
pip install -U setuptools wheel
pip install -r requirements.txt
pip install -r dev_requirements.txt
if [ "${{matrix.connection-type}}" == "hiredis" ]; then
pip install "hiredis<3.0.0"
fi
invoke devenv
sleep 10 # time to settle
if [ "${{matrix.event-loop}}" == "uvloop" ]; then
invoke ${{matrix.test-type}}-tests --uvloop --protocol=3
else
invoke ${{matrix.test-type}}-tests --protocol=3
fi
- name: Upload test results and profiling data
uses: actions/upload-artifact@v4
with:
Expand Down
34 changes: 17 additions & 17 deletions doctests/dt_set.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,11 @@
r.sadd("bikes:racing:usa", "bike:1", "bike:4")
# HIDE_END
res7 = r.sinter("bikes:racing:france", "bikes:racing:usa")
print(res7) # >>> ['bike:1']
print(res7) # >>> {'bike:1'}
# STEP_END

# REMOVE_START
assert res7 == ["bike:1"]
assert res7 == {"bike:1"}
# REMOVE_END

# STEP_START scard
Expand All @@ -83,12 +83,12 @@
print(res9) # >>> 3

res10 = r.smembers("bikes:racing:france")
print(res10) # >>> ['bike:1', 'bike:2', 'bike:3']
print(res10) # >>> {'bike:1', 'bike:2', 'bike:3'}
# STEP_END

# REMOVE_START
assert res9 == 3
assert res10 == ['bike:1', 'bike:2', 'bike:3']
assert res10 == {'bike:1', 'bike:2', 'bike:3'}
# REMOVE_END

# STEP_START smismember
Expand All @@ -109,11 +109,11 @@
r.sadd("bikes:racing:usa", "bike:1", "bike:4")

res13 = r.sdiff("bikes:racing:france", "bikes:racing:usa")
print(res13) # >>> ['bike:2', 'bike:3']
print(res13) # >>> {'bike:2', 'bike:3'}
# STEP_END

# REMOVE_START
assert res13 == ['bike:2', 'bike:3']
assert res13 == {'bike:2', 'bike:3'}
r.delete("bikes:racing:france")
r.delete("bikes:racing:usa")
# REMOVE_END
Expand All @@ -124,27 +124,27 @@
r.sadd("bikes:racing:italy", "bike:1", "bike:2", "bike:3", "bike:4")

res13 = r.sinter("bikes:racing:france", "bikes:racing:usa", "bikes:racing:italy")
print(res13) # >>> ['bike:1']
print(res13) # >>> {'bike:1'}

res14 = r.sunion("bikes:racing:france", "bikes:racing:usa", "bikes:racing:italy")
print(res14) # >>> ['bike:1', 'bike:2', 'bike:3', 'bike:4']
print(res14) # >>> {'bike:1', 'bike:2', 'bike:3', 'bike:4'}

res15 = r.sdiff("bikes:racing:france", "bikes:racing:usa", "bikes:racing:italy")
print(res15) # >>> []
print(res15) # >>> {}

res16 = r.sdiff("bikes:racing:usa", "bikes:racing:france")
print(res16) # >>> ['bike:4']
print(res16) # >>> {'bike:4'}

res17 = r.sdiff("bikes:racing:france", "bikes:racing:usa")
print(res17) # >>> ['bike:2', 'bike:3']
print(res17) # >>> {'bike:2', 'bike:3'}
# STEP_END

# REMOVE_START
assert res13 == ['bike:1']
assert res14 == ['bike:1', 'bike:2', 'bike:3', 'bike:4']
assert res15 == []
assert res16 == ['bike:4']
assert res17 == ['bike:2', 'bike:3']
assert res13 == {'bike:1'}
assert res14 == {'bike:1', 'bike:2', 'bike:3', 'bike:4'}
assert res15 == {}
assert res16 == {'bike:4'}
assert res17 == {'bike:2', 'bike:3'}
r.delete("bikes:racing:france")
r.delete("bikes:racing:usa")
r.delete("bikes:racing:italy")
Expand All @@ -160,7 +160,7 @@
print(res19) # >>> bike:3

res20 = r.smembers("bikes:racing:france")
print(res20) # >>> ['bike:2', 'bike:4', 'bike:5']
print(res20) # >>> {'bike:2', 'bike:4', 'bike:5'}

res21 = r.srandmember("bikes:racing:france")
print(res21) # >>> bike:4
Expand Down
6 changes: 6 additions & 0 deletions redis/_parsers/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,9 @@ def string_keys_to_dict(key_string, callback):


_RedisCallbacksRESP2 = {
**string_keys_to_dict(
"SDIFF SINTER SMEMBERS SUNION", lambda r: r and set(r) or set()
),
**string_keys_to_dict(
"ZDIFF ZINTER ZPOPMAX ZPOPMIN ZRANGE ZRANGEBYSCORE ZRANK ZREVRANGE "
"ZREVRANGEBYSCORE ZREVRANK ZUNION",
Expand Down Expand Up @@ -829,6 +832,9 @@ def string_keys_to_dict(key_string, callback):


_RedisCallbacksRESP3 = {
**string_keys_to_dict(
"SDIFF SINTER SMEMBERS SUNION", lambda r: r and set(r) or set()
),
**string_keys_to_dict(
"ZRANGE ZINTER ZPOPMAX ZPOPMIN ZRANGEBYSCORE ZREVRANGE ZREVRANGEBYSCORE "
"ZUNION HGETALL XREADGROUP",
Expand Down
4 changes: 4 additions & 0 deletions redis/asyncio/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1423,6 +1423,10 @@ async def _execute_transaction( # noqa: C901
if not isinstance(r, Exception):
args, options = cmd
command_name = args[0]

# Remove keys entry, it needs only for cache.
options.pop("keys", None)

if command_name in self.response_callbacks:
r = self.response_callbacks[command_name](r, **options)
if inspect.isawaitable(r):
Expand Down
4 changes: 4 additions & 0 deletions redis/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -1163,6 +1163,10 @@ def _execute_command(self, target_node, *args, **kwargs):
asking = False
connection.send_command(*args, **kwargs)
response = redis_node.parse_response(connection, command, **kwargs)

# Remove keys entry, it needs only for cache.
kwargs.pop("keys", None)

if command in self.cluster_response_callbacks:
response = self.cluster_response_callbacks[command](
response, **kwargs
Expand Down
3 changes: 1 addition & 2 deletions redis/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
from .utils import (
CRYPTOGRAPHY_AVAILABLE,
HIREDIS_AVAILABLE,
HIREDIS_PACK_AVAILABLE,
SSL_AVAILABLE,
compare_versions,
ensure_string,
Expand Down Expand Up @@ -314,7 +313,7 @@ def __del__(self):
def _construct_command_packer(self, packer):
if packer is not None:
return packer
elif HIREDIS_PACK_AVAILABLE:
elif HIREDIS_AVAILABLE:
return HiredisRespSerializer()
else:
return PythonRespSerializer(self._buffer_cutoff, self.encoder.encode)
Expand Down
4 changes: 2 additions & 2 deletions redis/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@

# Only support Hiredis >= 3.0:
HIREDIS_AVAILABLE = int(hiredis.__version__.split(".")[0]) >= 3
HIREDIS_PACK_AVAILABLE = hasattr(hiredis, "pack_command")
if not HIREDIS_AVAILABLE:
raise ImportError("hiredis package should be >= 3.0.0")
except ImportError:
HIREDIS_AVAILABLE = False
HIREDIS_PACK_AVAILABLE = False

try:
import ssl # noqa
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
long_description_content_type="text/markdown",
keywords=["Redis", "key-value store", "database"],
license="MIT",
version="5.1.0b7",
version="5.1.1",
packages=find_packages(
include=[
"redis",
Expand Down
20 changes: 10 additions & 10 deletions tests/test_asyncio/test_cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -1752,38 +1752,38 @@ async def test_cluster_rpoplpush(self, r: RedisCluster) -> None:

async def test_cluster_sdiff(self, r: RedisCluster) -> None:
await r.sadd("{foo}a", "1", "2", "3")
assert set(await r.sdiff("{foo}a", "{foo}b")) == {b"1", b"2", b"3"}
assert await r.sdiff("{foo}a", "{foo}b") == {b"1", b"2", b"3"}
await r.sadd("{foo}b", "2", "3")
assert await r.sdiff("{foo}a", "{foo}b") == [b"1"]
assert await r.sdiff("{foo}a", "{foo}b") == {b"1"}

async def test_cluster_sdiffstore(self, r: RedisCluster) -> None:
await r.sadd("{foo}a", "1", "2", "3")
assert await r.sdiffstore("{foo}c", "{foo}a", "{foo}b") == 3
assert set(await r.smembers("{foo}c")) == {b"1", b"2", b"3"}
assert await r.smembers("{foo}c") == {b"1", b"2", b"3"}
await r.sadd("{foo}b", "2", "3")
assert await r.sdiffstore("{foo}c", "{foo}a", "{foo}b") == 1
assert await r.smembers("{foo}c") == [b"1"]
assert await r.smembers("{foo}c") == {b"1"}

async def test_cluster_sinter(self, r: RedisCluster) -> None:
await r.sadd("{foo}a", "1", "2", "3")
assert await r.sinter("{foo}a", "{foo}b") == []
assert await r.sinter("{foo}a", "{foo}b") == set()
await r.sadd("{foo}b", "2", "3")
assert set(await r.sinter("{foo}a", "{foo}b")) == {b"2", b"3"}
assert await r.sinter("{foo}a", "{foo}b") == {b"2", b"3"}

async def test_cluster_sinterstore(self, r: RedisCluster) -> None:
await r.sadd("{foo}a", "1", "2", "3")
assert await r.sinterstore("{foo}c", "{foo}a", "{foo}b") == 0
assert await r.smembers("{foo}c") == []
assert await r.smembers("{foo}c") == set()
await r.sadd("{foo}b", "2", "3")
assert await r.sinterstore("{foo}c", "{foo}a", "{foo}b") == 2
assert set(await r.smembers("{foo}c")) == {b"2", b"3"}
assert await r.smembers("{foo}c") == {b"2", b"3"}

async def test_cluster_smove(self, r: RedisCluster) -> None:
await r.sadd("{foo}a", "a1", "a2")
await r.sadd("{foo}b", "b1", "b2")
assert await r.smove("{foo}a", "{foo}b", "a1")
assert await r.smembers("{foo}a") == [b"a2"]
assert set(await r.smembers("{foo}b")) == {b"b1", b"b2", b"a1"}
assert await r.smembers("{foo}a") == {b"a2"}
assert await r.smembers("{foo}b") == {b"b1", b"b2", b"a1"}

async def test_cluster_sunion(self, r: RedisCluster) -> None:
await r.sadd("{foo}a", "1", "2")
Expand Down
20 changes: 10 additions & 10 deletions tests/test_asyncio/test_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -1415,34 +1415,34 @@ async def test_scard(self, r: redis.Redis):
@pytest.mark.onlynoncluster
async def test_sdiff(self, r: redis.Redis):
await r.sadd("a", "1", "2", "3")
assert set(await r.sdiff("a", "b")) == {b"1", b"2", b"3"}
assert await r.sdiff("a", "b") == {b"1", b"2", b"3"}
await r.sadd("b", "2", "3")
assert await r.sdiff("a", "b") == [b"1"]
assert await r.sdiff("a", "b") == {b"1"}

@pytest.mark.onlynoncluster
async def test_sdiffstore(self, r: redis.Redis):
await r.sadd("a", "1", "2", "3")
assert await r.sdiffstore("c", "a", "b") == 3
assert set(await r.smembers("c")) == {b"1", b"2", b"3"}
assert await r.smembers("c") == {b"1", b"2", b"3"}
await r.sadd("b", "2", "3")
assert await r.sdiffstore("c", "a", "b") == 1
assert await r.smembers("c") == [b"1"]
assert await r.smembers("c") == {b"1"}

@pytest.mark.onlynoncluster
async def test_sinter(self, r: redis.Redis):
await r.sadd("a", "1", "2", "3")
assert await r.sinter("a", "b") == []
assert await r.sinter("a", "b") == set()
await r.sadd("b", "2", "3")
assert set(await r.sinter("a", "b")) == {b"2", b"3"}
assert await r.sinter("a", "b") == {b"2", b"3"}

@pytest.mark.onlynoncluster
async def test_sinterstore(self, r: redis.Redis):
await r.sadd("a", "1", "2", "3")
assert await r.sinterstore("c", "a", "b") == 0
assert await r.smembers("c") == []
assert await r.smembers("c") == set()
await r.sadd("b", "2", "3")
assert await r.sinterstore("c", "a", "b") == 2
assert set(await r.smembers("c")) == {b"2", b"3"}
assert await r.smembers("c") == {b"2", b"3"}

async def test_sismember(self, r: redis.Redis):
await r.sadd("a", "1", "2", "3")
Expand All @@ -1460,8 +1460,8 @@ async def test_smove(self, r: redis.Redis):
await r.sadd("a", "a1", "a2")
await r.sadd("b", "b1", "b2")
assert await r.smove("a", "b", "a1")
assert await r.smembers("a") == [b"a2"]
assert set(await r.smembers("b")) == {b"b1", b"b2", b"a1"}
assert await r.smembers("a") == {b"a2"}
assert await r.smembers("b") == {b"b1", b"b2", b"a1"}

async def test_spop(self, r: redis.Redis):
s = [b"1", b"2", b"3"]
Expand Down
10 changes: 10 additions & 0 deletions tests/test_asyncio/test_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -417,3 +417,13 @@ async def test_pipeline_discard(self, r):
response = await pipe.execute()
assert response[0]
assert await r.get("foo") == b"bar"

@pytest.mark.onlynoncluster
async def test_send_set_commands_over_async_pipeline(self, r: redis.asyncio.Redis):
pipe = r.pipeline()
pipe.hset("hash:1", "foo", "bar")
pipe.hset("hash:1", "bar", "foo")
pipe.hset("hash:1", "baz", "bar")
pipe.hgetall("hash:1")
resp = await pipe.execute()
assert resp == [1, 1, 1, {b"bar": b"foo", b"baz": b"bar", b"foo": b"bar"}]
2 changes: 1 addition & 1 deletion tests/test_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def r(request):

@pytest.mark.skipif(HIREDIS_AVAILABLE, reason="PythonParser only")
@pytest.mark.onlynoncluster
# @skip_if_resp_version(2)
@skip_if_resp_version(2)
@skip_if_server_version_lt("7.4.0")
class TestCache:
@pytest.mark.parametrize(
Expand Down
Loading

0 comments on commit 700045c

Please sign in to comment.