From 0f54001c4582789ae32ff596d714e6a8b407f0d0 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Wed, 4 Oct 2023 15:44:57 +0200 Subject: [PATCH 1/3] Added `real_adapter_send` parameter to `RequestsMock` (#671) * Added `real_adapter_send` parameter to `RequestsMock` that will allow users to set through which function they would like to send real requests --- CHANGES | 2 + responses/__init__.py | 88 ++++++++++++++++++------------- responses/tests/test_responses.py | 28 ++++++++++ 3 files changed, 82 insertions(+), 36 deletions(-) diff --git a/CHANGES b/CHANGES index 59a59252..2bb40959 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ 0.24.0 ------ +* Added `real_adapter_send` parameter to `RequestsMock` that will allow users to set + through which function they would like to send real requests * Added support for re.Pattern based header matching. * Added support for gzipped response bodies to `json_params_matcher`. * Moved types-pyyaml dependency to `tests_requires` diff --git a/responses/__init__.py b/responses/__init__.py index a89e2d7e..ea7009dd 100644 --- a/responses/__init__.py +++ b/responses/__init__.py @@ -49,7 +49,8 @@ try: from requests.packages.urllib3.connection import HTTPHeaderDict except ImportError: # pragma: no cover - from urllib3.response import HTTPHeaderDict # type: ignore[attr-defined] + from urllib3.response import HTTPHeaderDict + try: from requests.packages.urllib3.util.url import parse_url except ImportError: # pragma: no cover @@ -84,14 +85,25 @@ def __call__( ) -> models.Response: ... - -# Block of type annotations -_Body = Union[str, BaseException, "Response", BufferedReader, bytes, None] -_F = Callable[..., Any] -_HeaderSet = Optional[Union[Mapping[str, str], List[Tuple[str, str]]]] -_MatcherIterable = Iterable[Callable[..., Tuple[bool, str]]] -_HTTPMethodOrResponse = Optional[Union[str, "BaseResponse"]] -_URLPatternType = Union["Pattern[str]", str] + # Block of type annotations + _Body = Union[str, BaseException, "Response", BufferedReader, bytes, None] + _F = Callable[..., Any] + _HeaderSet = Optional[Union[Mapping[str, str], List[Tuple[str, str]]]] + _MatcherIterable = Iterable[Callable[..., Tuple[bool, str]]] + _HTTPMethodOrResponse = Optional[Union[str, "BaseResponse"]] + _URLPatternType = Union["Pattern[str]", str] + _HTTPAdapterSend = Callable[ + [ + HTTPAdapter, + PreparedRequest, + bool, + float | tuple[float, float] | tuple[float, None] | None, + bool | str, + bytes | str | tuple[bytes | str, bytes | str] | None, + Mapping[str, str] | None, + ], + models.Response, + ] Call = namedtuple("Call", ["request", "response"]) @@ -250,7 +262,7 @@ def __getitem__(self, idx: slice) -> List[Call]: def __getitem__(self, idx: Union[int, slice]) -> Union[Call, List[Call]]: return self._calls[idx] - def add(self, request: "PreparedRequest", response: _Body) -> None: + def add(self, request: "PreparedRequest", response: "_Body") -> None: self._calls.append(Call(request, response)) def reset(self) -> None: @@ -258,8 +270,8 @@ def reset(self) -> None: def _ensure_url_default_path( - url: _URLPatternType, -) -> _URLPatternType: + url: "_URLPatternType", +) -> "_URLPatternType": """Add empty URL path '/' if doesn't exist. Examples @@ -376,7 +388,7 @@ class BaseResponse: def __init__( self, method: str, - url: _URLPatternType, + url: "_URLPatternType", match_querystring: Union[bool, object] = None, match: "_MatcherIterable" = (), *, @@ -384,7 +396,7 @@ def __init__( ) -> None: self.method: str = method # ensure the url has a default path set if the url is a string - self.url: _URLPatternType = _ensure_url_default_path(url) + self.url: "_URLPatternType" = _ensure_url_default_path(url) if self._should_match_querystring(match_querystring): match = tuple(match) + ( @@ -434,7 +446,7 @@ def _should_match_querystring( return bool(urlsplit(self.url).query) - def _url_matches(self, url: _URLPatternType, other: str) -> bool: + def _url_matches(self, url: "_URLPatternType", other: str) -> bool: """Compares two URLs. Compares only scheme, netloc and path. If 'url' is a re.Pattern, then checks that @@ -532,8 +544,8 @@ class Response(BaseResponse): def __init__( self, method: str, - url: _URLPatternType, - body: _Body = "", + url: "_URLPatternType", + body: "_Body" = "", json: Optional[Any] = None, status: int = 200, headers: Optional[Mapping[str, str]] = None, @@ -556,7 +568,7 @@ def __init__( else: content_type = "text/plain" - self.body: _Body = body + self.body: "_Body" = body self.status: int = status self.headers: Optional[Mapping[str, str]] = headers @@ -608,7 +620,7 @@ class CallbackResponse(BaseResponse): def __init__( self, method: str, - url: _URLPatternType, + url: "_URLPatternType", callback: Callable[[Any], Any], stream: Optional[bool] = None, content_type: Optional[str] = "text/plain", @@ -678,6 +690,8 @@ def __init__( passthru_prefixes: Tuple[str, ...] = (), target: str = "requests.adapters.HTTPAdapter.send", registry: Type[FirstMatchRegistry] = FirstMatchRegistry, + *, + real_adapter_send: "_HTTPAdapterSend" = _real_send, ) -> None: self._calls: CallList = CallList() self.reset() @@ -688,6 +702,7 @@ def __init__( self.target: str = target self._patcher: Optional["_mock_patcher[Any]"] = None self._thread_lock = _ThreadingLock() + self._real_send = real_adapter_send def get_registry(self) -> FirstMatchRegistry: """Returns current registry instance with responses. @@ -726,10 +741,10 @@ def reset(self) -> None: def add( self, - method: _HTTPMethodOrResponse = None, + method: "_HTTPMethodOrResponse" = None, url: "Optional[_URLPatternType]" = None, - body: _Body = "", - adding_headers: _HeaderSet = None, + body: "_Body" = "", + adding_headers: "_HeaderSet" = None, *args: Any, **kwargs: Any, ) -> BaseResponse: @@ -808,7 +823,7 @@ def _add_from_file(self, file_path: "Union[str, bytes, os.PathLike[Any]]") -> No auto_calculate_content_length=rsp["auto_calculate_content_length"], ) - def add_passthru(self, prefix: _URLPatternType) -> None: + def add_passthru(self, prefix: "_URLPatternType") -> None: """ Register a URL prefix or regex to passthru any non-matching mock requests to. @@ -829,7 +844,7 @@ def add_passthru(self, prefix: _URLPatternType) -> None: def remove( self, - method_or_response: _HTTPMethodOrResponse = None, + method_or_response: "_HTTPMethodOrResponse" = None, url: "Optional[_URLPatternType]" = None, ) -> List[BaseResponse]: """ @@ -852,9 +867,9 @@ def remove( def replace( self, - method_or_response: _HTTPMethodOrResponse = None, + method_or_response: "_HTTPMethodOrResponse" = None, url: "Optional[_URLPatternType]" = None, - body: _Body = "", + body: "_Body" = "", *args: Any, **kwargs: Any, ) -> BaseResponse: @@ -878,9 +893,9 @@ def replace( def upsert( self, - method_or_response: _HTTPMethodOrResponse = None, + method_or_response: "_HTTPMethodOrResponse" = None, url: "Optional[_URLPatternType]" = None, - body: _Body = "", + body: "_Body" = "", *args: Any, **kwargs: Any, ) -> BaseResponse: @@ -901,9 +916,10 @@ def upsert( def add_callback( self, method: str, - url: _URLPatternType, + url: "_URLPatternType", callback: Callable[ - ["PreparedRequest"], Union[Exception, Tuple[int, Mapping[str, str], _Body]] + ["PreparedRequest"], + Union[Exception, Tuple[int, Mapping[str, str], "_Body"]], ], match_querystring: Union[bool, FalseBool] = FalseBool(), content_type: Optional[str] = "text/plain", @@ -940,7 +956,7 @@ def __exit__(self, type: Any, value: Any, traceback: Any) -> bool: return success @overload - def activate(self, func: _F = ...) -> _F: + def activate(self, func: "_F" = ...) -> "_F": """Overload for scenario when 'responses.activate' is used.""" @overload @@ -958,15 +974,15 @@ def activate( def activate( self, - func: Optional[_F] = None, + func: Optional["_F"] = None, *, registry: Optional[Type[Any]] = None, assert_all_requests_are_fired: bool = False, - ) -> Union[Callable[["_F"], "_F"], _F]: + ) -> Union[Callable[["_F"], "_F"], "_F"]: if func is not None: return get_wrapped(func, self) - def deco_activate(function: _F) -> Callable[..., Any]: + def deco_activate(function: "_F") -> Callable[..., Any]: return get_wrapped( function, self, @@ -1028,7 +1044,7 @@ def _on_request( ] ): logger.info("request.allowed-passthru", extra={"url": request_url}) - return _real_send(adapter, request, **kwargs) + return self._real_send(adapter, request, **kwargs) # type: ignore error_msg = ( "Connection refused by Responses - the call doesn't " @@ -1055,7 +1071,7 @@ def _on_request( if match.passthrough: logger.info("request.passthrough-response", extra={"url": request_url}) - response = _real_send(adapter, request, **kwargs) # type: ignore[assignment] + response = self._real_send(adapter, request, **kwargs) # type: ignore else: try: response = adapter.build_response( # type: ignore[no-untyped-call] diff --git a/responses/tests/test_responses.py b/responses/tests/test_responses.py index 2ae3d655..b6c1a5ee 100644 --- a/responses/tests/test_responses.py +++ b/responses/tests/test_responses.py @@ -1838,6 +1838,34 @@ def run(): run() assert_reset() + def test_real_send_argument(self): + def run(): + # the following mock will serve to catch the real send request from another mock and + # will "donate" `unbound_on_send` method + mock_to_catch_real_send = responses.RequestsMock( + assert_all_requests_are_fired=True + ) + mock_to_catch_real_send.post( + "http://send-this-request-through.com", status=500 + ) + + with responses.RequestsMock( + assert_all_requests_are_fired=True, + real_adapter_send=mock_to_catch_real_send.unbound_on_send(), + ) as r_mock: + r_mock.add_passthru("http://send-this-request-through.com") + + r_mock.add(responses.POST, "https://example.org", status=200) + + response = requests.post("https://example.org") + assert response.status_code == 200 + + response = requests.post("http://send-this-request-through.com") + assert response.status_code == 500 + + run() + assert_reset() + def test_method_named_param(): @responses.activate From 9c0d1ac3d72f2e2a79798b9e9239688bc77c243b Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Wed, 4 Oct 2023 19:46:18 +0200 Subject: [PATCH 2/3] fix coverage on master (#677) * change the order of dependencies installation to ensure that `urllib3` is not overwritten --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa315a7e..eeb4cb24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,8 +48,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ${{ matrix.requests-version }} ${{ matrix.urllib3-version }} make install-deps + pip install ${{ matrix.requests-version }} ${{ matrix.urllib3-version }} - name: Run Pytest run: | From 923307c15e1355ca53b244c44123afd2ce01c6ab Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Fri, 6 Oct 2023 22:53:13 +0200 Subject: [PATCH 3/3] deduplicate content-type headers (#673) --- CHANGES | 1 + responses/__init__.py | 9 ++++++++- responses/tests/test_responses.py | 23 +++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 2bb40959..9e701307 100644 --- a/CHANGES +++ b/CHANGES @@ -5,6 +5,7 @@ through which function they would like to send real requests * Added support for re.Pattern based header matching. * Added support for gzipped response bodies to `json_params_matcher`. +* Fix `Content-Type` headers issue when the header was duplicated. See #644 * Moved types-pyyaml dependency to `tests_requires` 0.23.3 diff --git a/responses/__init__.py b/responses/__init__.py index ea7009dd..78a3a436 100644 --- a/responses/__init__.py +++ b/responses/__init__.py @@ -491,10 +491,17 @@ def _req_attr_matches( def get_headers(self) -> HTTPHeaderDict: headers = HTTPHeaderDict() # Duplicate headers are legal - if self.content_type is not None: + + # Add Content-Type if it exists and is not already in headers + if self.content_type and ( + not self.headers or "Content-Type" not in self.headers + ): headers["Content-Type"] = self.content_type + + # Extend headers if they exist if self.headers: headers.extend(self.headers) + return headers def get_response(self, request: "PreparedRequest") -> HTTPResponse: diff --git a/responses/tests/test_responses.py b/responses/tests/test_responses.py index b6c1a5ee..fcdb62b7 100644 --- a/responses/tests/test_responses.py +++ b/responses/tests/test_responses.py @@ -1330,6 +1330,29 @@ def run(): assert_reset() +def test_headers_deduplicated_content_type(): + """Test to ensure that we do not have two values for `content-type`. + + For more details see https://github.com/getsentry/responses/issues/644 + """ + + @responses.activate + def run(): + responses.get( + "https://example.org/", + json={}, + headers={"Content-Type": "application/json"}, + ) + responses.start() + + resp = requests.get("https://example.org/") + + assert resp.headers["Content-Type"] == "application/json" + + run() + assert_reset() + + def test_content_length_error(monkeypatch): """ Currently 'requests' does not enforce content length validation,