Merge branch 'master' into call-info-in-response

.github/workflows/ci.yml

@@ -48,8 +48,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install ${{ matrix.requests-version }} ${{ matrix.urllib3-version }}
         make install-deps
+        pip install ${{ matrix.requests-version }} ${{ matrix.urllib3-version }}
 
     - name: Run Pytest
       run: |

CHANGES

@@ -2,8 +2,11 @@
 ------
 
 * Added `BaseResponse.calls` to access calls data of a separate mocked request. See #664
+* Added `real_adapter_send` parameter to `RequestsMock` that will allow users to set
+  through which function they would like to send real requests
 * Added support for re.Pattern based header matching.
 * Added support for gzipped response bodies to `json_params_matcher`.
+* Fix `Content-Type` headers issue when the header was duplicated. See #644
 * Moved types-pyyaml dependency to `tests_requires`
 
 0.23.3

responses/__init__.py

@@ -49,7 +49,8 @@
 try:
     from requests.packages.urllib3.connection import HTTPHeaderDict
 except ImportError:  # pragma: no cover
-    from urllib3.response import HTTPHeaderDict  # type: ignore[attr-defined]
+    from urllib3.response import HTTPHeaderDict
+
 try:
     from requests.packages.urllib3.util.url import parse_url
 except ImportError:  # pragma: no cover
@@ -84,14 +85,25 @@ def __call__(
         ) -> models.Response:
             ...
 
-
-# Block of type annotations
-_Body = Union[str, BaseException, "Response", BufferedReader, bytes, None]
-_F = Callable[..., Any]
-_HeaderSet = Optional[Union[Mapping[str, str], List[Tuple[str, str]]]]
-_MatcherIterable = Iterable[Callable[..., Tuple[bool, str]]]
-_HTTPMethodOrResponse = Optional[Union[str, "BaseResponse"]]
-_URLPatternType = Union["Pattern[str]", str]
+    # Block of type annotations
+    _Body = Union[str, BaseException, "Response", BufferedReader, bytes, None]
+    _F = Callable[..., Any]
+    _HeaderSet = Optional[Union[Mapping[str, str], List[Tuple[str, str]]]]
+    _MatcherIterable = Iterable[Callable[..., Tuple[bool, str]]]
+    _HTTPMethodOrResponse = Optional[Union[str, "BaseResponse"]]
+    _URLPatternType = Union["Pattern[str]", str]
+    _HTTPAdapterSend = Callable[
+        [
+            HTTPAdapter,
+            PreparedRequest,
+            bool,
+            float | tuple[float, float] | tuple[float, None] | None,
+            bool | str,
+            bytes | str | tuple[bytes | str, bytes | str] | None,
+            Mapping[str, str] | None,
+        ],
+        models.Response,
+    ]
 
 
 Call = namedtuple("Call", ["request", "response"])
@@ -250,7 +262,7 @@ def __getitem__(self, idx: slice) -> List[Call]:
     def __getitem__(self, idx: Union[int, slice]) -> Union[Call, List[Call]]:
         return self._calls[idx]
 
-    def add(self, request: "PreparedRequest", response: _Body) -> None:
+    def add(self, request: "PreparedRequest", response: "_Body") -> None:
         self._calls.append(Call(request, response))
 
     def add_call(self, call: Call) -> None:
@@ -261,8 +273,8 @@ def reset(self) -> None:
 
 
 def _ensure_url_default_path(
-    url: _URLPatternType,
-) -> _URLPatternType:
+    url: "_URLPatternType",
+) -> "_URLPatternType":
     """Add empty URL path '/' if doesn't exist.
 
     Examples
@@ -379,15 +391,15 @@ class BaseResponse:
     def __init__(
         self,
         method: str,
-        url: _URLPatternType,
+        url: "_URLPatternType",
         match_querystring: Union[bool, object] = None,
         match: "_MatcherIterable" = (),
         *,
         passthrough: bool = False,
     ) -> None:
         self.method: str = method
         # ensure the url has a default path set if the url is a string
-        self.url: _URLPatternType = _ensure_url_default_path(url)
+        self.url: "_URLPatternType" = _ensure_url_default_path(url)
 
         if self._should_match_querystring(match_querystring):
             match = tuple(match) + (
@@ -437,7 +449,7 @@ def _should_match_querystring(
 
         return bool(urlsplit(self.url).query)
 
-    def _url_matches(self, url: _URLPatternType, other: str) -> bool:
+    def _url_matches(self, url: "_URLPatternType", other: str) -> bool:
         """Compares two URLs.
 
         Compares only scheme, netloc and path. If 'url' is a re.Pattern, then checks that
@@ -482,10 +494,17 @@ def _req_attr_matches(
 
     def get_headers(self) -> HTTPHeaderDict:
         headers = HTTPHeaderDict()  # Duplicate headers are legal
-        if self.content_type is not None:
+
+        # Add Content-Type if it exists and is not already in headers
+        if self.content_type and (
+            not self.headers or "Content-Type" not in self.headers
+        ):
             headers["Content-Type"] = self.content_type
+
+        # Extend headers if they exist
         if self.headers:
             headers.extend(self.headers)
+
         return headers
 
     def get_response(self, request: "PreparedRequest") -> HTTPResponse:
@@ -543,8 +562,8 @@ class Response(BaseResponse):
     def __init__(
         self,
         method: str,
-        url: _URLPatternType,
-        body: _Body = "",
+        url: "_URLPatternType",
+        body: "_Body" = "",
         json: Optional[Any] = None,
         status: int = 200,
         headers: Optional[Mapping[str, str]] = None,
@@ -567,7 +586,7 @@ def __init__(
             else:
                 content_type = "text/plain"
 
-        self.body: _Body = body
+        self.body: "_Body" = body
         self.status: int = status
         self.headers: Optional[Mapping[str, str]] = headers
 
@@ -619,7 +638,7 @@ class CallbackResponse(BaseResponse):
     def __init__(
         self,
         method: str,
-        url: _URLPatternType,
+        url: "_URLPatternType",
         callback: Callable[[Any], Any],
         stream: Optional[bool] = None,
         content_type: Optional[str] = "text/plain",
@@ -689,6 +708,8 @@ def __init__(
         passthru_prefixes: Tuple[str, ...] = (),
         target: str = "requests.adapters.HTTPAdapter.send",
         registry: Type[FirstMatchRegistry] = FirstMatchRegistry,
+        *,
+        real_adapter_send: "_HTTPAdapterSend" = _real_send,
     ) -> None:
         self._calls: CallList = CallList()
         self.reset()
@@ -699,6 +720,7 @@ def __init__(
         self.target: str = target
         self._patcher: Optional["_mock_patcher[Any]"] = None
         self._thread_lock = _ThreadingLock()
+        self._real_send = real_adapter_send
 
     def get_registry(self) -> FirstMatchRegistry:
         """Returns current registry instance with responses.
@@ -737,10 +759,10 @@ def reset(self) -> None:
 
     def add(
         self,
-        method: _HTTPMethodOrResponse = None,
+        method: "_HTTPMethodOrResponse" = None,
         url: "Optional[_URLPatternType]" = None,
-        body: _Body = "",
-        adding_headers: _HeaderSet = None,
+        body: "_Body" = "",
+        adding_headers: "_HeaderSet" = None,
         *args: Any,
         **kwargs: Any,
     ) -> BaseResponse:
@@ -819,7 +841,7 @@ def _add_from_file(self, file_path: "Union[str, bytes, os.PathLike[Any]]") -> No
                 auto_calculate_content_length=rsp["auto_calculate_content_length"],
             )
 
-    def add_passthru(self, prefix: _URLPatternType) -> None:
+    def add_passthru(self, prefix: "_URLPatternType") -> None:
         """
         Register a URL prefix or regex to passthru any non-matching mock requests to.
 
@@ -840,7 +862,7 @@ def add_passthru(self, prefix: _URLPatternType) -> None:
 
     def remove(
         self,
-        method_or_response: _HTTPMethodOrResponse = None,
+        method_or_response: "_HTTPMethodOrResponse" = None,
         url: "Optional[_URLPatternType]" = None,
     ) -> List[BaseResponse]:
         """
@@ -863,9 +885,9 @@ def remove(
 
     def replace(
         self,
-        method_or_response: _HTTPMethodOrResponse = None,
+        method_or_response: "_HTTPMethodOrResponse" = None,
         url: "Optional[_URLPatternType]" = None,
-        body: _Body = "",
+        body: "_Body" = "",
         *args: Any,
         **kwargs: Any,
     ) -> BaseResponse:
@@ -889,9 +911,9 @@ def replace(
 
     def upsert(
         self,
-        method_or_response: _HTTPMethodOrResponse = None,
+        method_or_response: "_HTTPMethodOrResponse" = None,
         url: "Optional[_URLPatternType]" = None,
-        body: _Body = "",
+        body: "_Body" = "",
         *args: Any,
         **kwargs: Any,
     ) -> BaseResponse:
@@ -912,9 +934,10 @@ def upsert(
     def add_callback(
         self,
         method: str,
-        url: _URLPatternType,
+        url: "_URLPatternType",
         callback: Callable[
-            ["PreparedRequest"], Union[Exception, Tuple[int, Mapping[str, str], _Body]]
+            ["PreparedRequest"],
+            Union[Exception, Tuple[int, Mapping[str, str], "_Body"]],
         ],
         match_querystring: Union[bool, FalseBool] = FalseBool(),
         content_type: Optional[str] = "text/plain",
@@ -951,7 +974,7 @@ def __exit__(self, type: Any, value: Any, traceback: Any) -> bool:
         return success
 
     @overload
-    def activate(self, func: _F = ...) -> _F:
+    def activate(self, func: "_F" = ...) -> "_F":
         """Overload for scenario when 'responses.activate' is used."""
 
     @overload
@@ -969,15 +992,15 @@ def activate(
 
     def activate(
         self,
-        func: Optional[_F] = None,
+        func: Optional["_F"] = None,
         *,
         registry: Optional[Type[Any]] = None,
         assert_all_requests_are_fired: bool = False,
-    ) -> Union[Callable[["_F"], "_F"], _F]:
+    ) -> Union[Callable[["_F"], "_F"], "_F"]:
         if func is not None:
             return get_wrapped(func, self)
 
-        def deco_activate(function: _F) -> Callable[..., Any]:
+        def deco_activate(function: "_F") -> Callable[..., Any]:
             return get_wrapped(
                 function,
                 self,
@@ -1039,7 +1062,7 @@ def _on_request(
                 ]
             ):
                 logger.info("request.allowed-passthru", extra={"url": request_url})
-                return _real_send(adapter, request, **kwargs)
+                return self._real_send(adapter, request, **kwargs)  # type: ignore
 
             error_msg = (
                 "Connection refused by Responses - the call doesn't "
@@ -1066,7 +1089,7 @@ def _on_request(
 
         if match.passthrough:
             logger.info("request.passthrough-response", extra={"url": request_url})
-            response = _real_send(adapter, request, **kwargs)  # type: ignore[assignment]
+            response = self._real_send(adapter, request, **kwargs)  # type: ignore
         else:
             try:
                 response = adapter.build_response(  # type: ignore[no-untyped-call]

responses/tests/test_responses.py

@@ -1331,6 +1331,29 @@ def run():
     assert_reset()
 
 
+def test_headers_deduplicated_content_type():
+    """Test to ensure that we do not have two values for `content-type`.
+
+    For more details see https://github.com/getsentry/responses/issues/644
+    """
+
+    @responses.activate
+    def run():
+        responses.get(
+            "https://example.org/",
+            json={},
+            headers={"Content-Type": "application/json"},
+        )
+        responses.start()
+
+        resp = requests.get("https://example.org/")
+
+        assert resp.headers["Content-Type"] == "application/json"
+
+    run()
+    assert_reset()
+
+
 def test_content_length_error(monkeypatch):
     """
     Currently 'requests' does not enforce content length validation,
@@ -1839,6 +1862,34 @@ def run():
         run()
         assert_reset()
 
+    def test_real_send_argument(self):
+        def run():
+            # the following mock will serve to catch the real send request from another mock and
+            # will "donate" `unbound_on_send` method
+            mock_to_catch_real_send = responses.RequestsMock(
+                assert_all_requests_are_fired=True
+            )
+            mock_to_catch_real_send.post(
+                "http://send-this-request-through.com", status=500
+            )
+
+            with responses.RequestsMock(
+                assert_all_requests_are_fired=True,
+                real_adapter_send=mock_to_catch_real_send.unbound_on_send(),
+            ) as r_mock:
+                r_mock.add_passthru("http://send-this-request-through.com")
+
+                r_mock.add(responses.POST, "https://example.org", status=200)
+
+                response = requests.post("https://example.org")
+                assert response.status_code == 200
+
+                response = requests.post("http://send-this-request-through.com")
+                assert response.status_code == 500
+
+        run()
+        assert_reset()
+
 
 def test_method_named_param():
     @responses.activate