From abf69ee6a850d089827e5df0bb71840c37f20049 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 29 May 2026 14:30:57 -0400 Subject: [PATCH 01/45] Add HTTPXWrapper backed by httpx2 behind use_httpx flag --- datadog_checks_base/changelog.d/22676.added | 2 +- .../datadog_checks/base/checks/base.py | 12 +- .../openmetrics/v2/scraper/base_scraper.py | 7 +- .../datadog_checks/base/utils/http_httpx.py | 424 ++++++++++++++++++ datadog_checks_base/pyproject.toml | 1 + .../tests/base/utils/http_httpx/__init__.py | 3 + .../tests/base/utils/http_httpx/common.py | 12 + .../tests/base/utils/http_httpx/conftest.py | 46 ++ .../base/utils/http_httpx/test_auth_basic.py | 34 ++ .../base/utils/http_httpx/test_config.py | 136 ++++++ .../base/utils/http_httpx/test_exceptions.py | 44 ++ .../base/utils/http_httpx/test_lifecycle.py | 47 ++ .../base/utils/http_httpx/test_methods.py | 42 ++ .../base/utils/http_httpx/test_response.py | 198 ++++++++ .../test_stream_connection_lines.py | 53 +++ 15 files changed, 1054 insertions(+), 7 deletions(-) create mode 100644 datadog_checks_base/datadog_checks/base/utils/http_httpx.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/__init__.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/common.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/conftest.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_config.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_methods.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_response.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_stream_connection_lines.py diff --git a/datadog_checks_base/changelog.d/22676.added b/datadog_checks_base/changelog.d/22676.added index ff63715e3c69d..8125821a3d951 100644 --- a/datadog_checks_base/changelog.d/22676.added +++ b/datadog_checks_base/changelog.d/22676.added @@ -1 +1 @@ -Add library-agnostic HTTP mocks/proto/exceptions and migrate intg tests. +Add library-agnostic HTTP mocks/proto/exceptions and migrate intg tests. Add ``HTTPXWrapper``, an httpx2-backed HTTP client opt-in via the ``use_httpx`` instance config flag. The default remains the existing requests-based client. diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 13d107b5cea73..9ec690ab2432b 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -412,10 +412,16 @@ def http(self) -> HTTPClientProtocol: Only new checks or checks on Agent 6.13+ can and should use this for HTTP requests. """ if not hasattr(self, '_http'): - # See Performance Optimizations in this package's README.md. - from datadog_checks.base.utils.http import RequestsWrapper + instance = self.instance or {} + if is_affirmative(instance.get('use_httpx', False)): + from datadog_checks.base.utils.http_httpx import HTTPXWrapper + + self._http = HTTPXWrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) + else: + # See Performance Optimizations in this package's README.md. + from datadog_checks.base.utils.http import RequestsWrapper - self._http = RequestsWrapper(self.instance or {}, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) + self._http = RequestsWrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) return self._http diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py index 24719c329efef..a3211f6c39405 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py @@ -23,6 +23,7 @@ from datadog_checks.base.constants import ServiceCheck from datadog_checks.base.errors import ConfigurationError from datadog_checks.base.utils.functions import no_op, return_true +from datadog_checks.base.utils.http_exceptions import HTTPConnectionError class OpenMetricsScraper: @@ -405,11 +406,11 @@ def stream_connection_lines(self): self._content_type = connection.headers.get('Content-Type', '') for line in connection.iter_lines(decode_unicode=True): yield line - except ConnectionError as e: + except (ConnectionError, HTTPConnectionError) as e: if self.ignore_connection_errors: - self.log.warning("OpenMetrics endpoint %s is not accessible", self.endpoint) + self.log.warning("OpenMetrics endpoint %s is not accessible: %s", self.endpoint, e) else: - raise e + raise def filter_connection_lines(self, line_streamer): """ diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py new file mode 100644 index 0000000000000..a91b7c37acb74 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -0,0 +1,424 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import logging +from collections.abc import Iterator, Mapping +from datetime import timedelta +from typing import Any + +import httpx2 as httpx +from binary import KIBIBYTE + +from datadog_checks.base.config import is_affirmative + +from .headers import get_default_headers, update_headers +from .http_exceptions import ( + HTTPConnectionError, + HTTPError, + HTTPInvalidURLError, + HTTPRequestError, + HTTPStatusError, + HTTPTimeoutError, +) + +LOGGER = logging.getLogger(__name__) + +DEFAULT_TIMEOUT = 10 +# Matches the effective chunk size used by RequestsWrapper.iter_content (http.py:415 multiplies by KIBIBYTE). +DEFAULT_CHUNK_SIZE = 16 * KIBIBYTE + +STANDARD_FIELDS = { + 'allow_redirects': True, + 'connect_timeout': None, + 'extra_headers': None, + 'headers': None, + 'log_requests': False, + 'password': None, + 'read_timeout': None, + 'timeout': DEFAULT_TIMEOUT, + 'tls_ca_cert': None, + 'tls_cert': None, + 'tls_private_key': None, + 'tls_verify': True, + 'username': None, +} + +DEFAULT_REMAPPED_FIELDS: dict[str, dict[str, Any]] = {} + +REQUEST_KWARGS = frozenset( + { + 'params', + 'json', + 'data', + 'content', + 'files', + 'cookies', + 'headers', + 'extra_headers', + 'timeout', + 'follow_redirects', + } +) + + +def _make_timeout(connect: float, read: float) -> httpx.Timeout: + return httpx.Timeout(connect=connect, read=read, write=None, pool=None) + + +def _build_basic_auth(config: dict[str, Any]) -> httpx.BasicAuth | None: + if config['username'] is not None and config['password'] is not None: + return httpx.BasicAuth(config['username'], config['password']) + return None + + +def _build_verify(config: dict[str, Any]) -> bool | str: + if isinstance(config['tls_ca_cert'], str): + return config['tls_ca_cert'] + if not is_affirmative(config['tls_verify']): + return False + return True + + +def _build_cert(config: dict[str, Any]) -> str | tuple[str, str] | None: + cert = config['tls_cert'] + if not isinstance(cert, str): + return None + private_key = config['tls_private_key'] + if isinstance(private_key, str): + return (cert, private_key) + return cert + + +def _build_timeout(config: dict[str, Any]) -> tuple[float, float]: + base = float(config['timeout']) + connect = float(config['connect_timeout']) if config['connect_timeout'] is not None else base + read = float(config['read_timeout']) if config['read_timeout'] is not None else base + return connect, read + + +def _map_httpx_exception(exc: httpx.HTTPError | httpx.InvalidURL) -> HTTPError: + """Translate an httpx2 exception into the library-agnostic equivalent.""" + # ConnectError -> HTTPConnectionError pairs 1:1 with the Step 3a widening (PR #22864). + # Mid-stream NetworkError/ReadError/WriteError stay HTTPRequestError on purpose. + if isinstance(exc, httpx.InvalidURL): + return HTTPInvalidURLError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) + if isinstance(exc, httpx.TimeoutException): + return HTTPTimeoutError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) + if isinstance(exc, httpx.ConnectError): + return HTTPConnectionError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) + if isinstance(exc, httpx.HTTPStatusError): + return HTTPStatusError( + str(exc) or exc.__class__.__name__, + request=getattr(exc, 'request', None), + response=getattr(exc, 'response', None), + ) + if isinstance(exc, httpx.RequestError): + return HTTPRequestError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) + return HTTPError(str(exc) or exc.__class__.__name__) + + +class HTTPXResponseAdapter: + """Wraps an httpx2.Response to satisfy HTTPResponseProtocol.""" + + __slots__ = ('_response',) + + def __init__(self, response: httpx.Response) -> None: + self._response = response + + @property + def status_code(self) -> int: + return self._response.status_code + + @property + def content(self) -> bytes: + return self._response.read() + + @property + def text(self) -> str: + self._response.read() + return self._response.text + + @property + def headers(self) -> Mapping[str, str]: + return self._response.headers + + @property + def ok(self) -> bool: + return self._response.status_code < 400 + + @property + def reason(self) -> str: + return self._response.reason_phrase + + @property + def encoding(self) -> str | None: + return self._response.encoding + + @encoding.setter + def encoding(self, value: str | None) -> None: + self._response.encoding = value + + @property + def url(self) -> str: + return str(self._response.url) + + @property + def cookies(self) -> httpx.Cookies: + return self._response.cookies + + @property + def elapsed(self) -> timedelta: + try: + return self._response.elapsed + except RuntimeError: + LOGGER.debug('elapsed unavailable for response from %s', self._response.url) + return timedelta(0) + + def json(self, **kwargs: Any) -> Any: + self._response.read() + return self._response.json(**kwargs) + + def raise_for_status(self) -> None: + # Mirror requests semantics (4xx/5xx only); httpx2 also raises on 3xx. + if self._response.status_code < 400: + return + try: + self._response.raise_for_status() + except httpx.HTTPStatusError as exc: + raise _map_httpx_exception(exc) from exc + + def close(self) -> None: + self._response.close() + + def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: + effective_size = chunk_size if chunk_size is not None else DEFAULT_CHUNK_SIZE + encoding = self._response.encoding or 'utf-8' + for chunk in self._response.iter_bytes(chunk_size=effective_size): + yield chunk.decode(encoding) if decode_unicode else chunk + + def iter_lines( + self, + chunk_size: int | None = None, # noqa: ARG002 - httpx2 buffers lines internally; kept for HTTPResponseProtocol parity + decode_unicode: bool = False, + delimiter: bytes | str | None = None, + ) -> Iterator[bytes | str]: + if delimiter is not None: + raise NotImplementedError("HTTPXResponseAdapter.iter_lines does not support custom delimiters") + encoding = self._response.encoding or 'utf-8' + for line in self._response.iter_lines(): + yield line if decode_unicode else line.encode(encoding) + + def __enter__(self) -> 'HTTPXResponseAdapter': + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: + self.close() + return None + + +class HTTPXWrapper: + """Implements HTTPClientProtocol using a single shared httpx2.Client per wrapper.""" + + __slots__ = ( + '_client', + '_log_requests', + 'logger', + 'options', + ) + + def __init__( + self, + instance: dict[str, Any], + init_config: dict[str, Any] | None = None, + remapper: dict[str, dict[str, Any]] | None = None, + logger: logging.Logger | None = None, + transport: httpx.BaseTransport | None = None, + ) -> None: + self.logger = logger or LOGGER + init_config = init_config or {} + + config = self._resolve_config(instance, init_config, remapper) + + headers = get_default_headers() + if config['headers']: + headers.clear() + update_headers(headers, config['headers']) + if config['extra_headers']: + update_headers(headers, config['extra_headers']) + + auth = _build_basic_auth(config) + verify = _build_verify(config) + cert = _build_cert(config) + timeout = _build_timeout(config) + allow_redirects = is_affirmative(config['allow_redirects']) + + # proxies=None mirrors RequestsWrapper.options for consumers (e.g. http_check). Wiring is Phase 3. + self.options: dict[str, Any] = { + 'auth': auth, + 'cert': cert, + 'headers': headers, + 'proxies': None, + 'timeout': timeout, + 'verify': verify, + 'allow_redirects': allow_redirects, + } + + self._log_requests = is_affirmative(config['log_requests']) + self._client = self._build_client(transport) + + @staticmethod + def _resolve_config( + instance: dict[str, Any], + init_config: dict[str, Any], + remapper: dict[str, dict[str, Any]] | None, + ) -> dict[str, Any]: + default_fields = dict(STANDARD_FIELDS) + default_fields['log_requests'] = init_config.get('log_requests', default_fields['log_requests']) + default_fields['timeout'] = init_config.get('timeout', default_fields['timeout']) + + config = {field: instance.get(field, value) for field, value in default_fields.items()} + + remapper = dict(remapper) if remapper else {} + remapper.update(DEFAULT_REMAPPED_FIELDS) + + for remapped_field, data in remapper.items(): + field = data.get('name') + if field not in STANDARD_FIELDS: + continue + if field in instance: + continue + + default = default_fields[field] + if data.get('invert'): + default = not default + + value = instance.get(remapped_field, data.get('default', default)) + if data.get('invert'): + value = not is_affirmative(value) + + config[field] = value + return config + + def _build_client(self, transport: httpx.BaseTransport | None) -> httpx.Client: + kwargs: dict[str, Any] = { + 'headers': self.options['headers'], + 'timeout': _make_timeout(self.options['timeout'][0], self.options['timeout'][1]), + 'follow_redirects': self.options['allow_redirects'], + 'verify': self.options['verify'], + } + if self.options['cert'] is not None: + kwargs['cert'] = self.options['cert'] + if self.options['auth'] is not None: + kwargs['auth'] = self.options['auth'] + if transport is not None: + kwargs['transport'] = transport + return httpx.Client(**kwargs) + + def get_header(self, name: str, default: str | None = None) -> str | None: + for key, value in self.options['headers'].items(): + if key.lower() == name.lower(): + return value + return default + + def set_header(self, name: str, value: str) -> None: + # Mirror into both stores: options['headers'] is the public shape, _client.headers is what httpx2 sends. + for key in list(self.options['headers']): + if key.lower() == name.lower(): + self.options['headers'][key] = value + self._client.headers[key] = value + return + self.options['headers'][name] = value + self._client.headers[name] = value + + def get(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('GET', url, options) + + def post(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('POST', url, options) + + def put(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('PUT', url, options) + + def delete(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('DELETE', url, options) + + def head(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('HEAD', url, options) + + def patch(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('PATCH', url, options) + + def options_method(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('OPTIONS', url, options) + + def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXResponseAdapter: + if self._log_requests: + self.logger.debug('Sending %s request to %s', method, url) + + request_kwargs = self._build_request_kwargs(options) + follow_redirects = request_kwargs.pop('follow_redirects', httpx.USE_CLIENT_DEFAULT) + try: + request = self._client.build_request(method, url, **request_kwargs) + response = self._client.send(request, stream=True, follow_redirects=follow_redirects) + except (httpx.HTTPError, httpx.InvalidURL) as exc: + raise _map_httpx_exception(exc) from exc + return HTTPXResponseAdapter(response) + + def _build_request_kwargs(self, options: dict[str, Any]) -> dict[str, Any]: + """Translate call-site options to httpx2.Client.request kwargs.""" + # OM v2 scraper injects stream=True unconditionally (base_scraper.py:459). The wrapper + # always streams internally, so drop the kwarg silently rather than raising on it. + options = {k: v for k, v in options.items() if k != 'stream'} + + unknown = set(options) - REQUEST_KWARGS + if unknown: + raise TypeError(f"HTTPXWrapper does not support per-request kwargs: {sorted(unknown)}") + kwargs: dict[str, Any] = {} + passthrough = ('params', 'json', 'data', 'content', 'files', 'cookies') + for key in passthrough: + if key in options: + kwargs[key] = options[key] + + extra_headers = options.get('extra_headers') + headers = options.get('headers') + merged_headers: dict[str, str] | None = None + if headers is not None or extra_headers is not None: + merged_headers = {} + if headers is not None: + merged_headers.update(headers) + if extra_headers is not None: + merged_headers.update(extra_headers) + if merged_headers is not None: + kwargs['headers'] = merged_headers + + if 'timeout' in options: + timeout_value = options['timeout'] + if isinstance(timeout_value, (tuple, list)) and len(timeout_value) == 2: + kwargs['timeout'] = _make_timeout(float(timeout_value[0]), float(timeout_value[1])) + else: + kwargs['timeout'] = float(timeout_value) # type: ignore[arg-type] + + if 'follow_redirects' in options: + kwargs['follow_redirects'] = bool(options['follow_redirects']) + + return kwargs + + def close(self) -> None: + client = getattr(self, '_client', None) + if client is not None: + client.close() + + def __enter__(self) -> 'HTTPXWrapper': + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: + self.close() + return None + + def __del__(self) -> None: # no cov + try: + self.close() + except AttributeError: + pass diff --git a/datadog_checks_base/pyproject.toml b/datadog_checks_base/pyproject.toml index d52b26f0a731a..7e0d41fba94c6 100644 --- a/datadog_checks_base/pyproject.toml +++ b/datadog_checks_base/pyproject.toml @@ -38,6 +38,7 @@ deps = [ "cachetools==7.0.5", "cryptography==46.0.7", "ddtrace==3.19.5", + "httpx2==2.2.0", "jellyfish==1.2.1", "lazy-loader==0.5", "prometheus-client==0.24.1", diff --git a/datadog_checks_base/tests/base/utils/http_httpx/__init__.py b/datadog_checks_base/tests/base/utils/http_httpx/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/common.py b/datadog_checks_base/tests/base/utils/http_httpx/common.py new file mode 100644 index 0000000000000..d63c9b58cb84a --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/common.py @@ -0,0 +1,12 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import base64 + + +def parse_basic_auth(header_value: str) -> tuple[str, str]: + scheme, _, b64 = header_value.partition(' ') + assert scheme.lower() == 'basic' + user_pass = base64.b64decode(b64).decode('utf-8') + user, _, password = user_pass.partition(':') + return user, password diff --git a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py new file mode 100644 index 0000000000000..272e7b1bc1228 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py @@ -0,0 +1,46 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from collections.abc import Callable + +import httpx2 as httpx +import pytest + + +@pytest.fixture +def status_transport_factory() -> Callable[[int, bytes | str], httpx.MockTransport]: + def _factory(status_code: int, body: bytes | str = b''): + def handler(_request: httpx.Request) -> httpx.Response: + if isinstance(body, str): + return httpx.Response(status_code, text=body) + return httpx.Response(status_code, content=body) + + return httpx.MockTransport(handler) + + return _factory + + +@pytest.fixture +def raising_transport_factory() -> Callable[[Exception], httpx.MockTransport]: + def _factory(exc: Exception): + def handler(_request: httpx.Request) -> httpx.Response: + raise exc + + return httpx.MockTransport(handler) + + return _factory + + +@pytest.fixture +def captured_requests() -> list[httpx.Request]: + return [] + + +@pytest.fixture +def capturing_transport(captured_requests: list[httpx.Request]) -> httpx.MockTransport: + def handler(request: httpx.Request) -> httpx.Response: + _ = request.content + captured_requests.append(request) + return httpx.Response(200, json={'ok': True}) + + return httpx.MockTransport(handler) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py new file mode 100644 index 0000000000000..23081a10a34b1 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py @@ -0,0 +1,34 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + +from .common import parse_basic_auth + + +def test_basic_auth_sent_on_request(capturing_transport, captured_requests): + http = HTTPXWrapper( + {'username': 'alice', 'password': 'secret'}, + {}, + transport=capturing_transport, + ) + http.get('http://example.test/') + user, password = parse_basic_auth(captured_requests[0].headers['authorization']) + assert user == 'alice' + assert password == 'secret' + + +@pytest.mark.parametrize( + 'instance', + [ + pytest.param({}, id='no-credentials'), + pytest.param({'username': 'alice'}, id='username-only'), + pytest.param({'password': 'secret'}, id='password-only'), + ], +) +def test_no_authorization_header_set(instance, capturing_transport, captured_requests): + http = HTTPXWrapper(instance, {}, transport=capturing_transport) + http.get('http://example.test/') + assert 'authorization' not in captured_requests[0].headers diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py new file mode 100644 index 0000000000000..e83e74cfb95e2 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py @@ -0,0 +1,136 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + + +def test_default_headers_include_user_agent(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + assert any(key.lower() == 'user-agent' for key in http.options['headers']) + + +def test_extra_headers_merge(capturing_transport, captured_requests): + http = HTTPXWrapper({'extra_headers': {'X-Extra': 'value'}}, {}, transport=capturing_transport) + http.get('http://example.test/') + assert captured_requests[0].headers['x-extra'] == 'value' + + +def test_headers_override_defaults(capturing_transport, captured_requests): + http = HTTPXWrapper({'headers': {'User-Agent': 'custom-agent/1.0'}}, {}, transport=capturing_transport) + http.get('http://example.test/') + assert captured_requests[0].headers['user-agent'] == 'custom-agent/1.0' + + +def test_per_request_headers_merge_into_request(capturing_transport, captured_requests): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.get('http://example.test/', headers={'X-Per-Request': 'yes'}) + assert captured_requests[0].headers['x-per-request'] == 'yes' + + +def test_timeout_from_instance(capturing_transport): + http = HTTPXWrapper({'timeout': 25}, {}, transport=capturing_transport) + connect, read = http.options['timeout'] + assert connect == 25.0 + assert read == 25.0 + + +def test_connect_and_read_timeout_split(capturing_transport): + http = HTTPXWrapper({'connect_timeout': 5, 'read_timeout': 30}, {}, transport=capturing_transport) + connect, read = http.options['timeout'] + assert connect == 5.0 + assert read == 30.0 + + +def test_verify_defaults_to_true(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + assert http.options['verify'] is True + + +def test_verify_false_when_tls_verify_off(capturing_transport): + http = HTTPXWrapper({'tls_verify': False}, {}, transport=capturing_transport) + assert http.options['verify'] is False + + +def test_tls_ca_cert_uses_path(capturing_transport): + http = HTTPXWrapper({'tls_ca_cert': '/etc/ssl/ca.pem'}, {}, transport=capturing_transport) + assert http.options['verify'] == '/etc/ssl/ca.pem' + + +def test_tls_client_cert_string(capturing_transport): + http = HTTPXWrapper({'tls_cert': '/etc/ssl/client.pem'}, {}, transport=capturing_transport) + assert http.options['cert'] == '/etc/ssl/client.pem' + + +def test_tls_client_cert_with_key(capturing_transport): + http = HTTPXWrapper( + {'tls_cert': '/etc/ssl/client.pem', 'tls_private_key': '/etc/ssl/client.key'}, + {}, + transport=capturing_transport, + ) + assert http.options['cert'] == ('/etc/ssl/client.pem', '/etc/ssl/client.key') + + +def test_tls_no_cert_when_not_configured(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + assert http.options['cert'] is None + + +def test_options_proxies_is_none(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + assert http.options['proxies'] is None + + +@pytest.mark.parametrize( + 'lookup_name,default,expected', + [ + pytest.param('x-foo', None, 'bar', id='lowercase-lookup'), + pytest.param('X-FOO', None, 'bar', id='uppercase-lookup'), + pytest.param('missing', None, None, id='missing-no-default'), + pytest.param('missing', 'fallback', 'fallback', id='missing-with-default'), + ], +) +def test_get_header(capturing_transport, lookup_name, default, expected): + http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) + assert http.get_header(lookup_name, default=default) == expected + + +def test_set_header_overrides_existing(capturing_transport): + http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) + http.set_header('X-FOO', 'new') + assert http.get_header('x-foo') == 'new' + + +def test_set_header_propagates_to_outgoing_request(capturing_transport, captured_requests): + http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) + http.set_header('X-FOO', 'updated') + http.get('http://example.test/') + assert captured_requests[0].headers['x-foo'] == 'updated' + + +def test_set_header_adds_new_header(capturing_transport, captured_requests): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.set_header('X-New', 'value') + assert http.get_header('x-new') == 'value' + http.get('http://example.test/') + assert captured_requests[0].headers['x-new'] == 'value' + + +def test_remapper_renames_field(capturing_transport): + remapper = {'ssl_validation': {'name': 'tls_verify'}} + http = HTTPXWrapper({'ssl_validation': False}, {}, remapper=remapper, transport=capturing_transport) + assert http.options['verify'] is False + + +@pytest.mark.parametrize( + 'kwarg,value', + [ + pytest.param('proxies', {'http': 'http://proxy:8080'}, id='proxies'), + pytest.param('allow_redirects', False, id='allow-redirects-uses-httpx-name'), + ], +) +def test_request_rejects_unknown_kwarg(capturing_transport, kwarg, value): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError, match=kwarg): + http.get('http://example.test/', **{kwarg: value}) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py new file mode 100644 index 0000000000000..7ac2ca585ba57 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py @@ -0,0 +1,44 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import httpx2 as httpx +import pytest + +from datadog_checks.base.utils.http_exceptions import ( + HTTPConnectionError, + HTTPInvalidURLError, + HTTPRequestError, + HTTPTimeoutError, +) +from datadog_checks.base.utils.http_httpx import HTTPXWrapper, _map_httpx_exception + + +@pytest.mark.parametrize( + 'raised,expected', + [ + pytest.param(httpx.ConnectTimeout('boom'), HTTPTimeoutError, id='connect-timeout'), + pytest.param(httpx.ReadTimeout('slow'), HTTPTimeoutError, id='read-timeout'), + pytest.param(httpx.PoolTimeout('pool'), HTTPTimeoutError, id='pool-timeout'), + pytest.param(httpx.ConnectError('refused'), HTTPConnectionError, id='connect-error'), + pytest.param(httpx.ReadError('mid-stream'), HTTPRequestError, id='read-error'), + pytest.param(httpx.LocalProtocolError('bad'), HTTPRequestError, id='local-protocol-error'), + pytest.param(httpx.RequestError('generic'), HTTPRequestError, id='request-error'), + ], +) +def test_request_exception_mapping(raising_transport_factory, raised, expected): + transport = raising_transport_factory(raised) + http = HTTPXWrapper({}, {}, transport=transport) + with pytest.raises(expected): + http.get('http://example.test/') + + +def test_map_httpx_exception_routes_invalid_url(): + mapped = _map_httpx_exception(httpx.InvalidURL('bad url')) + assert isinstance(mapped, HTTPInvalidURLError) + + +def test_request_raises_invalid_url_error(raising_transport_factory): + transport = raising_transport_factory(httpx.InvalidURL('bad url')) + http = HTTPXWrapper({}, {}, transport=transport) + with pytest.raises(HTTPInvalidURLError): + http.get('http://example.test/') diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py new file mode 100644 index 0000000000000..db9b5ba319a06 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py @@ -0,0 +1,47 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import sys + +import pytest + +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + + +def test_close_is_idempotent(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.close() + http.close() + + +def test_context_manager(capturing_transport): + with HTTPXWrapper({}, {}, transport=capturing_transport) as http: + response = http.get('http://example.test/') + assert response.status_code == 200 + + +def test_module_import_fails_without_httpx2(monkeypatch): + monkeypatch.setitem(sys.modules, 'httpx2', None) + monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.http_httpx', raising=False) + with pytest.raises(ImportError): + import datadog_checks.base.utils.http_httpx # noqa: F401 + + +@pytest.mark.parametrize( + 'instance,expected_cls_name', + [ + pytest.param({'use_httpx': True}, 'HTTPXWrapper', id='opt-in'), + pytest.param({'use_httpx': False}, 'RequestsWrapper', id='explicit-default'), + pytest.param({}, 'RequestsWrapper', id='unset-default'), + ], +) +def test_agentcheck_http_dispatch(instance, expected_cls_name): + from datadog_checks.base import AgentCheck + + check = AgentCheck('test', {}, [instance]) + try: + assert type(check.http).__name__ == expected_cls_name + finally: + close = getattr(check.http, 'close', None) + if close is not None: + close() diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py new file mode 100644 index 0000000000000..55da483fc74ff --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py @@ -0,0 +1,42 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import json + +import pytest + +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + +HTTP_VERBS = { + 'get': 'GET', + 'post': 'POST', + 'put': 'PUT', + 'delete': 'DELETE', + 'head': 'HEAD', + 'patch': 'PATCH', + 'options_method': 'OPTIONS', +} + + +@pytest.mark.parametrize('method,verb', HTTP_VERBS.items()) +def test_method_dispatches_with_correct_verb(method, verb, captured_requests, capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + fn = getattr(http, method) + response = fn('http://example.test/path', headers={'X-Test': '1'}) + + assert response.status_code == 200 + assert captured_requests[0].method == verb + + +def test_post_json_body_is_serialized(capturing_transport, captured_requests): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.post('http://example.test/path', json={'a': 1, 'b': 'two'}) + req = captured_requests[0] + assert req.headers['content-type'] == 'application/json' + assert json.loads(req.content) == {'a': 1, 'b': 'two'} + + +def test_request_accepts_stream_kwarg(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + response = http.get('http://example.test/', stream=True) + assert response.status_code == 200 diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py new file mode 100644 index 0000000000000..d3e7700e46670 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -0,0 +1,198 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import logging +from datetime import timedelta + +import httpx2 as httpx +import pytest + +from datadog_checks.base.utils.http_exceptions import HTTPStatusError +from datadog_checks.base.utils.http_httpx import DEFAULT_CHUNK_SIZE, HTTPXResponseAdapter, HTTPXWrapper + + +@pytest.mark.parametrize('status_code', [404, 500]) +def test_response_raise_for_status_raises_on_error_codes(status_transport_factory, status_code): + transport = status_transport_factory(status_code, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + with pytest.raises(HTTPStatusError): + response.raise_for_status() + + +def test_response_iter_content_bytes(status_transport_factory): + transport = status_transport_factory(200, b'abcdef') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + chunks = list(response.iter_content(chunk_size=2)) + assert b''.join(chunks) == b'abcdef' + + +def test_response_iter_content_decode_unicode(status_transport_factory): + transport = status_transport_factory(200, b'abcdef') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + chunks = list(response.iter_content(chunk_size=3, decode_unicode=True)) + assert ''.join(chunks) == 'abcdef' + + +@pytest.mark.parametrize( + 'decode_unicode,expected', + [ + pytest.param(False, [b'a', b'b', b'c'], id='bytes'), + pytest.param(True, ['a', 'b', 'c'], id='decoded-unicode'), + ], +) +def test_response_iter_lines(status_transport_factory, decode_unicode, expected): + transport = status_transport_factory(200, b'a\nb\nc') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert list(response.iter_lines(decode_unicode=decode_unicode)) == expected + + +def test_response_iter_content_empty_body_yields_nothing(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert list(response.iter_content()) == [] + + +def test_response_iter_content_default_chunk_size_uses_default(status_transport_factory): + body = b'X' * (DEFAULT_CHUNK_SIZE * 3 + 5) + transport = status_transport_factory(200, body) + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + chunks = list(response.iter_content()) + assert b''.join(chunks) == body + assert all(len(chunk) <= DEFAULT_CHUNK_SIZE for chunk in chunks) + assert any(len(chunk) == DEFAULT_CHUNK_SIZE for chunk in chunks) + + +@pytest.mark.parametrize( + 'charset,raw,expected', + [ + pytest.param('utf-8', 'café'.encode('utf-8'), 'café', id='utf-8'), + pytest.param('iso-8859-1', 'café'.encode('iso-8859-1'), 'café', id='iso-8859-1'), + ], +) +def test_response_iter_content_decode_uses_response_encoding(charset, raw, expected): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=raw, headers={'Content-Type': f'text/plain; charset={charset}'}) + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + chunks = list(response.iter_content(chunk_size=64, decode_unicode=True)) + assert ''.join(chunks) == expected + + +@pytest.mark.parametrize( + 'charset,line', + [ + pytest.param('utf-8', 'café', id='utf-8'), + pytest.param('iso-8859-1', 'café', id='iso-8859-1'), + ], +) +def test_response_iter_lines_decode_uses_response_encoding(charset, line): + raw = (line + '\n' + line).encode(charset) + + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=raw, headers={'Content-Type': f'text/plain; charset={charset}'}) + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + encoded_lines = list(response.iter_lines(decode_unicode=False)) + assert encoded_lines == [line.encode(charset), line.encode(charset)] + + +def test_response_iter_lines_rejects_delimiter(status_transport_factory): + transport = status_transport_factory(200, b'a\nb\n') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + with pytest.raises(NotImplementedError): + list(response.iter_lines(delimiter=b'|')) + + +def test_response_elapsed_returns_zero_on_runtime_error(caplog): + class _FakeResponse: + url = 'http://example.test/' + + @property + def elapsed(self): + raise RuntimeError('not measured') + + adapter = HTTPXResponseAdapter(_FakeResponse()) # type: ignore[arg-type] + with caplog.at_level(logging.DEBUG, logger='datadog_checks.base.utils.http_httpx'): + assert adapter.elapsed == timedelta(0) + assert any('elapsed unavailable' in record.message for record in caplog.records) + + +@pytest.mark.parametrize('status_code,expected_ok', [(200, True), (204, True), (301, True), (400, False), (500, False)]) +def test_response_ok_property(status_transport_factory, status_code, expected_ok): + transport = status_transport_factory(status_code, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.ok is expected_ok + + +def test_response_reason_from_httpx_response(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.reason == 'OK' + + +def test_response_text_decodes_body(status_transport_factory): + transport = status_transport_factory(200, b'hello') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.text == 'hello' + + +def test_response_json_returns_decoded_object(): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={'a': 1}) + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + assert response.json() == {'a': 1} + + +def test_response_content_returns_raw_bytes(status_transport_factory): + transport = status_transport_factory(200, b'\x00\x01\x02') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.content == b'\x00\x01\x02' + + +def test_response_headers_exposed(): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={'X-Custom': 'value'}, content=b'') + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + assert response.headers['X-Custom'] == 'value' + + +def test_response_url_reflects_request_url(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/path') + assert str(response.url) == 'http://example.test/path' + + +def test_response_encoding_property_exposed(status_transport_factory): + transport = status_transport_factory(200, b'hello') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert hasattr(response, 'encoding') + encoding = response.encoding + assert encoding is None or isinstance(encoding, str) + + +def test_response_cookies_exposed(): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={'Set-Cookie': 'session=abc123'}, content=b'') + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + assert response.cookies['session'] == 'abc123' diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_stream_connection_lines.py b/datadog_checks_base/tests/base/utils/http_httpx/test_stream_connection_lines.py new file mode 100644 index 0000000000000..fb27824db248c --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_stream_connection_lines.py @@ -0,0 +1,53 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import logging +from unittest import mock + +import pytest +from requests.exceptions import ConnectionError as RequestsConnectionError + +from datadog_checks.base.checks.openmetrics.v2.scraper.base_scraper import OpenMetricsScraper +from datadog_checks.base.utils.http_exceptions import HTTPConnectionError + + +def _scraper_with_connection_error(exception, *, ignore_connection_errors): + scraper = OpenMetricsScraper.__new__(OpenMetricsScraper) + scraper.endpoint = 'http://example.test/metrics' + scraper.ignore_connection_errors = ignore_connection_errors + scraper.log = logging.getLogger('test_stream_connection_lines') + scraper.get_connection = mock.Mock(side_effect=exception) + return scraper + + +def test_connection_error_warning_path(caplog): + scraper = _scraper_with_connection_error( + HTTPConnectionError('refused'), + ignore_connection_errors=True, + ) + with caplog.at_level(logging.WARNING, logger='test_stream_connection_lines'): + assert list(scraper.stream_connection_lines()) == [] + assert any( + 'OpenMetrics endpoint http://example.test/metrics is not accessible' in record.message + for record in caplog.records + ) + assert any('refused' in record.message for record in caplog.records) + + +def test_connection_error_reraises_when_not_ignored(): + scraper = _scraper_with_connection_error( + HTTPConnectionError('refused'), + ignore_connection_errors=False, + ) + with pytest.raises(HTTPConnectionError): + list(scraper.stream_connection_lines()) + + +def test_requests_connection_error_still_handled(caplog): + scraper = _scraper_with_connection_error( + RequestsConnectionError('boom'), + ignore_connection_errors=True, + ) + with caplog.at_level(logging.WARNING, logger='test_stream_connection_lines'): + assert list(scraper.stream_connection_lines()) == [] + assert any('boom' in record.message for record in caplog.records) From a658ffda1843c46ce361818c510c531114a87c87 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 13:23:23 -0400 Subject: [PATCH 02/45] Address Round 1 review findings --- .../openmetrics/v2/scraper/base_scraper.py | 4 +- .../datadog_checks/base/utils/http_httpx.py | 67 +++++++++++++------ .../base/utils/http_httpx/test_config.py | 55 +++++++++++++++ .../base/utils/http_httpx/test_lifecycle.py | 17 +++-- .../base/utils/http_httpx/test_response.py | 36 ++++++---- 5 files changed, 137 insertions(+), 42 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py index a3211f6c39405..3ba8f717d09d6 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py @@ -13,7 +13,7 @@ from prometheus_client import Metric from prometheus_client.openmetrics.parser import text_fd_to_metric_families as parse_openmetrics from prometheus_client.parser import text_fd_to_metric_families as parse_prometheus -from requests.exceptions import ConnectionError +from requests.exceptions import ConnectionError as RequestsConnectionError from datadog_checks.base.agent import datadog_agent from datadog_checks.base.checks.openmetrics.v2.first_scrape_handler import first_scrape_handler @@ -406,7 +406,7 @@ def stream_connection_lines(self): self._content_type = connection.headers.get('Content-Type', '') for line in connection.iter_lines(decode_unicode=True): yield line - except (ConnectionError, HTTPConnectionError) as e: + except (RequestsConnectionError, HTTPConnectionError) as e: if self.ignore_connection_errors: self.log.warning("OpenMetrics endpoint %s is not accessible: %s", self.endpoint, e) else: diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index a91b7c37acb74..ec15f643c7125 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -6,7 +6,7 @@ import logging from collections.abc import Iterator, Mapping from datetime import timedelta -from typing import Any +from typing import Any, Self import httpx2 as httpx from binary import KIBIBYTE @@ -64,7 +64,10 @@ def _make_timeout(connect: float, read: float) -> httpx.Timeout: - return httpx.Timeout(connect=connect, read=read, write=None, pool=None) + # Apply read to write and pool so the tuple form has the same all-phase coverage as + # passing a scalar timeout to httpx.Timeout(). Leaving write/pool unbounded would + # let a slow connection-pool acquisition or write phase hang the check. + return httpx.Timeout(connect=connect, read=read, write=read, pool=read) def _build_basic_auth(config: dict[str, Any]) -> httpx.BasicAuth | None: @@ -158,6 +161,10 @@ def encoding(self) -> str | None: @encoding.setter def encoding(self, value: str | None) -> None: + # Pass through to httpx2. Note: in httpx2 the encoding only affects subsequent + # ``.text`` and ``iter_text``/``iter_lines`` decoding. It does NOT retroactively + # change ``response.content`` or already-yielded chunks. Callers that mutate + # encoding after streaming has started will see the new value only on the next read. self._response.encoding = value @property @@ -165,7 +172,10 @@ def url(self) -> str: return str(self._response.url) @property - def cookies(self) -> httpx.Cookies: + def cookies(self) -> Mapping[str, str]: + # httpx.Cookies satisfies Mapping[str, str] structurally. The narrower annotation + # avoids leaking the httpx2 type through the wrapper surface; callers wanting the + # richer httpx2 API can still read ``response._response.cookies`` directly. return self._response.cookies @property @@ -194,9 +204,12 @@ def close(self) -> None: def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: effective_size = chunk_size if chunk_size is not None else DEFAULT_CHUNK_SIZE - encoding = self._response.encoding or 'utf-8' - for chunk in self._response.iter_bytes(chunk_size=effective_size): - yield chunk.decode(encoding) if decode_unicode else chunk + if decode_unicode: + # httpx2 iter_text uses an incremental decoder so a multibyte char straddling a + # byte-chunk boundary does not raise UnicodeDecodeError. + yield from self._response.iter_text(chunk_size=effective_size) + return + yield from self._response.iter_bytes(chunk_size=effective_size) def iter_lines( self, @@ -208,9 +221,12 @@ def iter_lines( raise NotImplementedError("HTTPXResponseAdapter.iter_lines does not support custom delimiters") encoding = self._response.encoding or 'utf-8' for line in self._response.iter_lines(): - yield line if decode_unicode else line.encode(encoding) + # httpx2 already decoded the line using the declared charset. Re-encoding can corrupt + # bytes that round-trip lossily under that charset, so substitute with the Unicode + # replacement char on errors rather than raising. + yield line if decode_unicode else line.encode(encoding, errors='replace') - def __enter__(self) -> 'HTTPXResponseAdapter': + def __enter__(self) -> Self: return self def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: @@ -354,10 +370,14 @@ def options_method(self, url: str, **options: Any) -> HTTPXResponseAdapter: return self._request('OPTIONS', url, options) def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXResponseAdapter: + # All requests stream by default (stream=True). The returned HTTPXResponseAdapter holds + # the connection open until the body is read (via .content/.text/.json/.iter_*) or close() + # is called. Callers that do not fully consume the body should use it as a context manager + # or call .close() to release the connection back to the pool. if self._log_requests: self.logger.debug('Sending %s request to %s', method, url) - request_kwargs = self._build_request_kwargs(options) + request_kwargs = self._build_request_kwargs(options, method=method, url=url) follow_redirects = request_kwargs.pop('follow_redirects', httpx.USE_CLIENT_DEFAULT) try: request = self._client.build_request(method, url, **request_kwargs) @@ -366,32 +386,29 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXRespo raise _map_httpx_exception(exc) from exc return HTTPXResponseAdapter(response) - def _build_request_kwargs(self, options: dict[str, Any]) -> dict[str, Any]: + def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', url: str = '') -> dict[str, Any]: """Translate call-site options to httpx2.Client.request kwargs.""" # OM v2 scraper injects stream=True unconditionally (base_scraper.py:459). The wrapper - # always streams internally, so drop the kwarg silently rather than raising on it. + # always streams internally, so drop the kwarg rather than raising on it. Log at debug + # so the silent drop is observable when investigating per-request behavior. + if 'stream' in options: + self.logger.debug('HTTPXWrapper dropping unsupported per-request kwarg: stream') options = {k: v for k, v in options.items() if k != 'stream'} unknown = set(options) - REQUEST_KWARGS if unknown: - raise TypeError(f"HTTPXWrapper does not support per-request kwargs: {sorted(unknown)}") + context = f' for {method} {url}' if method or url else '' + raise TypeError(f"HTTPXWrapper does not support per-request kwargs{context}: {sorted(unknown)}") kwargs: dict[str, Any] = {} passthrough = ('params', 'json', 'data', 'content', 'files', 'cookies') for key in passthrough: if key in options: kwargs[key] = options[key] - extra_headers = options.get('extra_headers') headers = options.get('headers') - merged_headers: dict[str, str] | None = None + extra_headers = options.get('extra_headers') if headers is not None or extra_headers is not None: - merged_headers = {} - if headers is not None: - merged_headers.update(headers) - if extra_headers is not None: - merged_headers.update(extra_headers) - if merged_headers is not None: - kwargs['headers'] = merged_headers + kwargs['headers'] = {**(headers or {}), **(extra_headers or {})} if 'timeout' in options: timeout_value = options['timeout'] @@ -410,7 +427,7 @@ def close(self) -> None: if client is not None: client.close() - def __enter__(self) -> 'HTTPXWrapper': + def __enter__(self) -> Self: return self def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: @@ -421,4 +438,10 @@ def __del__(self) -> None: # no cov try: self.close() except AttributeError: + # A request was never made or an error occurred during instantiation before _client + # was ever defined (since __del__ executes even if __init__ fails). pass + except Exception as exc: + # Don't propagate from __del__; interpreter would only log a warning and continue. + # Log at debug so the swallow is observable when investigating. + self.logger.debug('HTTPXWrapper.close raised during __del__: %r', exc) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py index e83e74cfb95e2..7f29caea7f2f3 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py @@ -123,6 +123,37 @@ def test_remapper_renames_field(capturing_transport): assert http.options['verify'] is False +def test_remapper_invert_flips_value(capturing_transport): + # disable_ssl_validation=True means "do NOT verify" -> options['verify'] is False. + remapper = {'disable_ssl_validation': {'name': 'tls_verify', 'invert': True}} + http = HTTPXWrapper({'disable_ssl_validation': True}, {}, remapper=remapper, transport=capturing_transport) + assert http.options['verify'] is False + + +def test_remapper_invert_with_explicit_default(capturing_transport): + # Remapped field absent from instance, explicit default takes effect, then invert flips it. + remapper = {'disable_ssl_validation': {'name': 'tls_verify', 'invert': True, 'default': False}} + http = HTTPXWrapper({}, {}, remapper=remapper, transport=capturing_transport) + # default=False, invert flips to True -> verify is True. + assert http.options['verify'] is True + + +def test_remapper_instance_wins_over_remapped_field(capturing_transport): + # If the standard field is present in instance, the remapped alternative is ignored. + remapper = {'ssl_validation': {'name': 'tls_verify'}} + http = HTTPXWrapper( + {'ssl_validation': False, 'tls_verify': True}, {}, remapper=remapper, transport=capturing_transport + ) + assert http.options['verify'] is True + + +def test_remapper_ignores_unknown_target_field(capturing_transport): + # Remapper targeting a non-STANDARD_FIELDS key is silently ignored. + remapper = {'some_alias': {'name': 'definitely_not_a_known_field'}} + http = HTTPXWrapper({'some_alias': 'ignored'}, {}, remapper=remapper, transport=capturing_transport) + assert 'definitely_not_a_known_field' not in http.options + + @pytest.mark.parametrize( 'kwarg,value', [ @@ -134,3 +165,27 @@ def test_request_rejects_unknown_kwarg(capturing_transport, kwarg, value): http = HTTPXWrapper({}, {}, transport=capturing_transport) with pytest.raises(TypeError, match=kwarg): http.get('http://example.test/', **{kwarg: value}) + + +def test_init_config_timeout_used_when_instance_has_none(capturing_transport): + http = HTTPXWrapper({}, {'timeout': 42}, transport=capturing_transport) + connect, read = http.options['timeout'] + assert connect == 42.0 + assert read == 42.0 + + +def test_instance_timeout_overrides_init_config_timeout(capturing_transport): + http = HTTPXWrapper({'timeout': 7}, {'timeout': 42}, transport=capturing_transport) + connect, read = http.options['timeout'] + assert connect == 7.0 + assert read == 7.0 + + +def test_init_config_log_requests_used_when_instance_has_none(capturing_transport): + http = HTTPXWrapper({}, {'log_requests': True}, transport=capturing_transport) + assert http._log_requests is True + + +def test_instance_log_requests_overrides_init_config(capturing_transport): + http = HTTPXWrapper({'log_requests': False}, {'log_requests': True}, transport=capturing_transport) + assert http._log_requests is False diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py index db9b5ba319a06..0ac02187225db 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py @@ -23,8 +23,14 @@ def test_context_manager(capturing_transport): def test_module_import_fails_without_httpx2(monkeypatch): monkeypatch.setitem(sys.modules, 'httpx2', None) monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.http_httpx', raising=False) - with pytest.raises(ImportError): + with pytest.raises(ImportError, match='httpx2'): import datadog_checks.base.utils.http_httpx # noqa: F401 + # After monkeypatch teardown the module must import cleanly again so sibling tests are + # not affected by a leaked broken import state. + monkeypatch.undo() + import datadog_checks.base.utils.http_httpx as restored + + assert hasattr(restored, 'HTTPXWrapper') @pytest.mark.parametrize( @@ -42,6 +48,9 @@ def test_agentcheck_http_dispatch(instance, expected_cls_name): try: assert type(check.http).__name__ == expected_cls_name finally: - close = getattr(check.http, 'close', None) - if close is not None: - close() + # Read the cached attribute directly (AgentCheck.http is a property that may rebuild). + http = check._http + if http is not None: + close = getattr(http, 'close', None) + if close is not None: + close() diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index d3e7700e46670..84abddd92dd5c 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -113,14 +113,12 @@ def test_response_iter_lines_rejects_delimiter(status_transport_factory): def test_response_elapsed_returns_zero_on_runtime_error(caplog): - class _FakeResponse: - url = 'http://example.test/' - - @property - def elapsed(self): - raise RuntimeError('not measured') - - adapter = HTTPXResponseAdapter(_FakeResponse()) # type: ignore[arg-type] + # httpx2 raises RuntimeError from ``.elapsed`` if the response was constructed in-memory + # rather than measured against a real network round-trip. Use a real httpx.Response with + # an attached request so the adapter can format its debug log without secondary errors. + request = httpx.Request('GET', 'http://example.test/') + response = httpx.Response(200, content=b'x', request=request) + adapter = HTTPXResponseAdapter(response) with caplog.at_level(logging.DEBUG, logger='datadog_checks.base.utils.http_httpx'): assert adapter.elapsed == timedelta(0) assert any('elapsed unavailable' in record.message for record in caplog.records) @@ -180,13 +178,23 @@ def test_response_url_reflects_request_url(status_transport_factory): assert str(response.url) == 'http://example.test/path' -def test_response_encoding_property_exposed(status_transport_factory): - transport = status_transport_factory(200, b'hello') - http = HTTPXWrapper({}, {}, transport=transport) +def test_response_encoding_property_reflects_declared_charset(): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b'data', headers={'Content-Type': 'text/plain; charset=iso-8859-1'}) + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + assert response.encoding == 'iso-8859-1' + + +def test_response_encoding_property_settable_through_adapter(): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b'data', headers={'Content-Type': 'text/plain; charset=utf-8'}) + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) response = http.get('http://example.test/') - assert hasattr(response, 'encoding') - encoding = response.encoding - assert encoding is None or isinstance(encoding, str) + response.encoding = 'iso-8859-1' + assert response.encoding == 'iso-8859-1' def test_response_cookies_exposed(): From 978c64d7fa4ebe1608c9a7caa682de60d8a038db Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 13:23:23 -0400 Subject: [PATCH 03/45] Tighten comments from human review --- .../datadog_checks/base/utils/http_httpx.py | 47 +++++++------------ .../base/utils/http_httpx/test_response.py | 5 +- 2 files changed, 19 insertions(+), 33 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index ec15f643c7125..3c5cf8f62af5c 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -64,9 +64,7 @@ def _make_timeout(connect: float, read: float) -> httpx.Timeout: - # Apply read to write and pool so the tuple form has the same all-phase coverage as - # passing a scalar timeout to httpx.Timeout(). Leaving write/pool unbounded would - # let a slow connection-pool acquisition or write phase hang the check. + # Pass the read timeout to every phase so write and pool aren't left unbounded. return httpx.Timeout(connect=connect, read=read, write=read, pool=read) @@ -103,8 +101,6 @@ def _build_timeout(config: dict[str, Any]) -> tuple[float, float]: def _map_httpx_exception(exc: httpx.HTTPError | httpx.InvalidURL) -> HTTPError: """Translate an httpx2 exception into the library-agnostic equivalent.""" - # ConnectError -> HTTPConnectionError pairs 1:1 with the Step 3a widening (PR #22864). - # Mid-stream NetworkError/ReadError/WriteError stay HTTPRequestError on purpose. if isinstance(exc, httpx.InvalidURL): return HTTPInvalidURLError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) if isinstance(exc, httpx.TimeoutException): @@ -161,10 +157,8 @@ def encoding(self) -> str | None: @encoding.setter def encoding(self, value: str | None) -> None: - # Pass through to httpx2. Note: in httpx2 the encoding only affects subsequent - # ``.text`` and ``iter_text``/``iter_lines`` decoding. It does NOT retroactively - # change ``response.content`` or already-yielded chunks. Callers that mutate - # encoding after streaming has started will see the new value only on the next read. + # Encoding is consulted at decode time, not when set. Bytes already read by + # iter_text or iter_lines stay decoded under the previous encoding. self._response.encoding = value @property @@ -173,9 +167,8 @@ def url(self) -> str: @property def cookies(self) -> Mapping[str, str]: - # httpx.Cookies satisfies Mapping[str, str] structurally. The narrower annotation - # avoids leaking the httpx2 type through the wrapper surface; callers wanting the - # richer httpx2 API can still read ``response._response.cookies`` directly. + # Narrowed to Mapping to keep httpx2 out of the wrapper surface. Callers needing + # the richer httpx.Cookies API can reach self._response.cookies directly. return self._response.cookies @property @@ -205,15 +198,15 @@ def close(self) -> None: def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: effective_size = chunk_size if chunk_size is not None else DEFAULT_CHUNK_SIZE if decode_unicode: - # httpx2 iter_text uses an incremental decoder so a multibyte char straddling a - # byte-chunk boundary does not raise UnicodeDecodeError. + # iter_text uses an incremental decoder. Manual chunk decoding breaks on + # multibyte chars split across boundaries. yield from self._response.iter_text(chunk_size=effective_size) return yield from self._response.iter_bytes(chunk_size=effective_size) def iter_lines( self, - chunk_size: int | None = None, # noqa: ARG002 - httpx2 buffers lines internally; kept for HTTPResponseProtocol parity + chunk_size: int | None = None, # noqa: ARG002 - httpx2 buffers lines internally. Kept for HTTPResponseProtocol parity decode_unicode: bool = False, delimiter: bytes | str | None = None, ) -> Iterator[bytes | str]: @@ -221,9 +214,7 @@ def iter_lines( raise NotImplementedError("HTTPXResponseAdapter.iter_lines does not support custom delimiters") encoding = self._response.encoding or 'utf-8' for line in self._response.iter_lines(): - # httpx2 already decoded the line using the declared charset. Re-encoding can corrupt - # bytes that round-trip lossily under that charset, so substitute with the Unicode - # replacement char on errors rather than raising. + # errors='replace' tolerates bytes that don't round-trip under the declared charset. yield line if decode_unicode else line.encode(encoding, errors='replace') def __enter__(self) -> Self: @@ -270,7 +261,8 @@ def __init__( timeout = _build_timeout(config) allow_redirects = is_affirmative(config['allow_redirects']) - # proxies=None mirrors RequestsWrapper.options for consumers (e.g. http_check). Wiring is Phase 3. + # proxies=None mirrors RequestsWrapper.options for consumers (e.g. http_check). + # TODO: wire proxies through to httpx2 as part of the ongoing httpx migration. self.options: dict[str, Any] = { 'auth': auth, 'cert': cert, @@ -370,10 +362,8 @@ def options_method(self, url: str, **options: Any) -> HTTPXResponseAdapter: return self._request('OPTIONS', url, options) def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXResponseAdapter: - # All requests stream by default (stream=True). The returned HTTPXResponseAdapter holds - # the connection open until the body is read (via .content/.text/.json/.iter_*) or close() - # is called. Callers that do not fully consume the body should use it as a context manager - # or call .close() to release the connection back to the pool. + # stream=True keeps the connection open until the body is consumed (.content/.text/.iter_*) + # or close() is called. Callers should use it as a context manager when not reading the body. if self._log_requests: self.logger.debug('Sending %s request to %s', method, url) @@ -388,9 +378,8 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXRespo def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', url: str = '') -> dict[str, Any]: """Translate call-site options to httpx2.Client.request kwargs.""" - # OM v2 scraper injects stream=True unconditionally (base_scraper.py:459). The wrapper - # always streams internally, so drop the kwarg rather than raising on it. Log at debug - # so the silent drop is observable when investigating per-request behavior. + # Drop stream silently because OM v2 scraper passes it unconditionally (base_scraper.py:459). + # The wrapper streams internally, so the kwarg is meaningless. if 'stream' in options: self.logger.debug('HTTPXWrapper dropping unsupported per-request kwarg: stream') options = {k: v for k, v in options.items() if k != 'stream'} @@ -438,10 +427,8 @@ def __del__(self) -> None: # no cov try: self.close() except AttributeError: - # A request was never made or an error occurred during instantiation before _client - # was ever defined (since __del__ executes even if __init__ fails). + # __del__ runs even if __init__ failed before self._client was set. pass except Exception as exc: - # Don't propagate from __del__; interpreter would only log a warning and continue. - # Log at debug so the swallow is observable when investigating. + # Don't propagate from __del__ (interpreter handles it). Log at debug so the swallow is observable. self.logger.debug('HTTPXWrapper.close raised during __del__: %r', exc) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index 84abddd92dd5c..614347260a053 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -113,9 +113,8 @@ def test_response_iter_lines_rejects_delimiter(status_transport_factory): def test_response_elapsed_returns_zero_on_runtime_error(caplog): - # httpx2 raises RuntimeError from ``.elapsed`` if the response was constructed in-memory - # rather than measured against a real network round-trip. Use a real httpx.Response with - # an attached request so the adapter can format its debug log without secondary errors. + # httpx2's .elapsed raises RuntimeError for in-memory responses. The real httpx.Request is + # attached so the adapter's debug log can format request info without a secondary error. request = httpx.Request('GET', 'http://example.test/') response = httpx.Response(200, content=b'x', request=request) adapter = HTTPXResponseAdapter(response) From 559bd70bbd597e6ce5cd26982b11bc4d6db1fde4 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 12:22:52 -0400 Subject: [PATCH 04/45] Use httpx2 directly instead of aliasing as httpx --- .../datadog_checks/base/utils/http_httpx.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index 3c5cf8f62af5c..c959b851a2279 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -8,7 +8,7 @@ from datetime import timedelta from typing import Any, Self -import httpx2 as httpx +import httpx2 from binary import KIBIBYTE from datadog_checks.base.config import is_affirmative @@ -63,14 +63,14 @@ ) -def _make_timeout(connect: float, read: float) -> httpx.Timeout: +def _make_timeout(connect: float, read: float) -> httpx2.Timeout: # Pass the read timeout to every phase so write and pool aren't left unbounded. - return httpx.Timeout(connect=connect, read=read, write=read, pool=read) + return httpx2.Timeout(connect=connect, read=read, write=read, pool=read) -def _build_basic_auth(config: dict[str, Any]) -> httpx.BasicAuth | None: +def _build_basic_auth(config: dict[str, Any]) -> httpx2.BasicAuth | None: if config['username'] is not None and config['password'] is not None: - return httpx.BasicAuth(config['username'], config['password']) + return httpx2.BasicAuth(config['username'], config['password']) return None @@ -99,21 +99,21 @@ def _build_timeout(config: dict[str, Any]) -> tuple[float, float]: return connect, read -def _map_httpx_exception(exc: httpx.HTTPError | httpx.InvalidURL) -> HTTPError: +def _map_httpx_exception(exc: httpx2.HTTPError | httpx2.InvalidURL) -> HTTPError: """Translate an httpx2 exception into the library-agnostic equivalent.""" - if isinstance(exc, httpx.InvalidURL): + if isinstance(exc, httpx2.InvalidURL): return HTTPInvalidURLError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) - if isinstance(exc, httpx.TimeoutException): + if isinstance(exc, httpx2.TimeoutException): return HTTPTimeoutError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) - if isinstance(exc, httpx.ConnectError): + if isinstance(exc, httpx2.ConnectError): return HTTPConnectionError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) - if isinstance(exc, httpx.HTTPStatusError): + if isinstance(exc, httpx2.HTTPStatusError): return HTTPStatusError( str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None), response=getattr(exc, 'response', None), ) - if isinstance(exc, httpx.RequestError): + if isinstance(exc, httpx2.RequestError): return HTTPRequestError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) return HTTPError(str(exc) or exc.__class__.__name__) @@ -123,7 +123,7 @@ class HTTPXResponseAdapter: __slots__ = ('_response',) - def __init__(self, response: httpx.Response) -> None: + def __init__(self, response: httpx2.Response) -> None: self._response = response @property @@ -168,7 +168,7 @@ def url(self) -> str: @property def cookies(self) -> Mapping[str, str]: # Narrowed to Mapping to keep httpx2 out of the wrapper surface. Callers needing - # the richer httpx.Cookies API can reach self._response.cookies directly. + # the richer httpx2.Cookies API can reach self._response.cookies directly. return self._response.cookies @property @@ -189,7 +189,7 @@ def raise_for_status(self) -> None: return try: self._response.raise_for_status() - except httpx.HTTPStatusError as exc: + except httpx2.HTTPStatusError as exc: raise _map_httpx_exception(exc) from exc def close(self) -> None: @@ -241,7 +241,7 @@ def __init__( init_config: dict[str, Any] | None = None, remapper: dict[str, dict[str, Any]] | None = None, logger: logging.Logger | None = None, - transport: httpx.BaseTransport | None = None, + transport: httpx2.BaseTransport | None = None, ) -> None: self.logger = logger or LOGGER init_config = init_config or {} @@ -309,7 +309,7 @@ def _resolve_config( config[field] = value return config - def _build_client(self, transport: httpx.BaseTransport | None) -> httpx.Client: + def _build_client(self, transport: httpx2.BaseTransport | None) -> httpx2.Client: kwargs: dict[str, Any] = { 'headers': self.options['headers'], 'timeout': _make_timeout(self.options['timeout'][0], self.options['timeout'][1]), @@ -322,7 +322,7 @@ def _build_client(self, transport: httpx.BaseTransport | None) -> httpx.Client: kwargs['auth'] = self.options['auth'] if transport is not None: kwargs['transport'] = transport - return httpx.Client(**kwargs) + return httpx2.Client(**kwargs) def get_header(self, name: str, default: str | None = None) -> str | None: for key, value in self.options['headers'].items(): @@ -368,11 +368,11 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXRespo self.logger.debug('Sending %s request to %s', method, url) request_kwargs = self._build_request_kwargs(options, method=method, url=url) - follow_redirects = request_kwargs.pop('follow_redirects', httpx.USE_CLIENT_DEFAULT) + follow_redirects = request_kwargs.pop('follow_redirects', httpx2.USE_CLIENT_DEFAULT) try: request = self._client.build_request(method, url, **request_kwargs) response = self._client.send(request, stream=True, follow_redirects=follow_redirects) - except (httpx.HTTPError, httpx.InvalidURL) as exc: + except (httpx2.HTTPError, httpx2.InvalidURL) as exc: raise _map_httpx_exception(exc) from exc return HTTPXResponseAdapter(response) From 2caf19000e2235c93a5bc67e7421037ae27e2d6c Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 16:06:00 -0400 Subject: [PATCH 05/45] Rename to httpx2 --- datadog_checks_base/changelog.d/22676.added | 2 +- .../datadog_checks/base/checks/base.py | 4 +- .../base/utils/{http_httpx.py => httpx2.py} | 0 .../tests/base/utils/http_httpx/conftest.py | 46 ---------------- .../utils/{http_httpx => httpx2}/__init__.py | 0 .../utils/{http_httpx => httpx2}/common.py | 0 .../tests/base/utils/httpx2/conftest.py | 46 ++++++++++++++++ .../{http_httpx => httpx2}/test_auth_basic.py | 2 +- .../{http_httpx => httpx2}/test_config.py | 2 +- .../{http_httpx => httpx2}/test_exceptions.py | 22 ++++---- .../{http_httpx => httpx2}/test_lifecycle.py | 12 ++--- .../{http_httpx => httpx2}/test_methods.py | 2 +- .../{http_httpx => httpx2}/test_response.py | 54 +++++++++---------- .../test_stream_connection_lines.py | 0 14 files changed, 96 insertions(+), 96 deletions(-) rename datadog_checks_base/datadog_checks/base/utils/{http_httpx.py => httpx2.py} (100%) delete mode 100644 datadog_checks_base/tests/base/utils/http_httpx/conftest.py rename datadog_checks_base/tests/base/utils/{http_httpx => httpx2}/__init__.py (100%) rename datadog_checks_base/tests/base/utils/{http_httpx => httpx2}/common.py (100%) create mode 100644 datadog_checks_base/tests/base/utils/httpx2/conftest.py rename datadog_checks_base/tests/base/utils/{http_httpx => httpx2}/test_auth_basic.py (94%) rename datadog_checks_base/tests/base/utils/{http_httpx => httpx2}/test_config.py (99%) rename datadog_checks_base/tests/base/utils/{http_httpx => httpx2}/test_exceptions.py (51%) rename datadog_checks_base/tests/base/utils/{http_httpx => httpx2}/test_lifecycle.py (80%) rename datadog_checks_base/tests/base/utils/{http_httpx => httpx2}/test_methods.py (95%) rename datadog_checks_base/tests/base/utils/{http_httpx => httpx2}/test_response.py (77%) rename datadog_checks_base/tests/base/utils/{http_httpx => httpx2}/test_stream_connection_lines.py (100%) diff --git a/datadog_checks_base/changelog.d/22676.added b/datadog_checks_base/changelog.d/22676.added index 8125821a3d951..1d81f5b40429e 100644 --- a/datadog_checks_base/changelog.d/22676.added +++ b/datadog_checks_base/changelog.d/22676.added @@ -1 +1 @@ -Add library-agnostic HTTP mocks/proto/exceptions and migrate intg tests. Add ``HTTPXWrapper``, an httpx2-backed HTTP client opt-in via the ``use_httpx`` instance config flag. The default remains the existing requests-based client. +Add library-agnostic HTTP mocks/proto/exceptions and migrate intg tests. Add ``HTTPXWrapper``, an httpx2-backed HTTP client opt-in via the ``use_httpx2`` instance config flag. The default remains the existing requests-based client. diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 9ec690ab2432b..537629fbff9fd 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -413,8 +413,8 @@ def http(self) -> HTTPClientProtocol: """ if not hasattr(self, '_http'): instance = self.instance or {} - if is_affirmative(instance.get('use_httpx', False)): - from datadog_checks.base.utils.http_httpx import HTTPXWrapper + if is_affirmative(instance.get('use_httpx2', False)): + from datadog_checks.base.utils.httpx2 import HTTPXWrapper self._http = HTTPXWrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) else: diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py similarity index 100% rename from datadog_checks_base/datadog_checks/base/utils/http_httpx.py rename to datadog_checks_base/datadog_checks/base/utils/httpx2.py diff --git a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py deleted file mode 100644 index 272e7b1bc1228..0000000000000 --- a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py +++ /dev/null @@ -1,46 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -from collections.abc import Callable - -import httpx2 as httpx -import pytest - - -@pytest.fixture -def status_transport_factory() -> Callable[[int, bytes | str], httpx.MockTransport]: - def _factory(status_code: int, body: bytes | str = b''): - def handler(_request: httpx.Request) -> httpx.Response: - if isinstance(body, str): - return httpx.Response(status_code, text=body) - return httpx.Response(status_code, content=body) - - return httpx.MockTransport(handler) - - return _factory - - -@pytest.fixture -def raising_transport_factory() -> Callable[[Exception], httpx.MockTransport]: - def _factory(exc: Exception): - def handler(_request: httpx.Request) -> httpx.Response: - raise exc - - return httpx.MockTransport(handler) - - return _factory - - -@pytest.fixture -def captured_requests() -> list[httpx.Request]: - return [] - - -@pytest.fixture -def capturing_transport(captured_requests: list[httpx.Request]) -> httpx.MockTransport: - def handler(request: httpx.Request) -> httpx.Response: - _ = request.content - captured_requests.append(request) - return httpx.Response(200, json={'ok': True}) - - return httpx.MockTransport(handler) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/__init__.py b/datadog_checks_base/tests/base/utils/httpx2/__init__.py similarity index 100% rename from datadog_checks_base/tests/base/utils/http_httpx/__init__.py rename to datadog_checks_base/tests/base/utils/httpx2/__init__.py diff --git a/datadog_checks_base/tests/base/utils/http_httpx/common.py b/datadog_checks_base/tests/base/utils/httpx2/common.py similarity index 100% rename from datadog_checks_base/tests/base/utils/http_httpx/common.py rename to datadog_checks_base/tests/base/utils/httpx2/common.py diff --git a/datadog_checks_base/tests/base/utils/httpx2/conftest.py b/datadog_checks_base/tests/base/utils/httpx2/conftest.py new file mode 100644 index 0000000000000..f08eb6f795b0d --- /dev/null +++ b/datadog_checks_base/tests/base/utils/httpx2/conftest.py @@ -0,0 +1,46 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from collections.abc import Callable + +import httpx2 +import pytest + + +@pytest.fixture +def status_transport_factory() -> Callable[[int, bytes | str], httpx2.MockTransport]: + def _factory(status_code: int, body: bytes | str = b''): + def handler(_request: httpx2.Request) -> httpx2.Response: + if isinstance(body, str): + return httpx2.Response(status_code, text=body) + return httpx2.Response(status_code, content=body) + + return httpx2.MockTransport(handler) + + return _factory + + +@pytest.fixture +def raising_transport_factory() -> Callable[[Exception], httpx2.MockTransport]: + def _factory(exc: Exception): + def handler(_request: httpx2.Request) -> httpx2.Response: + raise exc + + return httpx2.MockTransport(handler) + + return _factory + + +@pytest.fixture +def captured_requests() -> list[httpx2.Request]: + return [] + + +@pytest.fixture +def capturing_transport(captured_requests: list[httpx2.Request]) -> httpx2.MockTransport: + def handler(request: httpx2.Request) -> httpx2.Response: + _ = request.content + captured_requests.append(request) + return httpx2.Response(200, json={'ok': True}) + + return httpx2.MockTransport(handler) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py b/datadog_checks_base/tests/base/utils/httpx2/test_auth_basic.py similarity index 94% rename from datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py rename to datadog_checks_base/tests/base/utils/httpx2/test_auth_basic.py index 23081a10a34b1..5873f0d66f9ab 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_auth_basic.py @@ -3,7 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from datadog_checks.base.utils.http_httpx import HTTPXWrapper +from datadog_checks.base.utils.httpx2 import HTTPXWrapper from .common import parse_basic_auth diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py similarity index 99% rename from datadog_checks_base/tests/base/utils/http_httpx/test_config.py rename to datadog_checks_base/tests/base/utils/httpx2/test_config.py index 7f29caea7f2f3..73a44a3c5c248 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -3,7 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from datadog_checks.base.utils.http_httpx import HTTPXWrapper +from datadog_checks.base.utils.httpx2 import HTTPXWrapper def test_default_headers_include_user_agent(capturing_transport): diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py similarity index 51% rename from datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py rename to datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py index 7ac2ca585ba57..59379714c5098 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py @@ -1,7 +1,7 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import httpx2 as httpx +import httpx2 import pytest from datadog_checks.base.utils.http_exceptions import ( @@ -10,19 +10,19 @@ HTTPRequestError, HTTPTimeoutError, ) -from datadog_checks.base.utils.http_httpx import HTTPXWrapper, _map_httpx_exception +from datadog_checks.base.utils.httpx2 import HTTPXWrapper, _map_httpx_exception @pytest.mark.parametrize( 'raised,expected', [ - pytest.param(httpx.ConnectTimeout('boom'), HTTPTimeoutError, id='connect-timeout'), - pytest.param(httpx.ReadTimeout('slow'), HTTPTimeoutError, id='read-timeout'), - pytest.param(httpx.PoolTimeout('pool'), HTTPTimeoutError, id='pool-timeout'), - pytest.param(httpx.ConnectError('refused'), HTTPConnectionError, id='connect-error'), - pytest.param(httpx.ReadError('mid-stream'), HTTPRequestError, id='read-error'), - pytest.param(httpx.LocalProtocolError('bad'), HTTPRequestError, id='local-protocol-error'), - pytest.param(httpx.RequestError('generic'), HTTPRequestError, id='request-error'), + pytest.param(httpx2.ConnectTimeout('boom'), HTTPTimeoutError, id='connect-timeout'), + pytest.param(httpx2.ReadTimeout('slow'), HTTPTimeoutError, id='read-timeout'), + pytest.param(httpx2.PoolTimeout('pool'), HTTPTimeoutError, id='pool-timeout'), + pytest.param(httpx2.ConnectError('refused'), HTTPConnectionError, id='connect-error'), + pytest.param(httpx2.ReadError('mid-stream'), HTTPRequestError, id='read-error'), + pytest.param(httpx2.LocalProtocolError('bad'), HTTPRequestError, id='local-protocol-error'), + pytest.param(httpx2.RequestError('generic'), HTTPRequestError, id='request-error'), ], ) def test_request_exception_mapping(raising_transport_factory, raised, expected): @@ -33,12 +33,12 @@ def test_request_exception_mapping(raising_transport_factory, raised, expected): def test_map_httpx_exception_routes_invalid_url(): - mapped = _map_httpx_exception(httpx.InvalidURL('bad url')) + mapped = _map_httpx_exception(httpx2.InvalidURL('bad url')) assert isinstance(mapped, HTTPInvalidURLError) def test_request_raises_invalid_url_error(raising_transport_factory): - transport = raising_transport_factory(httpx.InvalidURL('bad url')) + transport = raising_transport_factory(httpx2.InvalidURL('bad url')) http = HTTPXWrapper({}, {}, transport=transport) with pytest.raises(HTTPInvalidURLError): http.get('http://example.test/') diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py similarity index 80% rename from datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py rename to datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py index 0ac02187225db..339de63320433 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py @@ -5,7 +5,7 @@ import pytest -from datadog_checks.base.utils.http_httpx import HTTPXWrapper +from datadog_checks.base.utils.httpx2 import HTTPXWrapper def test_close_is_idempotent(capturing_transport): @@ -22,13 +22,13 @@ def test_context_manager(capturing_transport): def test_module_import_fails_without_httpx2(monkeypatch): monkeypatch.setitem(sys.modules, 'httpx2', None) - monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.http_httpx', raising=False) + monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.httpx2', raising=False) with pytest.raises(ImportError, match='httpx2'): - import datadog_checks.base.utils.http_httpx # noqa: F401 + import datadog_checks.base.utils.httpx2 # noqa: F401 # After monkeypatch teardown the module must import cleanly again so sibling tests are # not affected by a leaked broken import state. monkeypatch.undo() - import datadog_checks.base.utils.http_httpx as restored + import datadog_checks.base.utils.httpx2 as restored assert hasattr(restored, 'HTTPXWrapper') @@ -36,8 +36,8 @@ def test_module_import_fails_without_httpx2(monkeypatch): @pytest.mark.parametrize( 'instance,expected_cls_name', [ - pytest.param({'use_httpx': True}, 'HTTPXWrapper', id='opt-in'), - pytest.param({'use_httpx': False}, 'RequestsWrapper', id='explicit-default'), + pytest.param({'use_httpx2': True}, 'HTTPXWrapper', id='opt-in'), + pytest.param({'use_httpx2': False}, 'RequestsWrapper', id='explicit-default'), pytest.param({}, 'RequestsWrapper', id='unset-default'), ], ) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py b/datadog_checks_base/tests/base/utils/httpx2/test_methods.py similarity index 95% rename from datadog_checks_base/tests/base/utils/http_httpx/test_methods.py rename to datadog_checks_base/tests/base/utils/httpx2/test_methods.py index 55da483fc74ff..ba4883c2863ff 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_methods.py @@ -5,7 +5,7 @@ import pytest -from datadog_checks.base.utils.http_httpx import HTTPXWrapper +from datadog_checks.base.utils.httpx2 import HTTPXWrapper HTTP_VERBS = { 'get': 'GET', diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/httpx2/test_response.py similarity index 77% rename from datadog_checks_base/tests/base/utils/http_httpx/test_response.py rename to datadog_checks_base/tests/base/utils/httpx2/test_response.py index 614347260a053..ee55500697bfe 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_response.py @@ -4,11 +4,11 @@ import logging from datetime import timedelta -import httpx2 as httpx +import httpx2 import pytest from datadog_checks.base.utils.http_exceptions import HTTPStatusError -from datadog_checks.base.utils.http_httpx import DEFAULT_CHUNK_SIZE, HTTPXResponseAdapter, HTTPXWrapper +from datadog_checks.base.utils.httpx2 import DEFAULT_CHUNK_SIZE, HTTPXResponseAdapter, HTTPXWrapper @pytest.mark.parametrize('status_code', [404, 500]) @@ -76,10 +76,10 @@ def test_response_iter_content_default_chunk_size_uses_default(status_transport_ ], ) def test_response_iter_content_decode_uses_response_encoding(charset, raw, expected): - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(200, content=raw, headers={'Content-Type': f'text/plain; charset={charset}'}) + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, content=raw, headers={'Content-Type': f'text/plain; charset={charset}'}) - http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') chunks = list(response.iter_content(chunk_size=64, decode_unicode=True)) assert ''.join(chunks) == expected @@ -95,10 +95,10 @@ def handler(_request: httpx.Request) -> httpx.Response: def test_response_iter_lines_decode_uses_response_encoding(charset, line): raw = (line + '\n' + line).encode(charset) - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(200, content=raw, headers={'Content-Type': f'text/plain; charset={charset}'}) + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, content=raw, headers={'Content-Type': f'text/plain; charset={charset}'}) - http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') encoded_lines = list(response.iter_lines(decode_unicode=False)) assert encoded_lines == [line.encode(charset), line.encode(charset)] @@ -113,12 +113,12 @@ def test_response_iter_lines_rejects_delimiter(status_transport_factory): def test_response_elapsed_returns_zero_on_runtime_error(caplog): - # httpx2's .elapsed raises RuntimeError for in-memory responses. The real httpx.Request is + # httpx2's .elapsed raises RuntimeError for in-memory responses. The real httpx2.Request is # attached so the adapter's debug log can format request info without a secondary error. - request = httpx.Request('GET', 'http://example.test/') - response = httpx.Response(200, content=b'x', request=request) + request = httpx2.Request('GET', 'http://example.test/') + response = httpx2.Response(200, content=b'x', request=request) adapter = HTTPXResponseAdapter(response) - with caplog.at_level(logging.DEBUG, logger='datadog_checks.base.utils.http_httpx'): + with caplog.at_level(logging.DEBUG, logger='datadog_checks.base.utils.httpx2'): assert adapter.elapsed == timedelta(0) assert any('elapsed unavailable' in record.message for record in caplog.records) @@ -146,10 +146,10 @@ def test_response_text_decodes_body(status_transport_factory): def test_response_json_returns_decoded_object(): - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json={'a': 1}) + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, json={'a': 1}) - http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') assert response.json() == {'a': 1} @@ -162,10 +162,10 @@ def test_response_content_returns_raw_bytes(status_transport_factory): def test_response_headers_exposed(): - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(200, headers={'X-Custom': 'value'}, content=b'') + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, headers={'X-Custom': 'value'}, content=b'') - http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') assert response.headers['X-Custom'] == 'value' @@ -178,28 +178,28 @@ def test_response_url_reflects_request_url(status_transport_factory): def test_response_encoding_property_reflects_declared_charset(): - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(200, content=b'data', headers={'Content-Type': 'text/plain; charset=iso-8859-1'}) + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, content=b'data', headers={'Content-Type': 'text/plain; charset=iso-8859-1'}) - http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') assert response.encoding == 'iso-8859-1' def test_response_encoding_property_settable_through_adapter(): - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(200, content=b'data', headers={'Content-Type': 'text/plain; charset=utf-8'}) + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, content=b'data', headers={'Content-Type': 'text/plain; charset=utf-8'}) - http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') response.encoding = 'iso-8859-1' assert response.encoding == 'iso-8859-1' def test_response_cookies_exposed(): - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(200, headers={'Set-Cookie': 'session=abc123'}, content=b'') + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, headers={'Set-Cookie': 'session=abc123'}, content=b'') - http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') assert response.cookies['session'] == 'abc123' diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_stream_connection_lines.py b/datadog_checks_base/tests/base/utils/httpx2/test_stream_connection_lines.py similarity index 100% rename from datadog_checks_base/tests/base/utils/http_httpx/test_stream_connection_lines.py rename to datadog_checks_base/tests/base/utils/httpx2/test_stream_connection_lines.py From cf7853d4ac3bad309521867626978f878c4dd385 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 16:14:43 -0400 Subject: [PATCH 06/45] Rename HTTPXWrapper symbols to HTTPX2Wrapper --- datadog_checks_base/changelog.d/22676.added | 2 +- .../datadog_checks/base/checks/base.py | 4 +- .../datadog_checks/base/utils/httpx2.py | 36 ++++++------ .../base/utils/httpx2/test_auth_basic.py | 6 +- .../tests/base/utils/httpx2/test_config.py | 56 +++++++++---------- .../base/utils/httpx2/test_exceptions.py | 10 ++-- .../tests/base/utils/httpx2/test_lifecycle.py | 10 ++-- .../tests/base/utils/httpx2/test_methods.py | 8 +-- .../tests/base/utils/httpx2/test_response.py | 42 +++++++------- 9 files changed, 87 insertions(+), 87 deletions(-) diff --git a/datadog_checks_base/changelog.d/22676.added b/datadog_checks_base/changelog.d/22676.added index 1d81f5b40429e..6a7dc34f16936 100644 --- a/datadog_checks_base/changelog.d/22676.added +++ b/datadog_checks_base/changelog.d/22676.added @@ -1 +1 @@ -Add library-agnostic HTTP mocks/proto/exceptions and migrate intg tests. Add ``HTTPXWrapper``, an httpx2-backed HTTP client opt-in via the ``use_httpx2`` instance config flag. The default remains the existing requests-based client. +Add library-agnostic HTTP mocks/proto/exceptions and migrate intg tests. Add ``HTTPX2Wrapper``, an httpx2-backed HTTP client opt-in via the ``use_httpx2`` instance config flag. The default remains the existing requests-based client. diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 537629fbff9fd..1386851dba4c5 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -414,9 +414,9 @@ def http(self) -> HTTPClientProtocol: if not hasattr(self, '_http'): instance = self.instance or {} if is_affirmative(instance.get('use_httpx2', False)): - from datadog_checks.base.utils.httpx2 import HTTPXWrapper + from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper - self._http = HTTPXWrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) + self._http = HTTPX2Wrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) else: # See Performance Optimizations in this package's README.md. from datadog_checks.base.utils.http import RequestsWrapper diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index c959b851a2279..d66b56d62edc7 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -99,7 +99,7 @@ def _build_timeout(config: dict[str, Any]) -> tuple[float, float]: return connect, read -def _map_httpx_exception(exc: httpx2.HTTPError | httpx2.InvalidURL) -> HTTPError: +def _map_httpx2_exception(exc: httpx2.HTTPError | httpx2.InvalidURL) -> HTTPError: """Translate an httpx2 exception into the library-agnostic equivalent.""" if isinstance(exc, httpx2.InvalidURL): return HTTPInvalidURLError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) @@ -118,7 +118,7 @@ def _map_httpx_exception(exc: httpx2.HTTPError | httpx2.InvalidURL) -> HTTPError return HTTPError(str(exc) or exc.__class__.__name__) -class HTTPXResponseAdapter: +class HTTPX2ResponseAdapter: """Wraps an httpx2.Response to satisfy HTTPResponseProtocol.""" __slots__ = ('_response',) @@ -190,7 +190,7 @@ def raise_for_status(self) -> None: try: self._response.raise_for_status() except httpx2.HTTPStatusError as exc: - raise _map_httpx_exception(exc) from exc + raise _map_httpx2_exception(exc) from exc def close(self) -> None: self._response.close() @@ -211,7 +211,7 @@ def iter_lines( delimiter: bytes | str | None = None, ) -> Iterator[bytes | str]: if delimiter is not None: - raise NotImplementedError("HTTPXResponseAdapter.iter_lines does not support custom delimiters") + raise NotImplementedError("HTTPX2ResponseAdapter.iter_lines does not support custom delimiters") encoding = self._response.encoding or 'utf-8' for line in self._response.iter_lines(): # errors='replace' tolerates bytes that don't round-trip under the declared charset. @@ -225,7 +225,7 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: return None -class HTTPXWrapper: +class HTTPX2Wrapper: """Implements HTTPClientProtocol using a single shared httpx2.Client per wrapper.""" __slots__ = ( @@ -340,28 +340,28 @@ def set_header(self, name: str, value: str) -> None: self.options['headers'][name] = value self._client.headers[name] = value - def get(self, url: str, **options: Any) -> HTTPXResponseAdapter: + def get(self, url: str, **options: Any) -> HTTPX2ResponseAdapter: return self._request('GET', url, options) - def post(self, url: str, **options: Any) -> HTTPXResponseAdapter: + def post(self, url: str, **options: Any) -> HTTPX2ResponseAdapter: return self._request('POST', url, options) - def put(self, url: str, **options: Any) -> HTTPXResponseAdapter: + def put(self, url: str, **options: Any) -> HTTPX2ResponseAdapter: return self._request('PUT', url, options) - def delete(self, url: str, **options: Any) -> HTTPXResponseAdapter: + def delete(self, url: str, **options: Any) -> HTTPX2ResponseAdapter: return self._request('DELETE', url, options) - def head(self, url: str, **options: Any) -> HTTPXResponseAdapter: + def head(self, url: str, **options: Any) -> HTTPX2ResponseAdapter: return self._request('HEAD', url, options) - def patch(self, url: str, **options: Any) -> HTTPXResponseAdapter: + def patch(self, url: str, **options: Any) -> HTTPX2ResponseAdapter: return self._request('PATCH', url, options) - def options_method(self, url: str, **options: Any) -> HTTPXResponseAdapter: + def options_method(self, url: str, **options: Any) -> HTTPX2ResponseAdapter: return self._request('OPTIONS', url, options) - def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXResponseAdapter: + def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPX2ResponseAdapter: # stream=True keeps the connection open until the body is consumed (.content/.text/.iter_*) # or close() is called. Callers should use it as a context manager when not reading the body. if self._log_requests: @@ -373,21 +373,21 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXRespo request = self._client.build_request(method, url, **request_kwargs) response = self._client.send(request, stream=True, follow_redirects=follow_redirects) except (httpx2.HTTPError, httpx2.InvalidURL) as exc: - raise _map_httpx_exception(exc) from exc - return HTTPXResponseAdapter(response) + raise _map_httpx2_exception(exc) from exc + return HTTPX2ResponseAdapter(response) def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', url: str = '') -> dict[str, Any]: """Translate call-site options to httpx2.Client.request kwargs.""" # Drop stream silently because OM v2 scraper passes it unconditionally (base_scraper.py:459). # The wrapper streams internally, so the kwarg is meaningless. if 'stream' in options: - self.logger.debug('HTTPXWrapper dropping unsupported per-request kwarg: stream') + self.logger.debug('HTTPX2Wrapper dropping unsupported per-request kwarg: stream') options = {k: v for k, v in options.items() if k != 'stream'} unknown = set(options) - REQUEST_KWARGS if unknown: context = f' for {method} {url}' if method or url else '' - raise TypeError(f"HTTPXWrapper does not support per-request kwargs{context}: {sorted(unknown)}") + raise TypeError(f"HTTPX2Wrapper does not support per-request kwargs{context}: {sorted(unknown)}") kwargs: dict[str, Any] = {} passthrough = ('params', 'json', 'data', 'content', 'files', 'cookies') for key in passthrough: @@ -431,4 +431,4 @@ def __del__(self) -> None: # no cov pass except Exception as exc: # Don't propagate from __del__ (interpreter handles it). Log at debug so the swallow is observable. - self.logger.debug('HTTPXWrapper.close raised during __del__: %r', exc) + self.logger.debug('HTTPX2Wrapper.close raised during __del__: %r', exc) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_auth_basic.py b/datadog_checks_base/tests/base/utils/httpx2/test_auth_basic.py index 5873f0d66f9ab..034ce388ffb52 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_auth_basic.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_auth_basic.py @@ -3,13 +3,13 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from datadog_checks.base.utils.httpx2 import HTTPXWrapper +from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper from .common import parse_basic_auth def test_basic_auth_sent_on_request(capturing_transport, captured_requests): - http = HTTPXWrapper( + http = HTTPX2Wrapper( {'username': 'alice', 'password': 'secret'}, {}, transport=capturing_transport, @@ -29,6 +29,6 @@ def test_basic_auth_sent_on_request(capturing_transport, captured_requests): ], ) def test_no_authorization_header_set(instance, capturing_transport, captured_requests): - http = HTTPXWrapper(instance, {}, transport=capturing_transport) + http = HTTPX2Wrapper(instance, {}, transport=capturing_transport) http.get('http://example.test/') assert 'authorization' not in captured_requests[0].headers diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index 73a44a3c5c248..9a88666c6f1e3 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -3,68 +3,68 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from datadog_checks.base.utils.httpx2 import HTTPXWrapper +from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper def test_default_headers_include_user_agent(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) assert any(key.lower() == 'user-agent' for key in http.options['headers']) def test_extra_headers_merge(capturing_transport, captured_requests): - http = HTTPXWrapper({'extra_headers': {'X-Extra': 'value'}}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'extra_headers': {'X-Extra': 'value'}}, {}, transport=capturing_transport) http.get('http://example.test/') assert captured_requests[0].headers['x-extra'] == 'value' def test_headers_override_defaults(capturing_transport, captured_requests): - http = HTTPXWrapper({'headers': {'User-Agent': 'custom-agent/1.0'}}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'headers': {'User-Agent': 'custom-agent/1.0'}}, {}, transport=capturing_transport) http.get('http://example.test/') assert captured_requests[0].headers['user-agent'] == 'custom-agent/1.0' def test_per_request_headers_merge_into_request(capturing_transport, captured_requests): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) http.get('http://example.test/', headers={'X-Per-Request': 'yes'}) assert captured_requests[0].headers['x-per-request'] == 'yes' def test_timeout_from_instance(capturing_transport): - http = HTTPXWrapper({'timeout': 25}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'timeout': 25}, {}, transport=capturing_transport) connect, read = http.options['timeout'] assert connect == 25.0 assert read == 25.0 def test_connect_and_read_timeout_split(capturing_transport): - http = HTTPXWrapper({'connect_timeout': 5, 'read_timeout': 30}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'connect_timeout': 5, 'read_timeout': 30}, {}, transport=capturing_transport) connect, read = http.options['timeout'] assert connect == 5.0 assert read == 30.0 def test_verify_defaults_to_true(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) assert http.options['verify'] is True def test_verify_false_when_tls_verify_off(capturing_transport): - http = HTTPXWrapper({'tls_verify': False}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'tls_verify': False}, {}, transport=capturing_transport) assert http.options['verify'] is False def test_tls_ca_cert_uses_path(capturing_transport): - http = HTTPXWrapper({'tls_ca_cert': '/etc/ssl/ca.pem'}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'tls_ca_cert': '/etc/ssl/ca.pem'}, {}, transport=capturing_transport) assert http.options['verify'] == '/etc/ssl/ca.pem' def test_tls_client_cert_string(capturing_transport): - http = HTTPXWrapper({'tls_cert': '/etc/ssl/client.pem'}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'tls_cert': '/etc/ssl/client.pem'}, {}, transport=capturing_transport) assert http.options['cert'] == '/etc/ssl/client.pem' def test_tls_client_cert_with_key(capturing_transport): - http = HTTPXWrapper( + http = HTTPX2Wrapper( {'tls_cert': '/etc/ssl/client.pem', 'tls_private_key': '/etc/ssl/client.key'}, {}, transport=capturing_transport, @@ -73,12 +73,12 @@ def test_tls_client_cert_with_key(capturing_transport): def test_tls_no_cert_when_not_configured(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) assert http.options['cert'] is None def test_options_proxies_is_none(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) assert http.options['proxies'] is None @@ -92,25 +92,25 @@ def test_options_proxies_is_none(capturing_transport): ], ) def test_get_header(capturing_transport, lookup_name, default, expected): - http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) assert http.get_header(lookup_name, default=default) == expected def test_set_header_overrides_existing(capturing_transport): - http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) http.set_header('X-FOO', 'new') assert http.get_header('x-foo') == 'new' def test_set_header_propagates_to_outgoing_request(capturing_transport, captured_requests): - http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) http.set_header('X-FOO', 'updated') http.get('http://example.test/') assert captured_requests[0].headers['x-foo'] == 'updated' def test_set_header_adds_new_header(capturing_transport, captured_requests): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) http.set_header('X-New', 'value') assert http.get_header('x-new') == 'value' http.get('http://example.test/') @@ -119,21 +119,21 @@ def test_set_header_adds_new_header(capturing_transport, captured_requests): def test_remapper_renames_field(capturing_transport): remapper = {'ssl_validation': {'name': 'tls_verify'}} - http = HTTPXWrapper({'ssl_validation': False}, {}, remapper=remapper, transport=capturing_transport) + http = HTTPX2Wrapper({'ssl_validation': False}, {}, remapper=remapper, transport=capturing_transport) assert http.options['verify'] is False def test_remapper_invert_flips_value(capturing_transport): # disable_ssl_validation=True means "do NOT verify" -> options['verify'] is False. remapper = {'disable_ssl_validation': {'name': 'tls_verify', 'invert': True}} - http = HTTPXWrapper({'disable_ssl_validation': True}, {}, remapper=remapper, transport=capturing_transport) + http = HTTPX2Wrapper({'disable_ssl_validation': True}, {}, remapper=remapper, transport=capturing_transport) assert http.options['verify'] is False def test_remapper_invert_with_explicit_default(capturing_transport): # Remapped field absent from instance, explicit default takes effect, then invert flips it. remapper = {'disable_ssl_validation': {'name': 'tls_verify', 'invert': True, 'default': False}} - http = HTTPXWrapper({}, {}, remapper=remapper, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, remapper=remapper, transport=capturing_transport) # default=False, invert flips to True -> verify is True. assert http.options['verify'] is True @@ -141,7 +141,7 @@ def test_remapper_invert_with_explicit_default(capturing_transport): def test_remapper_instance_wins_over_remapped_field(capturing_transport): # If the standard field is present in instance, the remapped alternative is ignored. remapper = {'ssl_validation': {'name': 'tls_verify'}} - http = HTTPXWrapper( + http = HTTPX2Wrapper( {'ssl_validation': False, 'tls_verify': True}, {}, remapper=remapper, transport=capturing_transport ) assert http.options['verify'] is True @@ -150,7 +150,7 @@ def test_remapper_instance_wins_over_remapped_field(capturing_transport): def test_remapper_ignores_unknown_target_field(capturing_transport): # Remapper targeting a non-STANDARD_FIELDS key is silently ignored. remapper = {'some_alias': {'name': 'definitely_not_a_known_field'}} - http = HTTPXWrapper({'some_alias': 'ignored'}, {}, remapper=remapper, transport=capturing_transport) + http = HTTPX2Wrapper({'some_alias': 'ignored'}, {}, remapper=remapper, transport=capturing_transport) assert 'definitely_not_a_known_field' not in http.options @@ -162,30 +162,30 @@ def test_remapper_ignores_unknown_target_field(capturing_transport): ], ) def test_request_rejects_unknown_kwarg(capturing_transport, kwarg, value): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) with pytest.raises(TypeError, match=kwarg): http.get('http://example.test/', **{kwarg: value}) def test_init_config_timeout_used_when_instance_has_none(capturing_transport): - http = HTTPXWrapper({}, {'timeout': 42}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {'timeout': 42}, transport=capturing_transport) connect, read = http.options['timeout'] assert connect == 42.0 assert read == 42.0 def test_instance_timeout_overrides_init_config_timeout(capturing_transport): - http = HTTPXWrapper({'timeout': 7}, {'timeout': 42}, transport=capturing_transport) + http = HTTPX2Wrapper({'timeout': 7}, {'timeout': 42}, transport=capturing_transport) connect, read = http.options['timeout'] assert connect == 7.0 assert read == 7.0 def test_init_config_log_requests_used_when_instance_has_none(capturing_transport): - http = HTTPXWrapper({}, {'log_requests': True}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {'log_requests': True}, transport=capturing_transport) assert http._log_requests is True def test_instance_log_requests_overrides_init_config(capturing_transport): - http = HTTPXWrapper({'log_requests': False}, {'log_requests': True}, transport=capturing_transport) + http = HTTPX2Wrapper({'log_requests': False}, {'log_requests': True}, transport=capturing_transport) assert http._log_requests is False diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py index 59379714c5098..d1f84b76ec559 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py @@ -10,7 +10,7 @@ HTTPRequestError, HTTPTimeoutError, ) -from datadog_checks.base.utils.httpx2 import HTTPXWrapper, _map_httpx_exception +from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper, _map_httpx2_exception @pytest.mark.parametrize( @@ -27,18 +27,18 @@ ) def test_request_exception_mapping(raising_transport_factory, raised, expected): transport = raising_transport_factory(raised) - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) with pytest.raises(expected): http.get('http://example.test/') -def test_map_httpx_exception_routes_invalid_url(): - mapped = _map_httpx_exception(httpx2.InvalidURL('bad url')) +def test_map_httpx2_exception_routes_invalid_url(): + mapped = _map_httpx2_exception(httpx2.InvalidURL('bad url')) assert isinstance(mapped, HTTPInvalidURLError) def test_request_raises_invalid_url_error(raising_transport_factory): transport = raising_transport_factory(httpx2.InvalidURL('bad url')) - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) with pytest.raises(HTTPInvalidURLError): http.get('http://example.test/') diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py index 339de63320433..77970199e09ea 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py @@ -5,17 +5,17 @@ import pytest -from datadog_checks.base.utils.httpx2 import HTTPXWrapper +from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper def test_close_is_idempotent(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) http.close() http.close() def test_context_manager(capturing_transport): - with HTTPXWrapper({}, {}, transport=capturing_transport) as http: + with HTTPX2Wrapper({}, {}, transport=capturing_transport) as http: response = http.get('http://example.test/') assert response.status_code == 200 @@ -30,13 +30,13 @@ def test_module_import_fails_without_httpx2(monkeypatch): monkeypatch.undo() import datadog_checks.base.utils.httpx2 as restored - assert hasattr(restored, 'HTTPXWrapper') + assert hasattr(restored, 'HTTPX2Wrapper') @pytest.mark.parametrize( 'instance,expected_cls_name', [ - pytest.param({'use_httpx2': True}, 'HTTPXWrapper', id='opt-in'), + pytest.param({'use_httpx2': True}, 'HTTPX2Wrapper', id='opt-in'), pytest.param({'use_httpx2': False}, 'RequestsWrapper', id='explicit-default'), pytest.param({}, 'RequestsWrapper', id='unset-default'), ], diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_methods.py b/datadog_checks_base/tests/base/utils/httpx2/test_methods.py index ba4883c2863ff..18c094e17f95b 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_methods.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_methods.py @@ -5,7 +5,7 @@ import pytest -from datadog_checks.base.utils.httpx2 import HTTPXWrapper +from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper HTTP_VERBS = { 'get': 'GET', @@ -20,7 +20,7 @@ @pytest.mark.parametrize('method,verb', HTTP_VERBS.items()) def test_method_dispatches_with_correct_verb(method, verb, captured_requests, capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) fn = getattr(http, method) response = fn('http://example.test/path', headers={'X-Test': '1'}) @@ -29,7 +29,7 @@ def test_method_dispatches_with_correct_verb(method, verb, captured_requests, ca def test_post_json_body_is_serialized(capturing_transport, captured_requests): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) http.post('http://example.test/path', json={'a': 1, 'b': 'two'}) req = captured_requests[0] assert req.headers['content-type'] == 'application/json' @@ -37,6 +37,6 @@ def test_post_json_body_is_serialized(capturing_transport, captured_requests): def test_request_accepts_stream_kwarg(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) response = http.get('http://example.test/', stream=True) assert response.status_code == 200 diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_response.py b/datadog_checks_base/tests/base/utils/httpx2/test_response.py index ee55500697bfe..37b8e532f4ea9 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_response.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_response.py @@ -8,13 +8,13 @@ import pytest from datadog_checks.base.utils.http_exceptions import HTTPStatusError -from datadog_checks.base.utils.httpx2 import DEFAULT_CHUNK_SIZE, HTTPXResponseAdapter, HTTPXWrapper +from datadog_checks.base.utils.httpx2 import DEFAULT_CHUNK_SIZE, HTTPX2ResponseAdapter, HTTPX2Wrapper @pytest.mark.parametrize('status_code', [404, 500]) def test_response_raise_for_status_raises_on_error_codes(status_transport_factory, status_code): transport = status_transport_factory(status_code, b'') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') with pytest.raises(HTTPStatusError): response.raise_for_status() @@ -22,7 +22,7 @@ def test_response_raise_for_status_raises_on_error_codes(status_transport_factor def test_response_iter_content_bytes(status_transport_factory): transport = status_transport_factory(200, b'abcdef') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') chunks = list(response.iter_content(chunk_size=2)) assert b''.join(chunks) == b'abcdef' @@ -30,7 +30,7 @@ def test_response_iter_content_bytes(status_transport_factory): def test_response_iter_content_decode_unicode(status_transport_factory): transport = status_transport_factory(200, b'abcdef') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') chunks = list(response.iter_content(chunk_size=3, decode_unicode=True)) assert ''.join(chunks) == 'abcdef' @@ -45,14 +45,14 @@ def test_response_iter_content_decode_unicode(status_transport_factory): ) def test_response_iter_lines(status_transport_factory, decode_unicode, expected): transport = status_transport_factory(200, b'a\nb\nc') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') assert list(response.iter_lines(decode_unicode=decode_unicode)) == expected def test_response_iter_content_empty_body_yields_nothing(status_transport_factory): transport = status_transport_factory(200, b'') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') assert list(response.iter_content()) == [] @@ -60,7 +60,7 @@ def test_response_iter_content_empty_body_yields_nothing(status_transport_factor def test_response_iter_content_default_chunk_size_uses_default(status_transport_factory): body = b'X' * (DEFAULT_CHUNK_SIZE * 3 + 5) transport = status_transport_factory(200, body) - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') chunks = list(response.iter_content()) assert b''.join(chunks) == body @@ -79,7 +79,7 @@ def test_response_iter_content_decode_uses_response_encoding(charset, raw, expec def handler(_request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, content=raw, headers={'Content-Type': f'text/plain; charset={charset}'}) - http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) + http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') chunks = list(response.iter_content(chunk_size=64, decode_unicode=True)) assert ''.join(chunks) == expected @@ -98,7 +98,7 @@ def test_response_iter_lines_decode_uses_response_encoding(charset, line): def handler(_request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, content=raw, headers={'Content-Type': f'text/plain; charset={charset}'}) - http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) + http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') encoded_lines = list(response.iter_lines(decode_unicode=False)) assert encoded_lines == [line.encode(charset), line.encode(charset)] @@ -106,7 +106,7 @@ def handler(_request: httpx2.Request) -> httpx2.Response: def test_response_iter_lines_rejects_delimiter(status_transport_factory): transport = status_transport_factory(200, b'a\nb\n') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') with pytest.raises(NotImplementedError): list(response.iter_lines(delimiter=b'|')) @@ -117,7 +117,7 @@ def test_response_elapsed_returns_zero_on_runtime_error(caplog): # attached so the adapter's debug log can format request info without a secondary error. request = httpx2.Request('GET', 'http://example.test/') response = httpx2.Response(200, content=b'x', request=request) - adapter = HTTPXResponseAdapter(response) + adapter = HTTPX2ResponseAdapter(response) with caplog.at_level(logging.DEBUG, logger='datadog_checks.base.utils.httpx2'): assert adapter.elapsed == timedelta(0) assert any('elapsed unavailable' in record.message for record in caplog.records) @@ -126,21 +126,21 @@ def test_response_elapsed_returns_zero_on_runtime_error(caplog): @pytest.mark.parametrize('status_code,expected_ok', [(200, True), (204, True), (301, True), (400, False), (500, False)]) def test_response_ok_property(status_transport_factory, status_code, expected_ok): transport = status_transport_factory(status_code, b'') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') assert response.ok is expected_ok def test_response_reason_from_httpx_response(status_transport_factory): transport = status_transport_factory(200, b'') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') assert response.reason == 'OK' def test_response_text_decodes_body(status_transport_factory): transport = status_transport_factory(200, b'hello') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') assert response.text == 'hello' @@ -149,14 +149,14 @@ def test_response_json_returns_decoded_object(): def handler(_request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, json={'a': 1}) - http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) + http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') assert response.json() == {'a': 1} def test_response_content_returns_raw_bytes(status_transport_factory): transport = status_transport_factory(200, b'\x00\x01\x02') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') assert response.content == b'\x00\x01\x02' @@ -165,14 +165,14 @@ def test_response_headers_exposed(): def handler(_request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, headers={'X-Custom': 'value'}, content=b'') - http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) + http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') assert response.headers['X-Custom'] == 'value' def test_response_url_reflects_request_url(status_transport_factory): transport = status_transport_factory(200, b'') - http = HTTPXWrapper({}, {}, transport=transport) + http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/path') assert str(response.url) == 'http://example.test/path' @@ -181,7 +181,7 @@ def test_response_encoding_property_reflects_declared_charset(): def handler(_request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, content=b'data', headers={'Content-Type': 'text/plain; charset=iso-8859-1'}) - http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) + http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') assert response.encoding == 'iso-8859-1' @@ -190,7 +190,7 @@ def test_response_encoding_property_settable_through_adapter(): def handler(_request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, content=b'data', headers={'Content-Type': 'text/plain; charset=utf-8'}) - http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) + http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') response.encoding = 'iso-8859-1' assert response.encoding == 'iso-8859-1' @@ -200,6 +200,6 @@ def test_response_cookies_exposed(): def handler(_request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, headers={'Set-Cookie': 'session=abc123'}, content=b'') - http = HTTPXWrapper({}, {}, transport=httpx2.MockTransport(handler)) + http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') assert response.cookies['session'] == 'abc123' From 25fc4567271bfaf29738e8d6a7f5f8ba72cf181a Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 17:01:57 -0400 Subject: [PATCH 07/45] Round 2 finding #3: move test_stream_connection_lines to OM scraper test dir --- .../openmetrics/test_v2/scraper}/test_stream_connection_lines.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename datadog_checks_base/tests/base/{utils/httpx2 => checks/openmetrics/test_v2/scraper}/test_stream_connection_lines.py (100%) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_stream_connection_lines.py b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/scraper/test_stream_connection_lines.py similarity index 100% rename from datadog_checks_base/tests/base/utils/httpx2/test_stream_connection_lines.py rename to datadog_checks_base/tests/base/checks/openmetrics/test_v2/scraper/test_stream_connection_lines.py From 88c5cb8ea396ffcb62d8a09c7c5113f9499f03bb Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 17:03:22 -0400 Subject: [PATCH 08/45] Round 2 finding #7: drop unreachable __del__ AttributeError --- datadog_checks_base/datadog_checks/base/utils/httpx2.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index d66b56d62edc7..15a06d585ccf6 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -426,9 +426,6 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: def __del__(self) -> None: # no cov try: self.close() - except AttributeError: - # __del__ runs even if __init__ failed before self._client was set. - pass except Exception as exc: # Don't propagate from __del__ (interpreter handles it). Log at debug so the swallow is observable. self.logger.debug('HTTPX2Wrapper.close raised during __del__: %r', exc) From 19afdeca9783d5d539626ed29c76c5642dad66d4 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 17:02:48 -0400 Subject: [PATCH 09/45] Round 2 finding #4: drop dead DEFAULT_REMAPPED_FIELDS constant --- datadog_checks_base/datadog_checks/base/utils/httpx2.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 15a06d585ccf6..5e83ab445e2bd 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -45,8 +45,6 @@ 'username': None, } -DEFAULT_REMAPPED_FIELDS: dict[str, dict[str, Any]] = {} - REQUEST_KWARGS = frozenset( { 'params', @@ -289,7 +287,6 @@ def _resolve_config( config = {field: instance.get(field, value) for field, value in default_fields.items()} remapper = dict(remapper) if remapper else {} - remapper.update(DEFAULT_REMAPPED_FIELDS) for remapped_field, data in remapper.items(): field = data.get('name') From 14eda496cf9f69fcc886a4ac56e7912618dd3e47 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 17:04:51 -0400 Subject: [PATCH 10/45] Round 2 finding #9: simplify lifecycle import-failure test --- .../tests/base/utils/httpx2/test_lifecycle.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py index 77970199e09ea..1247bf925f6bd 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py @@ -25,12 +25,6 @@ def test_module_import_fails_without_httpx2(monkeypatch): monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.httpx2', raising=False) with pytest.raises(ImportError, match='httpx2'): import datadog_checks.base.utils.httpx2 # noqa: F401 - # After monkeypatch teardown the module must import cleanly again so sibling tests are - # not affected by a leaked broken import state. - monkeypatch.undo() - import datadog_checks.base.utils.httpx2 as restored - - assert hasattr(restored, 'HTTPX2Wrapper') @pytest.mark.parametrize( From e1fefec9235cd550879fb041cf6c0402c2b2ef9c Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 17:04:09 -0400 Subject: [PATCH 11/45] Round 2 finding #8: assert stream-kwarg drop log via caplog --- .../tests/base/utils/httpx2/test_methods.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_methods.py b/datadog_checks_base/tests/base/utils/httpx2/test_methods.py index 18c094e17f95b..667253b3a3780 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_methods.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_methods.py @@ -2,6 +2,7 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) import json +import logging import pytest @@ -36,7 +37,9 @@ def test_post_json_body_is_serialized(capturing_transport, captured_requests): assert json.loads(req.content) == {'a': 1, 'b': 'two'} -def test_request_accepts_stream_kwarg(capturing_transport): +def test_stream_kwarg_logged_and_dropped(capturing_transport, caplog): http = HTTPX2Wrapper({}, {}, transport=capturing_transport) - response = http.get('http://example.test/', stream=True) + with caplog.at_level(logging.DEBUG, logger='datadog_checks.base.utils.httpx2'): + response = http.get('http://example.test/', stream=True) assert response.status_code == 200 + assert any('dropping unsupported per-request kwarg: stream' in r.message for r in caplog.records) From 0a650ea7b09313fb809662aae39bd0531be7a262 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 17:05:55 -0400 Subject: [PATCH 12/45] Round 2 finding #5: extract _build_http_client helper --- .../datadog_checks/base/checks/base.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 1386851dba4c5..8341aed481ac4 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -404,6 +404,16 @@ def _get_metric_limit(self, instance=None): return limit + def _build_http_client(self, instance) -> HTTPClientProtocol: + if is_affirmative(instance.get('use_httpx2', False)): + from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper + + return HTTPX2Wrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) + # See Performance Optimizations in this package's README.md. + from datadog_checks.base.utils.http import RequestsWrapper + + return RequestsWrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) + @property def http(self) -> HTTPClientProtocol: """ @@ -412,16 +422,7 @@ def http(self) -> HTTPClientProtocol: Only new checks or checks on Agent 6.13+ can and should use this for HTTP requests. """ if not hasattr(self, '_http'): - instance = self.instance or {} - if is_affirmative(instance.get('use_httpx2', False)): - from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper - - self._http = HTTPX2Wrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) - else: - # See Performance Optimizations in this package's README.md. - from datadog_checks.base.utils.http import RequestsWrapper - - self._http = RequestsWrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) + self._http = self._build_http_client(self.instance or {}) return self._http From b7da55fae43139a98c476a2be98257ca706593a5 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 17:12:59 -0400 Subject: [PATCH 13/45] Round 2 finding #10: case-fold headers/extra_headers merge --- .../datadog_checks/base/utils/httpx2.py | 7 ++++- .../tests/base/utils/httpx2/test_config.py | 31 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 5e83ab445e2bd..a69578576264d 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -394,7 +394,12 @@ def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', ur headers = options.get('headers') extra_headers = options.get('extra_headers') if headers is not None or extra_headers is not None: - kwargs['headers'] = {**(headers or {}), **(extra_headers or {})} + # httpx2.Headers folds keys case-insensitively so overlapping headers/extra_headers collapse + # to a single entry, matching RequestsWrapper's CaseInsensitiveDict behavior. + merged = httpx2.Headers(headers or {}) + if extra_headers: + merged.update(extra_headers) + kwargs['headers'] = merged if 'timeout' in options: timeout_value = options['timeout'] diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index 9a88666c6f1e3..358a36104ab05 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -29,6 +29,37 @@ def test_per_request_headers_merge_into_request(capturing_transport, captured_re assert captured_requests[0].headers['x-per-request'] == 'yes' +@pytest.mark.parametrize( + 'headers,extra_headers,canonical_key,expected_value', + [ + pytest.param({'X-Foo': 'a'}, {'x-foo': 'b'}, 'x-foo', 'b', id='extra-wins-on-case-different-overlap'), + pytest.param( + {'Content-Type': 'application/json'}, + {'content-type': 'text/plain'}, + 'content-type', + 'text/plain', + id='extra-wins-same-case', + ), + pytest.param({'X-Foo': 'a'}, None, 'x-foo', 'a', id='headers-only'), + pytest.param(None, {'X-Foo': 'b'}, 'x-foo', 'b', id='extra-only'), + ], +) +def test_per_request_headers_extra_headers_case_fold( + capturing_transport, captured_requests, headers, extra_headers, canonical_key, expected_value +): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + kwargs = {} + if headers is not None: + kwargs['headers'] = headers + if extra_headers is not None: + kwargs['extra_headers'] = extra_headers + http.get('http://example.test/', **kwargs) + + sent = captured_requests[0].headers + # Single case-folded entry on the outgoing request. + assert sent.get_list(canonical_key) == [expected_value] + + def test_timeout_from_instance(capturing_transport): http = HTTPX2Wrapper({'timeout': 25}, {}, transport=capturing_transport) connect, read = http.options['timeout'] From eb550b7e9cdf32bf95b6b4c3cb07d31329c2fe06 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 17:08:19 -0400 Subject: [PATCH 14/45] Round 2 finding #6: close response on adapter wrap failure --- .../datadog_checks/base/utils/httpx2.py | 7 ++++++- .../tests/base/utils/httpx2/test_lifecycle.py | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index a69578576264d..d77117ca949d7 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -371,7 +371,12 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPX2Resp response = self._client.send(request, stream=True, follow_redirects=follow_redirects) except (httpx2.HTTPError, httpx2.InvalidURL) as exc: raise _map_httpx2_exception(exc) from exc - return HTTPX2ResponseAdapter(response) + try: + return HTTPX2ResponseAdapter(response) + except BaseException: + # Close the open stream if anything between send(stream=True) and the wrap raises. + response.close() + raise def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', url: str = '') -> dict[str, Any]: """Translate call-site options to httpx2.Client.request kwargs.""" diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py index 1247bf925f6bd..e9e7ba5106c12 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py @@ -5,6 +5,7 @@ import pytest +from datadog_checks.base.utils import httpx2 as httpx2_module from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper @@ -20,6 +21,26 @@ def test_context_manager(capturing_transport): assert response.status_code == 200 +def test_request_closes_response_when_adapter_wrap_fails(capturing_transport, monkeypatch): + closed = [] + + def failing_init(self, response): + # Install the spy at adapter-init time so the close() calls that httpx2 makes + # while eagerly reading the in-memory MockTransport body during Response.__init__ + # are not counted; only the production guard's close() is observed. + original_close = response.close + response.close = lambda: (closed.append(True), original_close())[-1] + raise RuntimeError('boom') + + monkeypatch.setattr(httpx2_module.HTTPX2ResponseAdapter, '__init__', failing_init) + + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + with pytest.raises(RuntimeError, match='boom'): + http.get('http://example.test/') + + assert closed == [True] + + def test_module_import_fails_without_httpx2(monkeypatch): monkeypatch.setitem(sys.modules, 'httpx2', None) monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.httpx2', raising=False) From b8eb171e17da73e664832019cb2e5c21a0230479 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 4 Jun 2026 17:03:22 -0400 Subject: [PATCH 15/45] Round 2 finding #1: map httpx2 exceptions in iter_lines/iter_content --- .../datadog_checks/base/utils/httpx2.py | 24 ++++++++++------- .../tests/base/utils/httpx2/conftest.py | 15 +++++++++++ .../base/utils/httpx2/test_exceptions.py | 26 +++++++++++++++++++ 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index d77117ca949d7..a113ce6943c7c 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -195,12 +195,15 @@ def close(self) -> None: def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: effective_size = chunk_size if chunk_size is not None else DEFAULT_CHUNK_SIZE - if decode_unicode: - # iter_text uses an incremental decoder. Manual chunk decoding breaks on - # multibyte chars split across boundaries. - yield from self._response.iter_text(chunk_size=effective_size) - return - yield from self._response.iter_bytes(chunk_size=effective_size) + try: + if decode_unicode: + # iter_text uses an incremental decoder. Manual chunk decoding breaks on + # multibyte chars split across boundaries. + yield from self._response.iter_text(chunk_size=effective_size) + return + yield from self._response.iter_bytes(chunk_size=effective_size) + except (httpx2.HTTPError, httpx2.InvalidURL) as exc: + raise _map_httpx2_exception(exc) from exc def iter_lines( self, @@ -211,9 +214,12 @@ def iter_lines( if delimiter is not None: raise NotImplementedError("HTTPX2ResponseAdapter.iter_lines does not support custom delimiters") encoding = self._response.encoding or 'utf-8' - for line in self._response.iter_lines(): - # errors='replace' tolerates bytes that don't round-trip under the declared charset. - yield line if decode_unicode else line.encode(encoding, errors='replace') + try: + for line in self._response.iter_lines(): + # errors='replace' tolerates bytes that don't round-trip under the declared charset. + yield line if decode_unicode else line.encode(encoding, errors='replace') + except (httpx2.HTTPError, httpx2.InvalidURL) as exc: + raise _map_httpx2_exception(exc) from exc def __enter__(self) -> Self: return self diff --git a/datadog_checks_base/tests/base/utils/httpx2/conftest.py b/datadog_checks_base/tests/base/utils/httpx2/conftest.py index f08eb6f795b0d..cf99a2817779b 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/conftest.py +++ b/datadog_checks_base/tests/base/utils/httpx2/conftest.py @@ -31,6 +31,21 @@ def handler(_request: httpx2.Request) -> httpx2.Response: return _factory +@pytest.fixture +def mid_stream_raising_transport_factory() -> Callable[[Exception], httpx2.MockTransport]: + def _factory(exc: Exception): + def body(): + yield b'first\n' + raise exc + + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, content=body()) + + return httpx2.MockTransport(handler) + + return _factory + + @pytest.fixture def captured_requests() -> list[httpx2.Request]: return [] diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py index d1f84b76ec559..5273148ac4f6b 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py @@ -42,3 +42,29 @@ def test_request_raises_invalid_url_error(raising_transport_factory): http = HTTPX2Wrapper({}, {}, transport=transport) with pytest.raises(HTTPInvalidURLError): http.get('http://example.test/') + + +@pytest.mark.parametrize( + 'raised,expected', + [ + pytest.param(httpx2.ReadError('mid-stream'), HTTPRequestError, id='read-error'), + pytest.param(httpx2.ReadTimeout('slow'), HTTPTimeoutError, id='read-timeout'), + ], +) +@pytest.mark.parametrize( + 'iter_method,iter_kwargs', + [ + pytest.param('iter_lines', {}, id='iter_lines'), + pytest.param('iter_lines', {'decode_unicode': True}, id='iter_lines-decoded'), + pytest.param('iter_content', {}, id='iter_content-bytes'), + pytest.param('iter_content', {'decode_unicode': True}, id='iter_content-decoded'), + ], +) +def test_iter_methods_map_mid_stream_exceptions( + mid_stream_raising_transport_factory, raised, expected, iter_method, iter_kwargs +): + transport = mid_stream_raising_transport_factory(raised) + http = HTTPX2Wrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + with pytest.raises(expected): + list(getattr(response, iter_method)(**iter_kwargs)) From 3e2c413fbb3851e067c873992ec3bf4a9161ee31 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 12:48:44 -0400 Subject: [PATCH 16/45] Round 3 fix #4: tighten ImportError assertion to exc.value.name --- datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py index e9e7ba5106c12..7b9a3145d1a01 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py @@ -44,8 +44,9 @@ def failing_init(self, response): def test_module_import_fails_without_httpx2(monkeypatch): monkeypatch.setitem(sys.modules, 'httpx2', None) monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.httpx2', raising=False) - with pytest.raises(ImportError, match='httpx2'): + with pytest.raises(ImportError) as exc_info: import datadog_checks.base.utils.httpx2 # noqa: F401 + assert exc_info.value.name == 'httpx2' @pytest.mark.parametrize( From dbadf00529cf7d90fc2d5822eee5f3549730e7df Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 12:50:39 -0400 Subject: [PATCH 17/45] Round 3 fix #2: iter_lines bytes-through fidelity --- .../datadog_checks/base/utils/httpx2.py | 15 +++++++++++---- .../tests/base/utils/httpx2/test_response.py | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index a113ce6943c7c..b37c88c9b10ff 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -213,11 +213,18 @@ def iter_lines( ) -> Iterator[bytes | str]: if delimiter is not None: raise NotImplementedError("HTTPX2ResponseAdapter.iter_lines does not support custom delimiters") - encoding = self._response.encoding or 'utf-8' try: - for line in self._response.iter_lines(): - # errors='replace' tolerates bytes that don't round-trip under the declared charset. - yield line if decode_unicode else line.encode(encoding, errors='replace') + if decode_unicode: + yield from self._response.iter_lines() + return + buffer = b'' + for chunk in self._response.iter_bytes(): + buffer += chunk + while b'\n' in buffer: + line, _, buffer = buffer.partition(b'\n') + yield line.rstrip(b'\r') + if buffer: + yield buffer.rstrip(b'\r') except (httpx2.HTTPError, httpx2.InvalidURL) as exc: raise _map_httpx2_exception(exc) from exc diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_response.py b/datadog_checks_base/tests/base/utils/httpx2/test_response.py index 37b8e532f4ea9..3f2a57b63e122 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_response.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_response.py @@ -104,6 +104,24 @@ def handler(_request: httpx2.Request) -> httpx2.Response: assert encoded_lines == [line.encode(charset), line.encode(charset)] +def test_response_iter_lines_bytes_through_invalid_sequences(): + raw = b'invalid\xff\xfe\nsecond\n' + + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, content=raw, headers={'Content-Type': 'text/plain; charset=utf-8'}) + + http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) + response = http.get('http://example.test/') + assert list(response.iter_lines(decode_unicode=False)) == [b'invalid\xff\xfe', b'second'] + + +def test_response_iter_lines_bytes_through_crlf(status_transport_factory): + transport = status_transport_factory(200, b'a\r\nb\r\nc') + http = HTTPX2Wrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert list(response.iter_lines(decode_unicode=False)) == [b'a', b'b', b'c'] + + def test_response_iter_lines_rejects_delimiter(status_transport_factory): transport = status_transport_factory(200, b'a\nb\n') http = HTTPX2Wrapper({}, {}, transport=transport) From 94655535a7e7d22aed2f08a8ebd56892a31c460e Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 12:51:26 -0400 Subject: [PATCH 18/45] Round 3 fix #3: derive passthrough from REQUEST_KWARGS --- .../datadog_checks/base/utils/httpx2.py | 5 +++-- .../tests/base/utils/httpx2/test_config.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index b37c88c9b10ff..31e387a350473 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -59,6 +59,8 @@ 'follow_redirects', } ) +REQUEST_KWARGS_SPECIAL = frozenset({'headers', 'extra_headers', 'timeout', 'follow_redirects'}) +REQUEST_KWARGS_PASSTHROUGH = REQUEST_KWARGS - REQUEST_KWARGS_SPECIAL def _make_timeout(connect: float, read: float) -> httpx2.Timeout: @@ -404,8 +406,7 @@ def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', ur context = f' for {method} {url}' if method or url else '' raise TypeError(f"HTTPX2Wrapper does not support per-request kwargs{context}: {sorted(unknown)}") kwargs: dict[str, Any] = {} - passthrough = ('params', 'json', 'data', 'content', 'files', 'cookies') - for key in passthrough: + for key in REQUEST_KWARGS_PASSTHROUGH: if key in options: kwargs[key] = options[key] diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index 358a36104ab05..db7b506b21353 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -3,7 +3,17 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper +from datadog_checks.base.utils.httpx2 import ( + REQUEST_KWARGS, + REQUEST_KWARGS_PASSTHROUGH, + REQUEST_KWARGS_SPECIAL, + HTTPX2Wrapper, +) + + +def test_request_kwargs_invariant(): + assert REQUEST_KWARGS_PASSTHROUGH | REQUEST_KWARGS_SPECIAL == REQUEST_KWARGS + assert REQUEST_KWARGS_PASSTHROUGH & REQUEST_KWARGS_SPECIAL == frozenset() def test_default_headers_include_user_agent(capturing_transport): From 2894e7eacb7221a1e65b72f79ccb6d26c88d6e16 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 12:50:39 -0400 Subject: [PATCH 19/45] Round 3 fix #1: timeout=None guard + scalar _make_timeout consistency --- .../datadog_checks/base/utils/httpx2.py | 7 +++-- .../tests/base/utils/httpx2/test_config.py | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 31e387a350473..024e4e010d7cc 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -422,10 +422,13 @@ def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', ur if 'timeout' in options: timeout_value = options['timeout'] - if isinstance(timeout_value, (tuple, list)) and len(timeout_value) == 2: + if timeout_value is None: + kwargs['timeout'] = None + elif isinstance(timeout_value, (tuple, list)) and len(timeout_value) == 2: kwargs['timeout'] = _make_timeout(float(timeout_value[0]), float(timeout_value[1])) else: - kwargs['timeout'] = float(timeout_value) # type: ignore[arg-type] + value = float(timeout_value) + kwargs['timeout'] = _make_timeout(value, value) if 'follow_redirects' in options: kwargs['follow_redirects'] = bool(options['follow_redirects']) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index db7b506b21353..e995b2e2def70 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -84,6 +84,32 @@ def test_connect_and_read_timeout_split(capturing_transport): assert read == 30.0 +@pytest.mark.parametrize( + 'timeout_value,expected', + [ + pytest.param( + None, + {'connect': None, 'read': None, 'write': None, 'pool': None}, + id='none-means-no-timeout', + ), + pytest.param( + 7, + {'connect': 7.0, 'read': 7.0, 'write': 7.0, 'pool': 7.0}, + id='scalar-pins-all-phases', + ), + pytest.param( + (3, 5), + {'connect': 3.0, 'read': 5.0, 'write': 5.0, 'pool': 5.0}, + id='tuple-connect-read-pins-write-pool', + ), + ], +) +def test_per_request_timeout_value_construction(capturing_transport, captured_requests, timeout_value, expected): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.get('http://example.test/', timeout=timeout_value) + assert captured_requests[0].extensions['timeout'] == expected + + def test_verify_defaults_to_true(capturing_transport): http = HTTPX2Wrapper({}, {}, transport=capturing_transport) assert http.options['verify'] is True From d6429ff3c2a723447e8c33b0ea19094c91d623ff Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 15:14:25 -0400 Subject: [PATCH 20/45] Round 4 fix #1: map mid-stream httpx2 errors to HTTPConnectionError --- datadog_checks_base/datadog_checks/base/utils/httpx2.py | 2 ++ .../tests/base/utils/httpx2/test_exceptions.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 024e4e010d7cc..c9f9c03d16324 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -107,6 +107,8 @@ def _map_httpx2_exception(exc: httpx2.HTTPError | httpx2.InvalidURL) -> HTTPErro return HTTPTimeoutError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) if isinstance(exc, httpx2.ConnectError): return HTTPConnectionError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) + if isinstance(exc, (httpx2.ReadError, httpx2.WriteError, httpx2.CloseError)): + return HTTPConnectionError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) if isinstance(exc, httpx2.HTTPStatusError): return HTTPStatusError( str(exc) or exc.__class__.__name__, diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py index 5273148ac4f6b..756fc7b31cea5 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py @@ -20,7 +20,9 @@ pytest.param(httpx2.ReadTimeout('slow'), HTTPTimeoutError, id='read-timeout'), pytest.param(httpx2.PoolTimeout('pool'), HTTPTimeoutError, id='pool-timeout'), pytest.param(httpx2.ConnectError('refused'), HTTPConnectionError, id='connect-error'), - pytest.param(httpx2.ReadError('mid-stream'), HTTPRequestError, id='read-error'), + pytest.param(httpx2.ReadError('mid-stream'), HTTPConnectionError, id='read-error'), + pytest.param(httpx2.WriteError('broken-pipe'), HTTPConnectionError, id='write-error'), + pytest.param(httpx2.CloseError('half-closed'), HTTPConnectionError, id='close-error'), pytest.param(httpx2.LocalProtocolError('bad'), HTTPRequestError, id='local-protocol-error'), pytest.param(httpx2.RequestError('generic'), HTTPRequestError, id='request-error'), ], @@ -47,7 +49,7 @@ def test_request_raises_invalid_url_error(raising_transport_factory): @pytest.mark.parametrize( 'raised,expected', [ - pytest.param(httpx2.ReadError('mid-stream'), HTTPRequestError, id='read-error'), + pytest.param(httpx2.ReadError('mid-stream'), HTTPConnectionError, id='read-error'), pytest.param(httpx2.ReadTimeout('slow'), HTTPTimeoutError, id='read-timeout'), ], ) From d6c83a06e2fbc4f1bc4914a622c7657da10893e0 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 15:54:55 -0400 Subject: [PATCH 21/45] Round 4 fix #4: document init header dict-shape parity with RequestsWrapper --- .../datadog_checks/base/utils/httpx2.py | 3 +++ .../tests/base/utils/httpx2/test_config.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index c9f9c03d16324..3778107b2721e 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -263,6 +263,9 @@ def __init__( config = self._resolve_config(instance, init_config, remapper) + # options['headers'] is a plain dict for parity with RequestsWrapper (http.py:319). + # Case-folding is applied at request time via httpx2.Headers in _build_request_kwargs; + # get_header/set_header below do case-insensitive iteration over this dict. headers = get_default_headers() if config['headers']: headers.clear() diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index e995b2e2def70..913ac79e19676 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -70,6 +70,18 @@ def test_per_request_headers_extra_headers_case_fold( assert sent.get_list(canonical_key) == [expected_value] +def test_init_header_get_set_are_case_insensitive_with_case_different_overlap(capturing_transport): + http = HTTPX2Wrapper( + {'headers': {'X-Foo': 'a'}, 'extra_headers': {'x-foo': 'b'}}, + {}, + transport=capturing_transport, + ) + assert http.get_header('X-FOO') == http.get_header('x-foo') + http.set_header('X-Foo', 'c') + assert http.get_header('X-Foo') == 'c' + assert http.get_header('x-foo') == 'c' + + def test_timeout_from_instance(capturing_transport): http = HTTPX2Wrapper({'timeout': 25}, {}, transport=capturing_transport) connect, read = http.options['timeout'] From 0961d59daea90884d4f4cd13cdd0100575b0892a Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 14:23:12 -0400 Subject: [PATCH 22/45] Round 4 fix #3: add encoding/url/cookies/elapsed to HTTPResponseProtocol --- .../datadog_checks/base/utils/http_protocol.py | 12 +++++++++++- .../tests/base/utils/httpx2/test_response.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_protocol.py b/datadog_checks_base/datadog_checks/base/utils/http_protocol.py index 4d103de5930c3..824f0dc2c16a7 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_protocol.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_protocol.py @@ -4,9 +4,11 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Any, Iterator, Protocol +from datetime import timedelta +from typing import Any, Iterator, Protocol, runtime_checkable +@runtime_checkable class HTTPResponseProtocol(Protocol): status_code: int content: bytes @@ -17,6 +19,14 @@ class HTTPResponseProtocol(Protocol): def ok(self) -> bool: ... @property def reason(self) -> str: ... + @property + def encoding(self) -> str | None: ... + @property + def url(self) -> str: ... + @property + def cookies(self) -> Any: ... + @property + def elapsed(self) -> timedelta: ... def json(self, **kwargs: Any) -> Any: ... def raise_for_status(self) -> None: ... diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_response.py b/datadog_checks_base/tests/base/utils/httpx2/test_response.py index 3f2a57b63e122..81ee695a090f7 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_response.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_response.py @@ -8,6 +8,7 @@ import pytest from datadog_checks.base.utils.http_exceptions import HTTPStatusError +from datadog_checks.base.utils.http_protocol import HTTPResponseProtocol from datadog_checks.base.utils.httpx2 import DEFAULT_CHUNK_SIZE, HTTPX2ResponseAdapter, HTTPX2Wrapper @@ -221,3 +222,15 @@ def handler(_request: httpx2.Request) -> httpx2.Response: http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) response = http.get('http://example.test/') assert response.cookies['session'] == 'abc123' + + +def test_response_adapter_satisfies_protocol(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPX2Wrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert isinstance(response, HTTPResponseProtocol) + + +@pytest.mark.parametrize('attr', ['encoding', 'url', 'cookies', 'elapsed']) +def test_response_protocol_declares_extended_attrs(attr): + assert attr in HTTPResponseProtocol.__annotations__ or hasattr(HTTPResponseProtocol, attr) From 16d619d8c6657efe19075c57684dc988244fe030 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 14:21:14 -0400 Subject: [PATCH 23/45] Round 4 fix #2: replace bytes accumulator with bytearray in iter_lines --- .../datadog_checks/base/utils/httpx2.py | 14 +++++++------- .../tests/base/utils/httpx2/test_response.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 3778107b2721e..7858f410bfcb5 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -221,14 +221,14 @@ def iter_lines( if decode_unicode: yield from self._response.iter_lines() return - buffer = b'' + buf = bytearray() for chunk in self._response.iter_bytes(): - buffer += chunk - while b'\n' in buffer: - line, _, buffer = buffer.partition(b'\n') - yield line.rstrip(b'\r') - if buffer: - yield buffer.rstrip(b'\r') + buf.extend(chunk) + while (idx := buf.find(b'\n')) != -1: + yield bytes(buf[:idx]).rstrip(b'\r') + del buf[: idx + 1] + if buf: + yield bytes(buf).rstrip(b'\r') except (httpx2.HTTPError, httpx2.InvalidURL) as exc: raise _map_httpx2_exception(exc) from exc diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_response.py b/datadog_checks_base/tests/base/utils/httpx2/test_response.py index 81ee695a090f7..3b45b37d21950 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_response.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_response.py @@ -131,6 +131,24 @@ def test_response_iter_lines_rejects_delimiter(status_transport_factory): list(response.iter_lines(delimiter=b'|')) +def test_response_iter_lines_bytes_across_many_chunks(): + chunk_size = 1024 + num_chunks = 6 + body = b''.join((b'X' * (chunk_size - 1) + b'\n') for _ in range(num_chunks)) + b'tail' + + def streamed(): + for i in range(0, len(body), chunk_size): + yield body[i : i + chunk_size] + + def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(200, content=streamed()) + + http = HTTPX2Wrapper({}, {}, transport=httpx2.MockTransport(handler)) + response = http.get('http://example.test/') + lines = list(response.iter_lines(decode_unicode=False)) + assert lines == [b'X' * (chunk_size - 1)] * num_chunks + [b'tail'] + + def test_response_elapsed_returns_zero_on_runtime_error(caplog): # httpx2's .elapsed raises RuntimeError for in-memory responses. The real httpx2.Request is # attached so the adapter's debug log can format request info without a secondary error. From e6b797be8a17043140cb865ad3bd7bc881720da1 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 17:23:11 -0400 Subject: [PATCH 24/45] Round 5 finding #5: tighten HTTPResponseProtocol.cookies to Mapping[str, str] --- .../datadog_checks/base/utils/http_protocol.py | 2 +- .../tests/base/utils/httpx2/test_response.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_protocol.py b/datadog_checks_base/datadog_checks/base/utils/http_protocol.py index 824f0dc2c16a7..3e7efbf4a2a2c 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_protocol.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_protocol.py @@ -24,7 +24,7 @@ def encoding(self) -> str | None: ... @property def url(self) -> str: ... @property - def cookies(self) -> Any: ... + def cookies(self) -> Mapping[str, str]: ... @property def elapsed(self) -> timedelta: ... diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_response.py b/datadog_checks_base/tests/base/utils/httpx2/test_response.py index 3b45b37d21950..90dce421154e8 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_response.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_response.py @@ -2,6 +2,7 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) import logging +from collections.abc import Mapping from datetime import timedelta import httpx2 @@ -242,6 +243,13 @@ def handler(_request: httpx2.Request) -> httpx2.Response: assert response.cookies['session'] == 'abc123' +def test_response_cookies_is_mapping(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPX2Wrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert isinstance(response.cookies, Mapping) + + def test_response_adapter_satisfies_protocol(status_transport_factory): transport = status_transport_factory(200, b'') http = HTTPX2Wrapper({}, {}, transport=transport) From 8897f209f20aa86a32ab65adcf9db2facbf3294f Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 17:27:58 -0400 Subject: [PATCH 25/45] Round 5 finding #1: collapse _map_httpx2_exception branches via httpx2.NetworkError --- datadog_checks_base/datadog_checks/base/utils/httpx2.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 7858f410bfcb5..e9b7ca7b791d2 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -105,9 +105,7 @@ def _map_httpx2_exception(exc: httpx2.HTTPError | httpx2.InvalidURL) -> HTTPErro return HTTPInvalidURLError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) if isinstance(exc, httpx2.TimeoutException): return HTTPTimeoutError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) - if isinstance(exc, httpx2.ConnectError): - return HTTPConnectionError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) - if isinstance(exc, (httpx2.ReadError, httpx2.WriteError, httpx2.CloseError)): + if isinstance(exc, httpx2.NetworkError): return HTTPConnectionError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) if isinstance(exc, httpx2.HTTPStatusError): return HTTPStatusError( From 4a77e981774119828bbac91b88ff44c320817f53 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 17:28:16 -0400 Subject: [PATCH 26/45] Round 5 finding #2: lock in iter_lines decode CRLF parity test --- .../tests/base/utils/httpx2/test_response.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_response.py b/datadog_checks_base/tests/base/utils/httpx2/test_response.py index 90dce421154e8..2bb082418f957 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_response.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_response.py @@ -124,6 +124,13 @@ def test_response_iter_lines_bytes_through_crlf(status_transport_factory): assert list(response.iter_lines(decode_unicode=False)) == [b'a', b'b', b'c'] +def test_response_iter_lines_decoded_through_crlf(status_transport_factory): + transport = status_transport_factory(200, b'a\r\nb\r\nc') + http = HTTPX2Wrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert list(response.iter_lines(decode_unicode=True)) == ['a', 'b', 'c'] + + def test_response_iter_lines_rejects_delimiter(status_transport_factory): transport = status_transport_factory(200, b'a\nb\n') http = HTTPX2Wrapper({}, {}, transport=transport) From 3926cf19696acd8d9d72555caa63e5450d26dc6e Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 17:28:54 -0400 Subject: [PATCH 27/45] Round 5 finding #4: lock in headers={} benign no-op behavior --- datadog_checks_base/datadog_checks/base/utils/httpx2.py | 1 + datadog_checks_base/tests/base/utils/httpx2/test_config.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index e9b7ca7b791d2..d5ee221093035 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -416,6 +416,7 @@ def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', ur headers = options.get('headers') extra_headers = options.get('extra_headers') if headers is not None or extra_headers is not None: + # Empty headers/extra_headers fall through as a benign no-op: client-level defaults survive build_request. # httpx2.Headers folds keys case-insensitively so overlapping headers/extra_headers collapse # to a single entry, matching RequestsWrapper's CaseInsensitiveDict behavior. merged = httpx2.Headers(headers or {}) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index 913ac79e19676..2e4b039ff19f0 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -39,6 +39,12 @@ def test_per_request_headers_merge_into_request(capturing_transport, captured_re assert captured_requests[0].headers['x-per-request'] == 'yes' +def test_per_request_empty_headers_dict_preserves_client_defaults(capturing_transport, captured_requests): + http = HTTPX2Wrapper({'extra_headers': {'X-Default': 'kept'}}, {}, transport=capturing_transport) + http.get('http://example.test/', headers={}) + assert captured_requests[0].headers['x-default'] == 'kept' + + @pytest.mark.parametrize( 'headers,extra_headers,canonical_key,expected_value', [ From a6e076306070f7768b4c258d45f2782cc0cafbf3 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 17:25:34 -0400 Subject: [PATCH 28/45] Round 5 finding #3: document options['headers'] mirror constraint --- datadog_checks_base/datadog_checks/base/utils/httpx2.py | 3 +++ .../tests/base/utils/httpx2/test_config.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index d5ee221093035..099ed278191d6 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -279,6 +279,9 @@ def __init__( # proxies=None mirrors RequestsWrapper.options for consumers (e.g. http_check). # TODO: wire proxies through to httpx2 as part of the ongoing httpx migration. + # Note: options['headers'] is a mirror of _client.headers maintained by set_header. + # Direct mutation (e.g. self.options['headers']['X-Foo'] = 'bar') will NOT reach the wire. + # Use set_header/get_header for header mutations. self.options: dict[str, Any] = { 'auth': auth, 'cert': cert, diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index 2e4b039ff19f0..0482867ccdeac 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -202,6 +202,15 @@ def test_set_header_adds_new_header(capturing_transport, captured_requests): assert captured_requests[0].headers['x-new'] == 'value' +def test_direct_options_headers_mutation_does_not_propagate(capturing_transport, captured_requests): + # Characterization test for the documented constraint: options['headers'] is a mirror, + # not the source of truth. Direct mutation does NOT reach the wire. Use set_header instead. + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.options['headers']['X-Direct'] = 'leaked' + http.get('http://example.test/') + assert 'x-direct' not in captured_requests[0].headers + + def test_remapper_renames_field(capturing_transport): remapper = {'ssl_validation': {'name': 'tls_verify'}} http = HTTPX2Wrapper({'ssl_validation': False}, {}, remapper=remapper, transport=capturing_transport) From d14b18011385a83f8eb0959f67d89fc839e4d96e Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 17:26:10 -0400 Subject: [PATCH 29/45] Round 5 finding #6: use getattr guard in test_lifecycle teardown --- .../tests/base/utils/httpx2/test_lifecycle.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py index 7b9a3145d1a01..617e1225565ad 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_lifecycle.py @@ -65,8 +65,27 @@ def test_agentcheck_http_dispatch(instance, expected_cls_name): assert type(check.http).__name__ == expected_cls_name finally: # Read the cached attribute directly (AgentCheck.http is a property that may rebuild). - http = check._http + http = getattr(check, '_http', None) if http is not None: close = getattr(http, 'close', None) if close is not None: close() + + +def test_teardown_guard_preserves_build_error(): + from datadog_checks.base import AgentCheck + + class BoomCheck(AgentCheck): + def _build_http_client(self, instance): + raise RuntimeError('build failed') + + check = BoomCheck('test', {}, [{}]) + with pytest.raises(RuntimeError, match='build failed'): + try: + _ = check.http + finally: + http = getattr(check, '_http', None) + if http is not None: + close = getattr(http, 'close', None) + if close is not None: + close() From 56d9f7b27102ae794e2e89913257c9bfb756f1c0 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 17:25:14 -0400 Subject: [PATCH 30/45] Round 5 finding #7: rename misleading iter_lines bytes-path test --- datadog_checks_base/tests/base/utils/httpx2/test_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_response.py b/datadog_checks_base/tests/base/utils/httpx2/test_response.py index 2bb082418f957..d36c1b4878fd0 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_response.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_response.py @@ -94,7 +94,7 @@ def handler(_request: httpx2.Request) -> httpx2.Response: pytest.param('iso-8859-1', 'café', id='iso-8859-1'), ], ) -def test_response_iter_lines_decode_uses_response_encoding(charset, line): +def test_response_iter_lines_bytes_path_is_encoding_agnostic(charset, line): raw = (line + '\n' + line).encode(charset) def handler(_request: httpx2.Request) -> httpx2.Response: From cf4980f288dd58bd8f23274a775b900a58da6150 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 17:26:52 -0400 Subject: [PATCH 31/45] Round 5 finding #8: drop tautological protocol attrs test --- datadog_checks_base/tests/base/utils/httpx2/test_response.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_response.py b/datadog_checks_base/tests/base/utils/httpx2/test_response.py index d36c1b4878fd0..b99972484d6e9 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_response.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_response.py @@ -262,8 +262,3 @@ def test_response_adapter_satisfies_protocol(status_transport_factory): http = HTTPX2Wrapper({}, {}, transport=transport) response = http.get('http://example.test/') assert isinstance(response, HTTPResponseProtocol) - - -@pytest.mark.parametrize('attr', ['encoding', 'url', 'cookies', 'elapsed']) -def test_response_protocol_declares_extended_attrs(attr): - assert attr in HTTPResponseProtocol.__annotations__ or hasattr(HTTPResponseProtocol, attr) From e1b5f286e277744db7815137e550d5c1a5531537 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 18:40:15 -0400 Subject: [PATCH 32/45] Round 6 finding #1: collapse follow_redirects routing --- datadog_checks_base/datadog_checks/base/utils/httpx2.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 099ed278191d6..d1c5184ab083f 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -385,8 +385,10 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPX2Resp if self._log_requests: self.logger.debug('Sending %s request to %s', method, url) + follow_redirects = ( + bool(options['follow_redirects']) if 'follow_redirects' in options else httpx2.USE_CLIENT_DEFAULT + ) request_kwargs = self._build_request_kwargs(options, method=method, url=url) - follow_redirects = request_kwargs.pop('follow_redirects', httpx2.USE_CLIENT_DEFAULT) try: request = self._client.build_request(method, url, **request_kwargs) response = self._client.send(request, stream=True, follow_redirects=follow_redirects) @@ -437,9 +439,6 @@ def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', ur value = float(timeout_value) kwargs['timeout'] = _make_timeout(value, value) - if 'follow_redirects' in options: - kwargs['follow_redirects'] = bool(options['follow_redirects']) - return kwargs def close(self) -> None: From e0cb6ad4b1235c85ba6d2eca32650a51db91952a Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 18:41:55 -0400 Subject: [PATCH 33/45] Round 6 finding #2: assert request kwargs invariant at import --- .../datadog_checks/base/utils/httpx2.py | 2 ++ .../tests/base/utils/httpx2/test_config.py | 12 +----------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index d1c5184ab083f..ac824af7fb1c1 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -61,6 +61,8 @@ ) REQUEST_KWARGS_SPECIAL = frozenset({'headers', 'extra_headers', 'timeout', 'follow_redirects'}) REQUEST_KWARGS_PASSTHROUGH = REQUEST_KWARGS - REQUEST_KWARGS_SPECIAL +assert REQUEST_KWARGS_PASSTHROUGH | REQUEST_KWARGS_SPECIAL == REQUEST_KWARGS +assert REQUEST_KWARGS_PASSTHROUGH & REQUEST_KWARGS_SPECIAL == frozenset() def _make_timeout(connect: float, read: float) -> httpx2.Timeout: diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index 0482867ccdeac..a84024163e6bc 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -3,17 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from datadog_checks.base.utils.httpx2 import ( - REQUEST_KWARGS, - REQUEST_KWARGS_PASSTHROUGH, - REQUEST_KWARGS_SPECIAL, - HTTPX2Wrapper, -) - - -def test_request_kwargs_invariant(): - assert REQUEST_KWARGS_PASSTHROUGH | REQUEST_KWARGS_SPECIAL == REQUEST_KWARGS - assert REQUEST_KWARGS_PASSTHROUGH & REQUEST_KWARGS_SPECIAL == frozenset() +from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper def test_default_headers_include_user_agent(capturing_transport): From e42dd4f3a2781a841968b4bc28e36ab60fab4999 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 19:27:16 -0400 Subject: [PATCH 34/45] Round 7 finding #1: http-response-protocol-encoding-setter --- datadog_checks_base/datadog_checks/base/utils/http_protocol.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_protocol.py b/datadog_checks_base/datadog_checks/base/utils/http_protocol.py index 3e7efbf4a2a2c..2d31c24d1b5fb 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_protocol.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_protocol.py @@ -21,6 +21,8 @@ def ok(self) -> bool: ... def reason(self) -> str: ... @property def encoding(self) -> str | None: ... + @encoding.setter + def encoding(self, value: str | None) -> None: ... @property def url(self) -> str: ... @property From 6520a52efd0887148009bd3ae2de79aa5e3a36b8 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 5 Jun 2026 19:29:32 -0400 Subject: [PATCH 35/45] Round 7 finding #2: follow-redirects-positive-test-coverage --- .../tests/base/utils/httpx2/test_methods.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_methods.py b/datadog_checks_base/tests/base/utils/httpx2/test_methods.py index 667253b3a3780..4d88506b516e1 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_methods.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_methods.py @@ -4,6 +4,7 @@ import json import logging +import httpx2 import pytest from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper @@ -43,3 +44,29 @@ def test_stream_kwarg_logged_and_dropped(capturing_transport, caplog): response = http.get('http://example.test/', stream=True) assert response.status_code == 200 assert any('dropping unsupported per-request kwarg: stream' in r.message for r in caplog.records) + + +def _spy_on_send(http): + captured = {} + original_send = http._client.send + + def spy(request, **kwargs): + captured.update(kwargs) + return original_send(request, **kwargs) + + http._client.send = spy + return captured + + +def test_request_passes_follow_redirects_per_request(capturing_transport): + http = HTTPX2Wrapper({'allow_redirects': False}, {}, transport=capturing_transport) + captured = _spy_on_send(http) + http.get('http://example.test/', follow_redirects=True) + assert captured['follow_redirects'] is True + + +def test_request_uses_client_default_follow_redirects_when_omitted(capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + captured = _spy_on_send(http) + http.get('http://example.test/') + assert captured['follow_redirects'] is httpx2.USE_CLIENT_DEFAULT From 3af601531f507637db335252282a82461ec0d648 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Sun, 7 Jun 2026 12:42:44 -0400 Subject: [PATCH 36/45] Round 8 finding #1: wrap-content-text-json-exceptions --- .../datadog_checks/base/utils/httpx2.py | 19 +++++++++++---- .../base/utils/httpx2/test_exceptions.py | 23 +++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index ac824af7fb1c1..0f6b2a8336353 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -134,12 +134,18 @@ def status_code(self) -> int: @property def content(self) -> bytes: - return self._response.read() + try: + return self._response.read() + except (httpx2.HTTPError, httpx2.InvalidURL) as exc: + raise _map_httpx2_exception(exc) from exc @property def text(self) -> str: - self._response.read() - return self._response.text + try: + self._response.read() + return self._response.text + except (httpx2.HTTPError, httpx2.InvalidURL) as exc: + raise _map_httpx2_exception(exc) from exc @property def headers(self) -> Mapping[str, str]: @@ -182,8 +188,11 @@ def elapsed(self) -> timedelta: return timedelta(0) def json(self, **kwargs: Any) -> Any: - self._response.read() - return self._response.json(**kwargs) + try: + self._response.read() + return self._response.json(**kwargs) + except (httpx2.HTTPError, httpx2.InvalidURL) as exc: + raise _map_httpx2_exception(exc) from exc def raise_for_status(self) -> None: # Mirror requests semantics (4xx/5xx only); httpx2 also raises on 3xx. diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py index 756fc7b31cea5..893c8a25d8cad 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_exceptions.py @@ -70,3 +70,26 @@ def test_iter_methods_map_mid_stream_exceptions( response = http.get('http://example.test/') with pytest.raises(expected): list(getattr(response, iter_method)(**iter_kwargs)) + + +@pytest.mark.parametrize( + 'raised,expected', + [ + pytest.param(httpx2.ReadError('mid-stream'), HTTPConnectionError, id='read-error'), + pytest.param(httpx2.ReadTimeout('slow'), HTTPTimeoutError, id='read-timeout'), + ], +) +@pytest.mark.parametrize( + 'reader', + [ + pytest.param(lambda r: r.content, id='content'), + pytest.param(lambda r: r.text, id='text'), + pytest.param(lambda r: r.json(), id='json'), + ], +) +def test_buffered_reads_map_exceptions(mid_stream_raising_transport_factory, raised, expected, reader): + transport = mid_stream_raising_transport_factory(raised) + http = HTTPX2Wrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + with pytest.raises(expected): + reader(response) From 03f8345b40bfdc33a7ac94b3f9f3b04d321bfd22 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Sun, 7 Jun 2026 12:44:51 -0400 Subject: [PATCH 37/45] Round 8 finding #4: parametrize-warning-path-tests --- .../scraper/test_stream_connection_lines.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/datadog_checks_base/tests/base/checks/openmetrics/test_v2/scraper/test_stream_connection_lines.py b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/scraper/test_stream_connection_lines.py index fb27824db248c..4e35c8171ac23 100644 --- a/datadog_checks_base/tests/base/checks/openmetrics/test_v2/scraper/test_stream_connection_lines.py +++ b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/scraper/test_stream_connection_lines.py @@ -20,9 +20,16 @@ def _scraper_with_connection_error(exception, *, ignore_connection_errors): return scraper -def test_connection_error_warning_path(caplog): +@pytest.mark.parametrize( + 'exception', + [ + pytest.param(HTTPConnectionError('refused'), id='http-connection-error'), + pytest.param(RequestsConnectionError('boom'), id='requests-connection-error-backcompat'), + ], +) +def test_connection_error_warning_path(caplog, exception): scraper = _scraper_with_connection_error( - HTTPConnectionError('refused'), + exception, ignore_connection_errors=True, ) with caplog.at_level(logging.WARNING, logger='test_stream_connection_lines'): @@ -31,7 +38,7 @@ def test_connection_error_warning_path(caplog): 'OpenMetrics endpoint http://example.test/metrics is not accessible' in record.message for record in caplog.records ) - assert any('refused' in record.message for record in caplog.records) + assert any(str(exception) in record.message for record in caplog.records) def test_connection_error_reraises_when_not_ignored(): @@ -41,13 +48,3 @@ def test_connection_error_reraises_when_not_ignored(): ) with pytest.raises(HTTPConnectionError): list(scraper.stream_connection_lines()) - - -def test_requests_connection_error_still_handled(caplog): - scraper = _scraper_with_connection_error( - RequestsConnectionError('boom'), - ignore_connection_errors=True, - ) - with caplog.at_level(logging.WARNING, logger='test_stream_connection_lines'): - assert list(scraper.stream_connection_lines()) == [] - assert any('boom' in record.message for record in caplog.records) From bf93607353154fde9df68ef3de7016591ff72350 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Mon, 8 Jun 2026 09:30:56 -0400 Subject: [PATCH 38/45] Map self.options['headers'] mutations to httpx2 requests --- .../datadog_checks/base/utils/httpx2.py | 24 ++++--- .../tests/base/utils/httpx2/test_config.py | 9 --- .../httpx2/test_options_headers_mutation.py | 69 +++++++++++++++++++ 3 files changed, 82 insertions(+), 20 deletions(-) create mode 100644 datadog_checks_base/tests/base/utils/httpx2/test_options_headers_mutation.py diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 0f6b2a8336353..affeef9844628 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -290,9 +290,9 @@ def __init__( # proxies=None mirrors RequestsWrapper.options for consumers (e.g. http_check). # TODO: wire proxies through to httpx2 as part of the ongoing httpx migration. - # Note: options['headers'] is a mirror of _client.headers maintained by set_header. - # Direct mutation (e.g. self.options['headers']['X-Foo'] = 'bar') will NOT reach the wire. - # Use set_header/get_header for header mutations. + # options['headers'] is the per-request source of truth and is re-read in _build_request_kwargs, + # so direct mutation (__setitem__, update(), whole-dict replacement) reaches the wire. + # set_header keeps options['headers'] and _client.headers in sync for callers that prefer it. self.options: dict[str, Any] = { 'auth': auth, 'cert': cert, @@ -429,16 +429,18 @@ def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', ur if key in options: kwargs[key] = options[key] + # Re-read self.options['headers'] per request so post-init mutations (__setitem__, update(), + # whole-dict replacement) reach the wire, mirroring RequestsWrapper's ChainMap pattern at http.py:497. + # Layered order: self.options['headers'] < per-call headers < per-call extra_headers. + # httpx2.Headers folds keys case-insensitively, matching RequestsWrapper's CaseInsensitiveDict. + merged = httpx2.Headers(self.options.get('headers') or {}) headers = options.get('headers') + if headers: + merged.update(headers) extra_headers = options.get('extra_headers') - if headers is not None or extra_headers is not None: - # Empty headers/extra_headers fall through as a benign no-op: client-level defaults survive build_request. - # httpx2.Headers folds keys case-insensitively so overlapping headers/extra_headers collapse - # to a single entry, matching RequestsWrapper's CaseInsensitiveDict behavior. - merged = httpx2.Headers(headers or {}) - if extra_headers: - merged.update(extra_headers) - kwargs['headers'] = merged + if extra_headers: + merged.update(extra_headers) + kwargs['headers'] = merged if 'timeout' in options: timeout_value = options['timeout'] diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index a84024163e6bc..52ec1dcc3e7cb 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -192,15 +192,6 @@ def test_set_header_adds_new_header(capturing_transport, captured_requests): assert captured_requests[0].headers['x-new'] == 'value' -def test_direct_options_headers_mutation_does_not_propagate(capturing_transport, captured_requests): - # Characterization test for the documented constraint: options['headers'] is a mirror, - # not the source of truth. Direct mutation does NOT reach the wire. Use set_header instead. - http = HTTPX2Wrapper({}, {}, transport=capturing_transport) - http.options['headers']['X-Direct'] = 'leaked' - http.get('http://example.test/') - assert 'x-direct' not in captured_requests[0].headers - - def test_remapper_renames_field(capturing_transport): remapper = {'ssl_validation': {'name': 'tls_verify'}} http = HTTPX2Wrapper({'ssl_validation': False}, {}, remapper=remapper, transport=capturing_transport) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_options_headers_mutation.py b/datadog_checks_base/tests/base/utils/httpx2/test_options_headers_mutation.py new file mode 100644 index 0000000000000..ada7c1d22771f --- /dev/null +++ b/datadog_checks_base/tests/base/utils/httpx2/test_options_headers_mutation.py @@ -0,0 +1,69 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper + + +def test_options_headers_setitem_reaches_wire(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.options['headers']['X-Token'] = 'after-init' + http.get('http://example.test/') + assert captured_requests[0].headers['x-token'] == 'after-init' + + +def test_options_headers_update_reaches_wire(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.options['headers'].update({'X-Token': 'after-init', 'X-Extra': 'two'}) + http.get('http://example.test/') + assert captured_requests[0].headers['x-token'] == 'after-init' + assert captured_requests[0].headers['x-extra'] == 'two' + + +def test_options_headers_whole_dict_replace_reaches_wire(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.options['headers'] = {'X-Token': 'after-init'} + http.get('http://example.test/') + assert captured_requests[0].headers['x-token'] == 'after-init' + + +def test_options_headers_mid_stream_mutation_observed(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.options['headers']['X-Token'] = 'first' + http.get('http://example.test/') + http.options['headers']['X-Token'] = 'second' + http.get('http://example.test/') + assert captured_requests[0].headers['x-token'] == 'first' + assert captured_requests[1].headers['x-token'] == 'second' + + +def test_set_header_still_works_after_change(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.set_header('X-Token', 'set-via-helper') + http.get('http://example.test/') + assert captured_requests[0].headers['x-token'] == 'set-via-helper' + + +def test_per_call_headers_override_options_headers(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.options['headers']['X-Token'] = 'wrapper-default' + http.get('http://example.test/', headers={'X-Token': 'per-call'}) + assert captured_requests[0].headers['x-token'] == 'per-call' + + +def test_extra_headers_override_per_call_headers_with_options_default(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.options['headers']['X-Token'] = 'wrapper-default' + http.get( + 'http://example.test/', + headers={'X-Token': 'per-call'}, + extra_headers={'X-Token': 'per-call-extra'}, + ) + assert captured_requests[0].headers['x-token'] == 'per-call-extra' + + +def test_options_headers_case_insensitive_collapse_with_per_call(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.options['headers']['X-FOO'] = 'a' + http.get('http://example.test/', headers={'x-foo': 'b'}) + sent = captured_requests[0].headers + assert sent.get_list('x-foo') == ['b'] From 54661165dbfc2daf9c8027fe391c451082808557 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Mon, 8 Jun 2026 09:32:45 -0400 Subject: [PATCH 39/45] Improve HTTPX2Wrapper TypeError with per-kwarg remediation hints --- .../datadog_checks/base/utils/httpx2.py | 12 ++- .../utils/httpx2/test_unknown_kwarg_hints.py | 76 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 0f6b2a8336353..a32de656c8ec2 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -64,6 +64,13 @@ assert REQUEST_KWARGS_PASSTHROUGH | REQUEST_KWARGS_SPECIAL == REQUEST_KWARGS assert REQUEST_KWARGS_PASSTHROUGH & REQUEST_KWARGS_SPECIAL == frozenset() +UNKNOWN_KWARG_HINTS: dict[str, str] = { + 'verify': "configure 'tls_verify' in instance config, or drop the per-call kwarg", + 'persist': 'drop the kwarg, httpx2 pools connections by default', + 'cert': "configure 'tls_cert' and 'tls_private_key' in instance config", + 'proxies': 'proxy support is not yet wired through to httpx2, see TODO in httpx2.py', +} + def _make_timeout(connect: float, read: float) -> httpx2.Timeout: # Pass the read timeout to every phase so write and pool aren't left unbounded. @@ -423,7 +430,10 @@ def _build_request_kwargs(self, options: dict[str, Any], *, method: str = '', ur unknown = set(options) - REQUEST_KWARGS if unknown: context = f' for {method} {url}' if method or url else '' - raise TypeError(f"HTTPX2Wrapper does not support per-request kwargs{context}: {sorted(unknown)}") + sorted_unknown = sorted(unknown) + hints = [f' {k}: {UNKNOWN_KWARG_HINTS[k]}' for k in sorted_unknown if k in UNKNOWN_KWARG_HINTS] + hint_block = ('\n' + '\n'.join(hints)) if hints else '' + raise TypeError(f"HTTPX2Wrapper does not support per-request kwargs{context}: {sorted_unknown}{hint_block}") kwargs: dict[str, Any] = {} for key in REQUEST_KWARGS_PASSTHROUGH: if key in options: diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py b/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py new file mode 100644 index 0000000000000..f223d0a8c206c --- /dev/null +++ b/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py @@ -0,0 +1,76 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper + + +def test_unknown_kwarg_typeerror_includes_kwarg_name(capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError, match='persist'): + http.get('http://example.test/', persist=True) + + +def test_unknown_kwarg_typeerror_includes_remediation_for_verify(capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError) as excinfo: + http.get('http://example.test/', verify=False) + message = str(excinfo.value) + assert 'verify' in message + assert 'tls_verify' in message + + +def test_unknown_kwarg_typeerror_includes_remediation_for_persist(capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError) as excinfo: + http.get('http://example.test/', persist=True) + message = str(excinfo.value) + assert 'persist' in message + assert 'pools connections' in message + + +def test_unknown_kwarg_typeerror_includes_remediation_for_cert(capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError) as excinfo: + http.get('http://example.test/', cert='/etc/ssl/client.pem') + message = str(excinfo.value) + assert 'cert' in message + assert 'tls_cert' in message + assert 'tls_private_key' in message + + +def test_unknown_kwarg_typeerror_includes_remediation_for_proxies(capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError) as excinfo: + http.get('http://example.test/', proxies={'http': 'http://proxy:8080'}) + message = str(excinfo.value) + assert 'proxies' in message + assert 'proxy support' in message + + +def test_unknown_kwarg_without_hint_falls_back_gracefully(capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError) as excinfo: + http.get('http://example.test/', foobar=1) + message = str(excinfo.value) + assert 'foobar' in message + assert 'tls_verify' not in message + assert 'pools connections' not in message + + +def test_unknown_kwarg_multi_lists_each_hint(capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError) as excinfo: + http.get('http://example.test/', verify=False, persist=True) + message = str(excinfo.value) + assert 'verify' in message + assert 'persist' in message + assert 'tls_verify' in message + assert 'pools connections' in message + + +def test_stream_kwarg_still_silently_dropped(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.get('http://example.test/', stream=True) + assert len(captured_requests) == 1 From 3eb1fd020bf9b85f2e51aea9df6f1e8358377a27 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Mon, 8 Jun 2026 10:04:04 -0400 Subject: [PATCH 40/45] Tighten test_stream_kwarg_still_silently_dropped to assert response, not call count --- .../tests/base/utils/httpx2/test_unknown_kwarg_hints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py b/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py index f223d0a8c206c..d6bb6a380aee0 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py @@ -70,7 +70,7 @@ def test_unknown_kwarg_multi_lists_each_hint(capturing_transport): assert 'pools connections' in message -def test_stream_kwarg_still_silently_dropped(capturing_transport, captured_requests): +def test_stream_kwarg_still_silently_dropped(capturing_transport): http = HTTPX2Wrapper({}, {}, transport=capturing_transport) - http.get('http://example.test/', stream=True) - assert len(captured_requests) == 1 + response = http.get('http://example.test/', stream=True) + assert response.status_code == 200 From 78b106c2562103c7a58e66c9980d6329b4b69be4 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Mon, 8 Jun 2026 11:08:26 -0400 Subject: [PATCH 41/45] Add allow_redirects hint to UNKNOWN_KWARG_HINTS --- .../datadog_checks/base/utils/httpx2.py | 1 + .../tests/base/utils/httpx2/test_config.py | 11 ++++++----- .../base/utils/httpx2/test_unknown_kwarg_hints.py | 9 +++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 49e4449d2326e..c9ed2575386f2 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -65,6 +65,7 @@ assert REQUEST_KWARGS_PASSTHROUGH & REQUEST_KWARGS_SPECIAL == frozenset() UNKNOWN_KWARG_HINTS: dict[str, str] = { + 'allow_redirects': "use 'follow_redirects' instead (httpx2 name for this kwarg)", 'verify': "configure 'tls_verify' in instance config, or drop the per-call kwarg", 'persist': 'drop the kwarg, httpx2 pools connections by default', 'cert': "configure 'tls_cert' and 'tls_private_key' in instance config", diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_config.py b/datadog_checks_base/tests/base/utils/httpx2/test_config.py index 52ec1dcc3e7cb..7bb5baa659587 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_config.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_config.py @@ -230,16 +230,17 @@ def test_remapper_ignores_unknown_target_field(capturing_transport): @pytest.mark.parametrize( - 'kwarg,value', + 'kwarg,value,hint_substring', [ - pytest.param('proxies', {'http': 'http://proxy:8080'}, id='proxies'), - pytest.param('allow_redirects', False, id='allow-redirects-uses-httpx-name'), + pytest.param('proxies', {'http': 'http://proxy:8080'}, 'proxy support', id='proxies'), + pytest.param('allow_redirects', False, 'follow_redirects', id='allow-redirects-uses-httpx-name'), ], ) -def test_request_rejects_unknown_kwarg(capturing_transport, kwarg, value): +def test_request_rejects_unknown_kwarg(capturing_transport, kwarg, value, hint_substring): http = HTTPX2Wrapper({}, {}, transport=capturing_transport) - with pytest.raises(TypeError, match=kwarg): + with pytest.raises(TypeError, match=kwarg) as excinfo: http.get('http://example.test/', **{kwarg: value}) + assert hint_substring in str(excinfo.value) def test_init_config_timeout_used_when_instance_has_none(capturing_transport): diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py b/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py index d6bb6a380aee0..22a13882a6d55 100644 --- a/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py +++ b/datadog_checks_base/tests/base/utils/httpx2/test_unknown_kwarg_hints.py @@ -49,6 +49,15 @@ def test_unknown_kwarg_typeerror_includes_remediation_for_proxies(capturing_tran assert 'proxy support' in message +def test_unknown_kwarg_typeerror_includes_remediation_for_allow_redirects(capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError) as excinfo: + http.get('http://example.test/', allow_redirects=False) + message = str(excinfo.value) + assert 'allow_redirects' in message + assert 'follow_redirects' in message + + def test_unknown_kwarg_without_hint_falls_back_gracefully(capturing_transport): http = HTTPX2Wrapper({}, {}, transport=capturing_transport) with pytest.raises(TypeError) as excinfo: From 476e2d572f00de58540d513392ea561da6d0fedb Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Mon, 8 Jun 2026 11:08:27 -0400 Subject: [PATCH 42/45] Drop dead _client.headers writes from set_header --- datadog_checks_base/datadog_checks/base/utils/httpx2.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 49e4449d2326e..1f0add1f41cf8 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -299,7 +299,7 @@ def __init__( # TODO: wire proxies through to httpx2 as part of the ongoing httpx migration. # options['headers'] is the per-request source of truth and is re-read in _build_request_kwargs, # so direct mutation (__setitem__, update(), whole-dict replacement) reaches the wire. - # set_header keeps options['headers'] and _client.headers in sync for callers that prefer it. + # set_header mutates options['headers'] in place; the per-request merge above re-reads it. self.options: dict[str, Any] = { 'auth': auth, 'cert': cert, @@ -367,14 +367,11 @@ def get_header(self, name: str, default: str | None = None) -> str | None: return default def set_header(self, name: str, value: str) -> None: - # Mirror into both stores: options['headers'] is the public shape, _client.headers is what httpx2 sends. for key in list(self.options['headers']): if key.lower() == name.lower(): self.options['headers'][key] = value - self._client.headers[key] = value return self.options['headers'][name] = value - self._client.headers[name] = value def get(self, url: str, **options: Any) -> HTTPX2ResponseAdapter: return self._request('GET', url, options) From ce35d7bb7fdafddc2c44c8b508ce15e5c9d5ec9e Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Mon, 8 Jun 2026 21:16:47 -0400 Subject: [PATCH 43/45] Accept per-request auth kwarg in HTTPX2Wrapper --- .../datadog_checks/base/utils/httpx2.py | 6 +- .../base/utils/httpx2/test_request_auth.py | 66 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 datadog_checks_base/tests/base/utils/httpx2/test_request_auth.py diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index 95f84ec2a0397..bf3776698505c 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -57,9 +57,10 @@ 'extra_headers', 'timeout', 'follow_redirects', + 'auth', } ) -REQUEST_KWARGS_SPECIAL = frozenset({'headers', 'extra_headers', 'timeout', 'follow_redirects'}) +REQUEST_KWARGS_SPECIAL = frozenset({'headers', 'extra_headers', 'timeout', 'follow_redirects', 'auth'}) REQUEST_KWARGS_PASSTHROUGH = REQUEST_KWARGS - REQUEST_KWARGS_SPECIAL assert REQUEST_KWARGS_PASSTHROUGH | REQUEST_KWARGS_SPECIAL == REQUEST_KWARGS assert REQUEST_KWARGS_PASSTHROUGH & REQUEST_KWARGS_SPECIAL == frozenset() @@ -404,10 +405,11 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPX2Resp follow_redirects = ( bool(options['follow_redirects']) if 'follow_redirects' in options else httpx2.USE_CLIENT_DEFAULT ) + auth = options['auth'] if 'auth' in options else httpx2.USE_CLIENT_DEFAULT request_kwargs = self._build_request_kwargs(options, method=method, url=url) try: request = self._client.build_request(method, url, **request_kwargs) - response = self._client.send(request, stream=True, follow_redirects=follow_redirects) + response = self._client.send(request, stream=True, follow_redirects=follow_redirects, auth=auth) except (httpx2.HTTPError, httpx2.InvalidURL) as exc: raise _map_httpx2_exception(exc) from exc try: diff --git a/datadog_checks_base/tests/base/utils/httpx2/test_request_auth.py b/datadog_checks_base/tests/base/utils/httpx2/test_request_auth.py new file mode 100644 index 0000000000000..7092fe39276bc --- /dev/null +++ b/datadog_checks_base/tests/base/utils/httpx2/test_request_auth.py @@ -0,0 +1,66 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import httpx2 +import pytest + +from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper + +from .common import parse_basic_auth + + +def test_per_request_auth_tuple_sets_basic_auth_header(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.get('http://example.test/', auth=('alice', 'secret')) + user, password = parse_basic_auth(captured_requests[0].headers['authorization']) + assert user == 'alice' + assert password == 'secret' + + +def test_per_request_auth_basicauth_instance_sets_basic_auth_header(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.get('http://example.test/', auth=httpx2.BasicAuth('bob', 'hunter2')) + user, password = parse_basic_auth(captured_requests[0].headers['authorization']) + assert user == 'bob' + assert password == 'hunter2' + + +def test_per_request_auth_shadows_client_level_auth(capturing_transport, captured_requests): + http = HTTPX2Wrapper( + {'username': 'alice', 'password': 'secret'}, + {}, + transport=capturing_transport, + ) + http.get('http://example.test/', auth=('carol', 'override')) + user, password = parse_basic_auth(captured_requests[0].headers['authorization']) + assert user == 'carol' + assert password == 'override' + + +def test_per_request_auth_alongside_other_kwargs_reaches_wire(capturing_transport, captured_requests): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + http.get( + 'http://example.test/resource', + auth=('alice', 'secret'), + params={'q': 'value'}, + headers={'X-Trace': 'abc'}, + ) + request = captured_requests[0] + user, password = parse_basic_auth(request.headers['authorization']) + assert user == 'alice' + assert password == 'secret' + assert request.headers['x-trace'] == 'abc' + assert request.url.params['q'] == 'value' + + +@pytest.mark.parametrize( + 'auth_value', + [ + pytest.param(('alice', 'secret'), id='tuple'), + pytest.param(httpx2.BasicAuth('alice', 'secret'), id='basicauth'), + ], +) +def test_per_request_auth_does_not_raise_typeerror(auth_value, capturing_transport): + http = HTTPX2Wrapper({}, {}, transport=capturing_transport) + response = http.get('http://example.test/', auth=auth_value) + assert response.status_code == 200 From e54c89dc3dad07f2a2c56890f6926b26d59a1266 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Wed, 17 Jun 2026 16:49:11 -0400 Subject: [PATCH 44/45] Type-hint _build_http_client instance parameter Co-Authored-By: Claude Opus 4.8 (1M context) --- datadog_checks_base/datadog_checks/base/checks/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 8341aed481ac4..f69f29e80a757 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -404,7 +404,7 @@ def _get_metric_limit(self, instance=None): return limit - def _build_http_client(self, instance) -> HTTPClientProtocol: + def _build_http_client(self, instance: dict[str, Any]) -> HTTPClientProtocol: if is_affirmative(instance.get('use_httpx2', False)): from datadog_checks.base.utils.httpx2 import HTTPX2Wrapper From 5d71fa6dd0ffb7839dcf1b61f9495462574fb620 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Wed, 17 Jun 2026 16:49:11 -0400 Subject: [PATCH 45/45] Document SSL folds into HTTPConnectionError in _map_httpx2_exception Co-Authored-By: Claude Opus 4.8 (1M context) --- datadog_checks_base/datadog_checks/base/utils/httpx2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datadog_checks_base/datadog_checks/base/utils/httpx2.py b/datadog_checks_base/datadog_checks/base/utils/httpx2.py index bf3776698505c..85fbdf390a6a0 100644 --- a/datadog_checks_base/datadog_checks/base/utils/httpx2.py +++ b/datadog_checks_base/datadog_checks/base/utils/httpx2.py @@ -116,6 +116,8 @@ def _map_httpx2_exception(exc: httpx2.HTTPError | httpx2.InvalidURL) -> HTTPErro return HTTPInvalidURLError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) if isinstance(exc, httpx2.TimeoutException): return HTTPTimeoutError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) + # httpx2 has no dedicated SSL/cert exception: TLS failures surface as ConnectError (a NetworkError), + # so they fold into the generic HTTPConnectionError. The HTTPSSLError subtype is intentionally unproduced here. if isinstance(exc, httpx2.NetworkError): return HTTPConnectionError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) if isinstance(exc, httpx2.HTTPStatusError):