Complete the backend-neutral HTTP protocol and migrate intgs onto it

activemq_xml/datadog_checks/activemq_xml/activemq_xml.py

@@ -3,10 +3,9 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 from xml.etree import ElementTree
 
-import requests
-
 from datadog_checks.base import AgentCheck
 from datadog_checks.base.config import _is_affirmative
+from datadog_checks.base.utils.http_exceptions import HTTPConnectionError, HTTPTimeoutError
 
 QUEUE_URL = "/admin/xml/queues.jsp"
 TOPIC_URL = "/admin/xml/topics.jsp"
@@ -57,7 +56,7 @@ def _fetch_data(self, base_url, xml_url, suppress_errors):
         try:
             r = self.http.get(url)
             r.raise_for_status()
-        except requests.exceptions.ConnectionError:
+        except (HTTPConnectionError, HTTPTimeoutError):
             if suppress_errors:
                 self.log.warning("ActiveMQ not contactable, but suppressing the error due to configuration.")
                 return False

amazon_msk/tests/conftest.py

@@ -7,8 +7,8 @@
 import mock
 import pytest
 
-from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.dev import docker_run
+from datadog_checks.dev.http import MockHTTPResponse
 
 from . import common
 

ambari/datadog_checks/ambari/ambari.py

@@ -1,11 +1,10 @@
 # (C) Datadog, Inc. 2019-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
-from requests.exceptions import ConnectionError, HTTPError, Timeout
-
 from datadog_checks.base import AgentCheck
 from datadog_checks.base.constants import ServiceCheck
 from datadog_checks.base.errors import CheckException
+from datadog_checks.base.utils.http_exceptions import HTTPConnectionError, HTTPStatusError, HTTPTimeoutError
 
 from . import common
 
@@ -187,11 +186,11 @@ def _make_request(self, url):
         try:
             resp = self.http.get(url)
             return resp.json()
-        except (HTTPError, ConnectionError) as e:
+        except (HTTPStatusError, HTTPConnectionError) as e:
             self.warning(
                 "Couldn't connect to URL: %s with exception: %s. Please verify the address is reachable", url, e
             )
-        except Timeout:
+        except HTTPTimeoutError:
             self.warning("Connection timeout when connecting to %s", url)
         except Exception as e:
             self.warning("Couldn't connect to URL: %s with exception: %s.", url, e)

apache/tests/conftest.py

@@ -8,9 +8,9 @@
 import requests
 
 from datadog_checks.apache import Apache
-from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.dev import docker_run
 from datadog_checks.dev.conditions import CheckEndpoints, WaitFor
+from datadog_checks.dev.http import MockHTTPResponse
 
 from .common import AUTO_STATUS_URL, BASE_URL, CHECK_NAME, HERE, STATUS_CONFIG, STATUS_URL
 

arangodb/datadog_checks/arangodb/check.py

@@ -3,8 +3,6 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 from urllib.parse import urlparse
 
-from requests import HTTPError
-
 from datadog_checks.base import OpenMetricsBaseCheckV2
 from datadog_checks.base.utils.http_exceptions import HTTPStatusError
 
@@ -57,7 +55,7 @@ def get_server_tag(self, tag_name, endpoint):
 
             return 'server_{}:{}'.format(tag_name, response.json()[tag_name])
 
-        except (HTTPError, HTTPStatusError):
+        except HTTPStatusError:
             self.log.debug("Unable to get server %s, skipping `server_%s` tag.", tag_name, tag_name)
         except Exception as e:
             self.log.debug(

arangodb/tests/test_arangodb.py

@@ -9,7 +9,7 @@
 
 from datadog_checks.arangodb import ArangodbCheck
 from datadog_checks.base.utils.http_exceptions import HTTPStatusError
-from datadog_checks.base.utils.http_testing import MockHTTPResponse
+from datadog_checks.dev.http import MockHTTPResponse
 from datadog_checks.dev.utils import get_metadata_metrics
 
 from .common import METRICS

avi_vantage/datadog_checks/avi_vantage/check.py

@@ -72,7 +72,7 @@ def configure_scrapers(self):
 
     @contextmanager
     def login(self):
-        self.http._session = None
+        self.http.close()
         login_url = self.base_url + "/login"
         logout_url = self.base_url + "/logout"
         try:
@@ -87,7 +87,7 @@ def login(self):
             self.service_check("can_connect", AgentCheck.OK, tags=self.config.tags)
 
         yield
-        csrf_token = self.http.session.cookies.get('csrftoken')
+        csrf_token = self.http.get_cookie('csrftoken')
         if csrf_token:
             logout_resp = self.http.post(
                 logout_url, extra_headers={'X-CSRFToken': csrf_token, 'Referer': self.base_url}

avi_vantage/tests/conftest.py

@@ -9,9 +9,9 @@
 
 import pytest
 
-from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.dev import docker_run, get_docker_hostname, get_here
 from datadog_checks.dev.conditions import CheckDockerLogs
+from datadog_checks.dev.http import MockHTTPResponse
 
 HERE = get_here()
 

cert_manager/tests/test_cert_manager.py

@@ -5,8 +5,8 @@
 
 import pytest
 
-from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.cert_manager import CertManagerCheck
+from datadog_checks.dev.http import MockHTTPResponse
 
 from .common import ACME_METRICS, CERT_METRICS, CONTROLLER_METRICS, MOCK_INSTANCE
 

cilium/tests/conftest.py

@@ -6,10 +6,10 @@
 import pytest
 
 from datadog_checks.base.utils.common import get_docker_hostname
-from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.cilium import CiliumCheck
 from datadog_checks.dev import run_command
 from datadog_checks.dev.conditions import WaitFor
+from datadog_checks.dev.http import MockHTTPResponse
 from datadog_checks.dev.kind import kind_run
 from datadog_checks.dev.kube_port_forward import port_forward
 from datadog_checks.dev.utils import get_active_env

cisco_aci/datadog_checks/cisco_aci/api.py

@@ -127,7 +127,7 @@ def __init__(
         self.sessions = {}
 
     def close(self):
-        self.http.session.close()
+        self.http.close()
 
     def setup_cert_login(self, aci_url):
         session_wrapper = self.wrapper_factory(

citrix_hypervisor/datadog_checks/citrix_hypervisor/check.py

@@ -10,7 +10,6 @@
 import yaml
 
 from datadog_checks.base import AgentCheck, ConfigurationError
-from datadog_checks.base.utils.http import requests
 
 from .metrics import build_metric
 
@@ -26,7 +25,7 @@ def _safely_process_metrics_response(data: ResponseWrapper) -> dict[str, dict]:
     # See https://github.com/yaml/pyyaml/issues/443
     try:
         return data.json()
-    except (requests.exceptions.JSONDecodeError, StdJSONDecodeError):
+    except StdJSONDecodeError:
         return yaml.load(data.content, Loader=yaml.SafeLoader)
 
 

citrix_hypervisor/tests/conftest.py

@@ -5,8 +5,8 @@
 
 import pytest
 
-from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.dev import docker_run
+from datadog_checks.dev.http import MockHTTPResponse
 
 from . import common
 

cloud_foundry_api/datadog_checks/cloud_foundry_api/cloud_foundry_api.py

@@ -7,8 +7,6 @@
 from typing import Any, Dict, Generator, Tuple  # noqa: F401
 from urllib.parse import urlparse
 
-from requests.exceptions import HTTPError as RequestsHTTPError
-from requests.exceptions import RequestException
 from semver import VersionInfo
 
 from datadog_checks.base import AgentCheck
@@ -66,12 +64,12 @@ def discover_api(self):
         self.log.info("Discovering Cloud Foundry API version and authentication endpoint")
         try:
             res = self.http.get(self._api_url)
-        except (RequestException, HTTPError):
+        except HTTPError:
             self.log.exception("Error connecting to the API server")
             raise
         try:
             res.raise_for_status()
-        except (RequestsHTTPError, HTTPStatusError):
+        except HTTPStatusError:
             self.log.exception("Error querying API information: response: %s", res.text)
             raise
         try:
@@ -116,13 +114,13 @@ def get_oauth_token(self):
                 auth=(self._client_id, self._client_secret),  # SKIP_HTTP_VALIDATION`
                 params={"grant_type": "client_credentials"},
             )
-        except (RequestException, HTTPError):
+        except HTTPError:
             self.log.exception("Error connecting to the UAA server")
             self.service_check(UAA_SERVICE_CHECK_NAME, CloudFoundryApiCheck.CRITICAL, tags=sc_tags)
             raise
         try:
             res.raise_for_status()
-        except (RequestsHTTPError, HTTPStatusError):
+        except HTTPStatusError:
             self.log.exception("Error authenticating to the UAA server: response: %s", res.text)
             self.service_check(UAA_SERVICE_CHECK_NAME, CloudFoundryApiCheck.CRITICAL, tags=sc_tags)
             raise
@@ -259,14 +257,14 @@ def scroll_api_pages(self, url, params, headers):
                 self.log.debug("Fetching events page %s", page)
                 try:
                     res = self.http.get(url, params=params, headers=headers)
-                except (RequestException, HTTPError):
+                except HTTPError:
                     self.log.exception("Error connecting to the Cloud Controller API: URL %s", url)
                     self.service_check(API_SERVICE_CHECK_NAME, CloudFoundryApiCheck.CRITICAL, tags=sc_tags)
                     raised = True
                     return
                 try:
                     res.raise_for_status()
-                except (RequestsHTTPError, HTTPStatusError):
+                except HTTPStatusError:
                     self.log.exception("Error querying Cloud Controller API: URL %s - response %s", url, res.text)
                     self.service_check(API_SERVICE_CHECK_NAME, CloudFoundryApiCheck.CRITICAL, tags=sc_tags)
                     raised = True

consul/datadog_checks/consul/consul.py

@@ -11,12 +11,10 @@
 from time import time as timestamp
 from urllib.parse import urljoin
 
-import requests
 from cachetools import TTLCache
-from requests import HTTPError
 
 from datadog_checks.base import ConfigurationError, OpenMetricsBaseCheck, is_affirmative
-from datadog_checks.base.utils.http_exceptions import HTTPTimeoutError
+from datadog_checks.base.utils.http_exceptions import HTTPStatusError, HTTPTimeoutError
 from datadog_checks.base.utils.serialization import json
 
 from .common import (
@@ -169,7 +167,7 @@ def consul_request(self, endpoint):
 
             resp.raise_for_status()
 
-        except (requests.exceptions.Timeout, HTTPTimeoutError) as e:
+        except HTTPTimeoutError as e:
             msg = 'Consul request to {} timed out'.format(url)
             self.log.exception(msg)
             self.service_check(
@@ -724,7 +722,7 @@ def _check_prometheus_endpoint(self):
                 "Update Consul or set back `use_prometheus_endpoint` to false to remove this warning. %s",
                 str(e),
             )
-        except HTTPError as e:
+        except HTTPStatusError as e:
             if e.response.status_code == 404:
                 self.log.warning(
                     "This Consul version (< 1.1.0) does not support the prometheus endpoint. "

control_m/datadog_checks/control_m/client.py

@@ -11,9 +11,8 @@
 if TYPE_CHECKING:
     from collections.abc import Callable
 
-    from requests import Response
-
     from datadog_checks.base import AgentCheck
+    from datadog_checks.base.utils.http_protocol import HTTPResponse
 
 
 class ControlMClient:
@@ -116,7 +115,7 @@ def ensure_token(self) -> None:
         self._log.info("Refreshing Control-M API token")
         self.login()
 
-    def request(self, method: str, url: str, **kwargs: Any) -> Response:
+    def request(self, method: str, url: str, **kwargs: Any) -> HTTPResponse:
         # Route the request through static token or session login, with 401 fallback.
         request_fn = getattr(self._http, method)
 
@@ -141,7 +140,7 @@ def request(self, method: str, url: str, **kwargs: Any) -> Response:
         self._static_token_retry_after = time.monotonic() + self._static_token_retry_interval
         return self._session_request(request_fn, url, **kwargs)
 
-    def _session_request(self, request_fn: Callable[..., Response], url: str, **kwargs: Any) -> Response:
+    def _session_request(self, request_fn: Callable[..., HTTPResponse], url: str, **kwargs: Any) -> HTTPResponse:
         # Used in session login mode only. Make a request to the API using the session token.
         self.ensure_token()
         extra = {"Authorization": f"Bearer {self._token}"}

couch/datadog_checks/couch/couch.py

@@ -7,12 +7,10 @@
 import math
 from urllib.parse import quote, urljoin
 
-import requests
-
 from datadog_checks.base import AgentCheck
 from datadog_checks.base.errors import CheckException, ConfigurationError
 from datadog_checks.base.utils.headers import headers
-from datadog_checks.base.utils.http_exceptions import HTTPTimeoutError
+from datadog_checks.base.utils.http_exceptions import HTTPStatusError, HTTPTimeoutError
 from datadog_checks.couch import errors
 
 
@@ -44,15 +42,15 @@ def get(self, url, service_check_tags, run_check=False):
                     AgentCheck.OK,
                     tags=service_check_tags,
                 )
-        except (requests.exceptions.Timeout, HTTPTimeoutError) as e:
+        except HTTPTimeoutError as e:
             self.service_check(
                 self.SERVICE_CHECK_NAME,
                 AgentCheck.CRITICAL,
                 tags=service_check_tags,
                 message="Request timeout: {0}, {1}".format(url, e),
             )
             raise
-        except requests.exceptions.HTTPError as e:
+        except HTTPStatusError as e:
             self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, tags=service_check_tags, message=str(e))
             raise
         except Exception as e:
@@ -176,7 +174,7 @@ def get_data(self, server, tags):
                 db_stats = None
                 try:
                     db_stats = self.agent_check.get(url, tags)
-                except requests.exceptions.HTTPError as e:
+                except HTTPStatusError as e:
                     couchdb['databases'][dbName] = None
                     if (e.response.status_code == 403) or (e.response.status_code == 401):
                         self.db_exclude[server].append(dbName)