saluki_io/net/util/retry/lifecycle/
http.rs

1use std::{borrow::Cow, error::Error as _, fmt, time::Duration};
2
3use http::StatusCode;
4use tracing::{debug, warn};
5
6use super::RetryLifecycle;
7
8/// A standard HTTP retry lifecycle that emits contextual information about HTTP requests and responses..
9///
10/// This lifecycle emits user-friendly logs about retry attempts, including the request URI and response code. It
11/// provides additional destructuring/introspection of errors to surface contextual information such as requests failing
12/// due to DNS, connection errors, TLS, and so on.
13#[derive(Clone)]
14pub struct StandardHttpRetryLifecycle;
15
16impl<B, B2, E> RetryLifecycle<http::Request<B>, http::Response<B2>, E> for StandardHttpRetryLifecycle
17where
18    E: DynError,
19{
20    fn before_retry(
21        &self, req: &http::Request<B>, res: &Result<http::Response<B2>, E>, retry_backoff: Duration, error_count: u32,
22    ) {
23        let request_uri = SanitizedRequestUri(req.uri());
24        let categorized_error = CategorizedError::try_categorize(res);
25
26        warn!(error_count, %request_uri, "{}. Retrying after {:?}.", categorized_error, retry_backoff);
27    }
28
29    fn after_success(&self, req: &http::Request<B>, _: &Result<http::Response<B2>, E>) {
30        let request_uri = SanitizedRequestUri(req.uri());
31        debug!(%request_uri, "Request succeeded.");
32    }
33}
34
35struct SanitizedRequestUri<'a>(&'a http::Uri);
36
37impl fmt::Display for SanitizedRequestUri<'_> {
38    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
39        // We _should_ always have a scheme and a host, but we'll just make sure they exist first, to be safe. We'll
40        // require needing both to be present to print either of them.
41        let maybe_scheme = self.0.scheme_str();
42        let maybe_host = self.0.host();
43
44        if let (Some(scheme), Some(host)) = (maybe_scheme, maybe_host) {
45            write!(f, "{}://{}", scheme, host)?;
46        }
47
48        let maybe_port = self.0.port_u16();
49        if let Some(port) = maybe_port {
50            write!(f, ":{}", port)?;
51        }
52
53        // Now print the request path, which is always present.
54        write!(f, "{}", self.0.path())
55    }
56}
57
58enum CategorizedError {
59    Client(String),
60    Tls(String),
61    Http(StatusCode),
62    Other(String),
63}
64
65impl CategorizedError {
66    fn try_categorize<B, E>(res: &Result<http::Response<B>, E>) -> Self
67    where
68        E: DynError,
69    {
70        match res {
71            Ok(resp) => Self::Http(resp.status()),
72            Err(e) => Self::extract_nested(e.as_dyn_error()),
73        }
74    }
75
76    fn extract_nested(error: &(dyn std::error::Error + 'static)) -> Self {
77        // See if we have a `hyper-util` error.
78        if let Some(hyper_error) = error.downcast_ref::<hyper_util::client::legacy::Error>() {
79            return match hyper_error.source() {
80                Some(source) => Self::extract_nested(source),
81                None => Self::Client(hyper_error.to_string()),
82            };
83        }
84
85        // See if we have a `rustls` error.
86        if let Some(rustls_error) = error.downcast_ref::<rustls::Error>() {
87            return Self::from_rustls(rustls_error);
88        }
89
90        // See if we have a generic `std::io::Error`.
91        //
92        // It may be wrapping something else, or it may be standalone, so we'll try and suss that out.
93        if let Some(io_error) = error.downcast_ref::<std::io::Error>() {
94            return match io_error.get_ref() {
95                Some(source) => Self::extract_nested(source),
96                None => Self::Other(io_error.to_string()),
97            };
98        }
99
100        Self::Other(error.to_string())
101    }
102
103    fn from_rustls(error: &rustls::Error) -> Self {
104        // We're really just specializing a few known types of errors to generate a better error message, but otherwise
105        // we'll fallback on the description given by the error itself.
106        let reason = match error {
107            rustls::Error::InvalidCertificate(cert_error) => format!(
108                "peer certificate is invalid: {}",
109                rustls_cert_error_to_string(cert_error)
110            ),
111            _ => error.to_string(),
112        };
113
114        Self::Tls(reason)
115    }
116}
117
118impl fmt::Display for CategorizedError {
119    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
120        match self {
121            CategorizedError::Client(reason) => write!(f, "Request failed due to a client error: {}", reason),
122            CategorizedError::Tls(reason) => write!(f, "Request failed due to a TLS error: {}", reason),
123            CategorizedError::Http(status_code) => write!(
124                f,
125                "Server responded with non-success status code {}.",
126                status_code.as_str()
127            ),
128            CategorizedError::Other(reason) => write!(f, "Request failed: {}", reason),
129        }
130    }
131}
132
133fn rustls_cert_error_to_string(cert_error: &rustls::CertificateError) -> Cow<'static, str> {
134    match cert_error {
135        rustls::CertificateError::BadEncoding => "certificate incorrectly encoded".into(),
136        rustls::CertificateError::Expired => "certificate expired (current time is after notAfter time)".into(),
137        rustls::CertificateError::NotValidYet => {
138            "certificate not valid yet (current time is before notBefore time)".into()
139        }
140        rustls::CertificateError::Revoked => "certificate has been revoked".into(),
141        rustls::CertificateError::UnhandledCriticalExtension => {
142            "certificate contains an extension marked critical, but it was not processed by the certificate validator"
143                .into()
144        }
145        rustls::CertificateError::UnknownIssuer => "certificate chain is not issued by a known root certificate".into(),
146        rustls::CertificateError::UnknownRevocationStatus => {
147            "certificate's revocation status could not be determined".into()
148        }
149        rustls::CertificateError::ExpiredRevocationList => {
150            "certificate's revocation status could not be determined due to an expired CRL".into()
151        }
152        rustls::CertificateError::BadSignature => {
153            "certificate is not signed correctly by the key of its alleged issuer".into()
154        }
155        rustls::CertificateError::NotValidForName => "certificate is not valid for the given entity name".into(),
156        rustls::CertificateError::InvalidPurpose => "certificate is not valid for the requested purpose".into(),
157        rustls::CertificateError::ApplicationVerificationFailure => {
158            "certificate is valid overall, but the handshake was rejected".into()
159        }
160
161        // This one could be a generic error that doesn't fit the above, returned by `rustls`, or it could be coming from
162        // a custom certificate verifier which we don't know about, or can't reasonably know about to compensate for
163        // here... so we'll just return it as-is.
164        rustls::CertificateError::Other(other) => format!("generic error: {}", other).into(),
165        other => format!("generic unhandled error: {:?}", other).into(),
166    }
167}
168
169// Market trait for accepting generically-typed errors that can be downcasted to dynamically-dispatched trait references.
170trait DynError {
171    fn as_dyn_error(&self) -> &(dyn std::error::Error + 'static);
172}
173
174impl DynError for Box<dyn std::error::Error + Send + Sync> {
175    fn as_dyn_error(&self) -> &(dyn std::error::Error + 'static) {
176        &**self
177    }
178}
179
180#[cfg(test)]
181mod tests {
182    use std::io;
183
184    use http::{Response, StatusCode, Uri};
185
186    use super::*;
187
188    type BoxError = Box<dyn std::error::Error + Send + Sync>;
189
190    fn categorize(res: Result<Response<()>, BoxError>) -> String {
191        CategorizedError::try_categorize(&res).to_string()
192    }
193
194    #[test]
195    fn categorizes_http_status_response() {
196        // A response (any `Ok`) is categorized by its status code, and non-success codes render with the code value.
197        let res: Result<Response<()>, BoxError> = Ok(Response::builder()
198            .status(StatusCode::INTERNAL_SERVER_ERROR)
199            .body(())
200            .unwrap());
201        assert_eq!(categorize(res), "Server responded with non-success status code 500.");
202    }
203
204    #[test]
205    fn categorizes_standalone_io_error_as_other() {
206        // A bare transport error that isn't a hyper/rustls/nested error falls through to the generic "Other" bucket.
207        let err: BoxError = Box::new(io::Error::from(io::ErrorKind::ConnectionRefused));
208        assert_eq!(categorize(Err(err)), "Request failed: connection refused");
209    }
210
211    #[test]
212    fn categorizes_rustls_certificate_error_as_tls() {
213        // A rustls certificate error is specialized into a TLS category with a human-readable reason.
214        let err: BoxError = Box::new(rustls::Error::InvalidCertificate(rustls::CertificateError::Expired));
215        assert_eq!(
216            categorize(Err(err)),
217            "Request failed due to a TLS error: peer certificate is invalid: certificate expired (current time is after notAfter time)"
218        );
219    }
220
221    #[test]
222    fn categorizes_io_wrapped_rustls_error_by_unwrapping_source() {
223        // An io::Error that wraps a rustls error is unwrapped via its source and categorized as TLS, not reported as
224        // a generic io failure.
225        let inner = rustls::Error::InvalidCertificate(rustls::CertificateError::Revoked);
226        let err: BoxError = Box::new(io::Error::other(inner));
227        assert_eq!(
228            categorize(Err(err)),
229            "Request failed due to a TLS error: peer certificate is invalid: certificate has been revoked"
230        );
231    }
232
233    #[test]
234    fn sanitized_request_uri_display() {
235        // Scheme, host, explicit port, and path are all rendered.
236        let uri: Uri = "http://localhost:8125/foo/bar".parse().unwrap();
237        assert_eq!(SanitizedRequestUri(&uri).to_string(), "http://localhost:8125/foo/bar");
238
239        // A default (implicit) port is omitted.
240        let uri: Uri = "https://api.datadoghq.com/api/v1/series".parse().unwrap();
241        assert_eq!(
242            SanitizedRequestUri(&uri).to_string(),
243            "https://api.datadoghq.com/api/v1/series"
244        );
245
246        // With neither scheme nor host, only the path is rendered.
247        let uri: Uri = "/health".parse().unwrap();
248        assert_eq!(SanitizedRequestUri(&uri).to_string(), "/health");
249    }
250}
saluki_io/net/util/retry/lifecycle/http.rs

saluki_io/net/util/retry/lifecycle/
http.rs
