saluki_io/net/util/retry/lifecycle/
telemetry.rs

1use std::sync::{Arc, Mutex};
2
3use metrics::Counter;
4use saluki_common::collections::FastHashMap;
5use saluki_metrics::MetricsBuilder;
6
7use super::http::RetryCause;
8
9const NETWORK_HTTP_REQUESTS_RETRY_CAUSES_TOTAL: &str = "network_http_requests_retry_causes_total";
10
11/// Counts retried requests by what caused the retry.
12///
13/// Attach this to [`StandardHttpRetryLifecycle`][super::StandardHttpRetryLifecycle] to count retries as they're decided,
14/// at the point where the response or the error that triggered them is still available.
15///
16/// ## Metrics
17///
18/// - `network_http_requests_retry_causes_total`: the total number of retried requests, tagged with the `domain` given at
19///   construction and with a bounded description of the failure:
20///   - `cause`: one of `http_status`, `http2`, `tls`, `timeout`, `connection`, `client`, or `other`.
21///   - `frame` and `initiator`: for `cause:http2` only. `frame` is the HTTP/2 frame the error came from, and
22///     `initiator` is the side of the connection that sent it, where `local` means this client ended the request rather
23///     than the remote endpoint.
24///   - `reason`: for `cause:http2`, the HTTP/2 reason code by name; for `cause:timeout`, the stage that timed out; for
25///     `cause:connection`, the kind of transport failure. Absent when the failure carries no reason we recognize.
26///
27/// Retries caused by response status codes aren't broken down by code here, since
28/// `network_http_requests_errors_total` already counts every non-success response by status code.
29#[derive(Clone)]
30pub struct RetryCauseTelemetry {
31    builder: MetricsBuilder,
32
33    /// Counters by cause, registered on first use.
34    ///
35    /// The key is an enumerated value, so this map is bounded by the number of causes that can be expressed, and it only
36    /// grows on the failure path.
37    counters: Arc<Mutex<FastHashMap<RetryCause, Counter>>>,
38}
39
40impl RetryCauseTelemetry {
41    /// Creates retry cause telemetry for requests sent to `domain`.
42    ///
43    /// The domain is used as-is for the `domain` tag, so pass the same value that the rest of a client's telemetry uses:
44    /// the scheme, host, and port of the remote endpoint.
45    pub fn from_builder(builder: &MetricsBuilder, domain: &str) -> Self {
46        Self {
47            builder: builder.clone().add_default_tag(("domain", domain.to_string())),
48            counters: Arc::new(Mutex::new(FastHashMap::default())),
49        }
50    }
51
52    /// Counts a single retry.
53    pub(super) fn increment(&self, cause: RetryCause) {
54        let mut counters = self.counters.lock().unwrap();
55        counters
56            .entry(cause)
57            .or_insert_with(|| {
58                self.builder
59                    .register_counter_with_tags(NETWORK_HTTP_REQUESTS_RETRY_CAUSES_TOTAL, cause.tags())
60            })
61            .increment(1);
62    }
63}
saluki_io/net/util/retry/lifecycle/telemetry.rs

saluki_io/net/util/retry/lifecycle/
telemetry.rs
