saluki_core/runtime/tree/snapshot.rs
1//! Serialized form of a supervision-tree snapshot.
2//!
3//! These are the wire types: the stable shape that [`SupervisionTreeHandle::snapshot`][super::SupervisionTreeHandle]
4//! produces, the API route serves, and the CLI decodes. They are deliberately separate from the live bookkeeping in
5//! the parent module -- a rename here changes a payload that shipped binaries parse, which is a very different kind of
6//! change from adjusting how the tree is tracked at runtime.
7
8use saluki_common::resource_tracking::ResourceStatsSnapshot;
9use serde::{Deserialize, Serialize};
10
11use crate::runtime::{restart::RestartType, supervisor::AutoShutdown, RestartMode};
12
13/// A point in time, as milliseconds since the Unix epoch.
14///
15/// Serialized as a plain integer. The standard library's own representation of a timestamp serializes as a pair of
16/// fields, which is awkward for a consumer that just wants to render a time.
17#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
18#[serde(transparent)]
19pub struct UnixMillis(pub u64);
20
21/// A point-in-time view of a supervision tree.
22#[derive(Clone, Debug, Deserialize, Serialize)]
23pub struct TreeSnapshot {
24 /// When the snapshot was taken.
25 pub captured_at: UnixMillis,
26
27 /// Whether allocations are being tracked at all.
28 ///
29 /// When false, every byte count in the snapshot reads zero because nothing is measuring, rather than because
30 /// nothing has been allocated. Distinguishing the two matters: the tracking allocator has to be installed as the
31 /// process's global allocator, which not every embedding does.
32 pub resource_tracking_enabled: bool,
33
34 /// Aggregate counts across the whole tree.
35 pub totals: TreeTotals,
36
37 /// The supervisor the snapshot was taken from, and everything beneath it.
38 pub root: NodeSnapshot,
39}
40
41/// Aggregate counts across a whole [`TreeSnapshot`].
42#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)]
43pub struct TreeTotals {
44 /// Number of supervisors in the tree.
45 pub supervisors: usize,
46
47 /// Number of leaf workers in the tree.
48 pub workers: usize,
49
50 /// Number of nodes currently running.
51 pub running: usize,
52
53 /// Number of nodes that ran and have since exited without being restarted.
54 pub exited: usize,
55
56 /// Number of nodes that are declared but have never run.
57 pub registered: usize,
58
59 /// Total restarts across every node in the tree.
60 pub restarts: u64,
61
62 /// Total live bytes across every distinct resource group in the tree.
63 ///
64 /// Summed over groups rather than over nodes: several nodes can share one group, and their usage is one figure
65 /// rather than one per node.
66 pub live_bytes: u64,
67
68 /// Total CPU time across every distinct resource group in the tree, in nanoseconds.
69 pub cpu_time_nanos: u64,
70
71 /// Depth of the deepest node, counting the root as 1.
72 pub max_depth: usize,
73}
74
75/// Whether a node supervises other nodes or performs work itself.
76#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
77#[serde(rename_all = "snake_case")]
78pub enum NodeKind {
79 /// A supervisor, which manages other nodes.
80 Supervisor,
81
82 /// A worker, which performs work and has no children of its own.
83 Worker,
84}
85
86/// Where a node is in its lifecycle.
87#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
88#[serde(rename_all = "snake_case")]
89pub enum NodeState {
90 /// Declared, but not currently running.
91 ///
92 /// Either it has never run, or -- for a supervisor being restarted -- it has stopped and its next generation has
93 /// not yet started.
94 Registered,
95
96 /// Currently running.
97 Running,
98
99 /// Ran, exited, and was not restarted.
100 Exited,
101}
102
103/// One node -- a supervisor or a worker -- in a [`TreeSnapshot`].
104#[derive(Clone, Debug, Deserialize, Serialize)]
105pub struct NodeSnapshot {
106 /// The node's bare name, as registered with its supervisor.
107 pub name: String,
108
109 /// Whether the node is a supervisor or a worker.
110 pub kind: NodeKind,
111
112 /// The node's fully qualified, dot-scoped process name. `None` if the node has never run.
113 pub process_name: Option<String>,
114
115 /// Identifier of the node's most recent process. `None` if the node has never run.
116 ///
117 /// A restart gives the node a new process, and so a new identifier. A node that has stopped keeps the identifier
118 /// it last ran under, which is what `state` is for: this says what the node ran as, and `state` says whether it
119 /// still is.
120 pub process_id: Option<u64>,
121
122 /// Where the node is in its lifecycle.
123 pub state: NodeState,
124
125 /// The node's restart policy.
126 pub restart: RestartType,
127
128 /// Whether the node's termination can drive its supervisor to shut down.
129 pub significant: bool,
130
131 /// When the node first became part of the tree.
132 ///
133 /// Constant across restarts, so the difference between this and `started_at` is the time the node has spent not
134 /// running since it was created.
135 pub created_at: UnixMillis,
136
137 /// When the node's most recent process started. `None` if the node has never run.
138 pub started_at: Option<UnixMillis>,
139
140 /// How long the node's current process has been running, in milliseconds. `None` unless it is running.
141 pub uptime_ms: Option<u64>,
142
143 /// How many times the node has been restarted since it was created.
144 ///
145 /// Counts every restart that gave the node a new process, whether it was restarted on its own account or brought
146 /// back as part of its supervisor restarting -- either by a group restart or by the supervisor itself being
147 /// restarted from above. Only statically declared nodes accumulate this across a supervisor restart, since only
148 /// they have an identity that survives one; a dynamically spawned node is never restored.
149 pub restart_count: u32,
150
151 /// When the node exited without being restarted. `None` unless it has exited.
152 pub exited_at: Option<UnixMillis>,
153
154 /// The resource group the node's allocations are attributed to. `None` if the node has never run.
155 ///
156 /// For a supervisor this is its own group. For a worker it is its supervisor's, since a worker inherits its
157 /// supervisor's group rather than owning one.
158 pub resource_group: Option<String>,
159
160 /// Resource usage attributed to this node.
161 ///
162 /// Populated for a supervisor, which owns a resource group covering itself and its workers. Always absent for a
163 /// worker, whose usage is counted against the supervisor named by `resource_group`.
164 #[serde(default, skip_serializing_if = "Option::is_none")]
165 pub resources: Option<ResourceUsage>,
166
167 /// How the node supervises its children. Absent for a worker.
168 #[serde(default, skip_serializing_if = "Option::is_none")]
169 pub supervision: Option<SupervisionSettings>,
170
171 /// The node's children. Empty for a worker.
172 pub children: Vec<NodeSnapshot>,
173}
174
175/// Cumulative resource usage for one resource group.
176///
177/// Counts are since the process started. Both allocation counts and CPU time depend on process-wide facilities that
178/// may not be available: see [`TreeSnapshot::resource_tracking_enabled`].
179#[derive(Clone, Debug, Default, Deserialize, Serialize)]
180pub struct ResourceUsage {
181 /// Bytes allocated.
182 pub allocated_bytes: u64,
183
184 /// Objects allocated.
185 pub allocated_objects: u64,
186
187 /// Bytes deallocated.
188 pub deallocated_bytes: u64,
189
190 /// Objects deallocated.
191 pub deallocated_objects: u64,
192
193 /// Bytes allocated and not yet deallocated.
194 pub live_bytes: u64,
195
196 /// Objects allocated and not yet deallocated.
197 pub live_objects: u64,
198
199 /// CPU time consumed, in nanoseconds.
200 ///
201 /// Always zero where per-thread CPU time is unavailable.
202 pub cpu_time_nanos: u64,
203}
204
205impl From<&ResourceStatsSnapshot> for ResourceUsage {
206 fn from(stats: &ResourceStatsSnapshot) -> Self {
207 Self {
208 allocated_bytes: stats.allocated_bytes as u64,
209 allocated_objects: stats.allocated_objects as u64,
210 deallocated_bytes: stats.deallocated_bytes as u64,
211 deallocated_objects: stats.deallocated_objects as u64,
212 live_bytes: stats.live_bytes() as u64,
213 live_objects: stats.live_objects() as u64,
214 cpu_time_nanos: stats.cpu_time_nanos,
215 }
216 }
217}
218
219/// How a supervisor supervises its children, and how it has fared.
220#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
221pub struct SupervisionSettings {
222 /// Whether a failing child is restarted alone or together with its siblings.
223 pub restart_mode: RestartMode,
224
225 /// How many restarts the supervisor tolerates within `restart_period_ms` before giving up.
226 pub restart_intensity: usize,
227
228 /// The window over which `restart_intensity` is measured, in milliseconds.
229 pub restart_period_ms: u64,
230
231 /// Whether the termination of a significant child drives the supervisor to shut down.
232 pub auto_shutdown: AutoShutdown,
233
234 /// How long the supervisor allows its children to drain during shutdown, in milliseconds. `None` if unbounded.
235 pub shutdown_budget_ms: Option<u64>,
236
237 /// Worker threads on the supervisor's own runtime. `None` if it runs on its parent's runtime.
238 pub dedicated_threads: Option<usize>,
239
240 /// How many child restarts the supervisor has performed, across all of its own generations.
241 ///
242 /// A group restart counts once here however many children it brought back, which is what distinguishes a
243 /// supervisor restarting its whole group repeatedly from a single child restarting repeatedly.
244 pub restarts_performed: u64,
245
246 /// How many times the supervisor has started running.
247 pub generation: u64,
248}