1pub mod saluki_keys;
12pub mod schema_gen;
13pub mod smoke_test_support;
14
15use std::collections::HashSet;
16use std::path::{Path, PathBuf};
17
18use indexmap::{IndexMap, IndexSet};
19use serde::Deserialize;
20
21use crate::smoke_test_support::ConfigurationStruct;
22
23#[derive(Debug, Clone, Deserialize)]
28pub struct SchemaOverlay {
29 pub inventory: IndexMap<String, KnownEntry>,
30 pub excluded: IndexMap<String, String>,
31}
32
33#[derive(Debug, Clone, Deserialize)]
35#[serde(tag = "support", rename_all = "snake_case")]
36pub enum KnownEntry {
37 Full(FullSupport),
39 Partial(PartialSupport),
41 #[serde(rename = "none")]
43 Unsupported(Unsupported),
44 Unknown(UnknownSupport),
46}
47
48#[derive(Debug, Clone, Deserialize)]
50#[serde(rename_all = "snake_case")]
51pub struct FullSupport {
52 pub pipelines: PipelineAffinity,
54 pub description: String,
56 #[serde(default)]
58 pub documentation: Option<String>,
59 #[serde(default)]
61 pub issue: Option<String>,
62 pub test_support: TestSupport,
64}
65
66#[derive(Debug, Clone, Deserialize)]
68#[serde(rename_all = "snake_case")]
69pub struct PartialSupport {
70 pub pipelines: PipelineAffinity,
72 pub description: String,
74 pub documentation: String,
76 #[serde(default)]
78 pub warn: bool,
79 #[serde(default)]
81 pub issue: Option<String>,
82 pub test_support: TestSupport,
84}
85
86#[derive(Debug, Clone, Deserialize)]
88#[serde(rename_all = "snake_case")]
89pub struct Unsupported {
90 pub pipelines: PipelineAffinity,
92 pub description: String,
94 #[serde(default)]
96 pub documentation: Option<String>,
97 pub severity: Severity,
99 pub planned: bool,
101 #[serde(default)]
103 pub issue: Option<String>,
104}
105
106#[derive(Debug, Clone, Deserialize)]
108#[serde(rename_all = "snake_case")]
109pub struct UnknownSupport {
110 #[serde(default)]
112 pub description: Option<String>,
113 #[serde(default)]
115 pub severity: Option<Severity>,
116 #[serde(default)]
118 pub issue: Option<String>,
119}
120
121#[derive(Debug, Clone, Deserialize)]
126#[serde(rename_all = "snake_case")]
127pub struct TestSupport {
128 #[serde(default)]
130 pub env_var_override: Option<Vec<String>>,
131 #[serde(default)]
133 pub additional_yaml_paths: Vec<String>,
134 #[serde(default)]
136 pub value_type_override: Option<ValueType>,
137 pub used_by: IndexSet<ConfigurationStruct>,
139 #[serde(default)]
141 pub test_json: Option<String>,
142 #[serde(default)]
145 pub additional_attributes: IndexMap<String, String>,
146}
147
148#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
150#[serde(rename_all = "snake_case")]
151pub enum Severity {
152 Low,
153 Medium,
154 High,
155}
156
157#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize)]
159#[serde(rename_all = "snake_case")]
160pub enum Pipeline {
161 #[serde(rename = "dogstatsd")]
162 DogStatsD,
163 Checks,
164 Otlp,
165 Traces,
166}
167
168#[derive(Debug, Clone)]
174pub enum PipelineAffinity {
175 CrossCutting,
177 Pipelines(Vec<Pipeline>),
179}
180
181impl<'de> serde::Deserialize<'de> for PipelineAffinity {
182 fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
183 #[derive(Deserialize, PartialEq)]
184 #[serde(rename_all = "snake_case")]
185 enum Token {
186 CrossCutting,
187 #[serde(rename = "dogstatsd")]
188 DogStatsD,
189 Checks,
190 Otlp,
191 Traces,
192 }
193
194 let tokens: Vec<Token> = Vec::deserialize(d)?;
195
196 if tokens.is_empty() {
197 return Err(serde::de::Error::custom("pipelines must be non-empty"));
198 }
199
200 let has_cc = tokens.iter().any(|t| t == &Token::CrossCutting);
201
202 if has_cc && tokens.len() > 1 {
203 return Err(serde::de::Error::custom(
204 "cross_cutting must appear alone in pipelines list",
205 ));
206 }
207
208 if has_cc {
209 return Ok(PipelineAffinity::CrossCutting);
210 }
211
212 let pipelines = tokens
213 .into_iter()
214 .map(|t| match t {
215 Token::DogStatsD => Pipeline::DogStatsD,
216 Token::Checks => Pipeline::Checks,
217 Token::Otlp => Pipeline::Otlp,
218 Token::Traces => Pipeline::Traces,
219 Token::CrossCutting => unreachable!(),
220 })
221 .collect();
222
223 Ok(PipelineAffinity::Pipelines(pipelines))
224 }
225}
226
227#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
229#[serde(rename_all = "snake_case")]
230pub enum ValueType {
231 Boolean,
232 Integer,
233 Float,
234 String,
235 StringList,
236}
237
238pub struct Files {
242 pub schema: PathBuf,
243 pub overlay: PathBuf,
244}
245
246impl Default for Files {
247 fn default() -> Self {
248 let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
249 .join("..")
250 .join("config")
251 .join("schema");
252 let schema = dir.join("core").join("core_schema.yaml");
253 let overlay = dir.join("schema_overlay.yaml");
254 Files { schema, overlay }
255 }
256}
257
258impl SchemaOverlay {
259 pub fn load(files: Files) -> Result<Self, Error> {
260 let loaded = Self::from_file(&files.overlay)?;
261 loaded.validate(&files.schema)?;
262 Ok(loaded)
263 }
264
265 fn from_yaml(s: &str) -> Result<Self, Error> {
266 let yaml: serde_yaml::Value = serde_yaml::from_str(s).map_err(Error::Yaml)?;
267 Self::lint_yaml(&yaml)?;
268 serde_yaml::from_value(yaml).map_err(Error::Yaml)
269 }
270
271 fn from_file(path: &Path) -> Result<Self, Error> {
272 let contents = std::fs::read_to_string(path).map_err(|e| Error::Io((path.into(), e)))?;
273 Self::from_yaml(&contents)
274 }
275
276 fn validate(&self, core_schema: &Path) -> Result<(), Error> {
277 self.validate_keys_match(core_schema)?;
278 self.validate_entries()?;
279 Ok(())
280 }
281
282 fn lint_yaml(yaml: &serde_yaml::Value) -> Result<(), Error> {
285 let mapping = yaml
286 .as_mapping()
287 .ok_or_else(|| Error::Validation("overlay must be a YAML mapping".to_string()))?;
288
289 let section_names: Vec<&str> = mapping.keys().filter_map(|k| k.as_str()).collect();
290
291 for required in ["inventory", "excluded"] {
292 if !section_names.contains(&required) {
293 return Err(Error::Validation(format!(
294 "overlay missing required section '{}'",
295 required
296 )));
297 }
298 }
299
300 let pos_known = section_names.iter().position(|&k| k == "inventory").unwrap();
301 let pos_ignored = section_names.iter().position(|&k| k == "excluded").unwrap();
302
303 if pos_known >= pos_ignored {
304 return Err(Error::Validation(
305 "sections must appear in order: known, ignored".to_string(),
306 ));
307 }
308
309 for section_name in ["inventory", "excluded"] {
310 if let Some(section) = yaml.get(section_name).and_then(|v| v.as_mapping()) {
311 let mut prev = "";
312 for key in section.keys().filter_map(|k| k.as_str()) {
313 if key < prev {
314 return Err(Error::Validation(format!(
315 "{}: key '{}' is out of alphabetical order (after '{}')",
316 section_name, key, prev
317 )));
318 }
319 prev = key;
320 }
321 }
322 }
323
324 Ok(())
325 }
326
327 fn validate_keys_match(&self, core_schema: &Path) -> Result<(), Error> {
329 let schema_keys = Self::schema_keys(core_schema)?;
330
331 for key in self.excluded.keys() {
332 if self.inventory.contains_key(key.as_str()) {
333 return Err(Error::Validation(format!(
334 "key '{}' appears in more than one overlay section",
335 key
336 )));
337 }
338 }
339
340 for key in self.inventory.keys().chain(self.excluded.keys()) {
341 if !schema_keys.contains(key.as_str()) {
342 return Err(Error::Validation(format!(
343 "overlay key '{}' is not present in the schema",
344 key
345 )));
346 }
347 }
348
349 let overlay_keys: HashSet<&str> = self
350 .inventory
351 .keys()
352 .chain(self.excluded.keys())
353 .map(|s| s.as_str())
354 .collect();
355 for key in &schema_keys {
356 if !overlay_keys.contains(key.as_str()) {
357 return Err(Error::Validation(format!(
358 "schema key '{}' is not covered by the overlay",
359 key
360 )));
361 }
362 }
363
364 Ok(())
365 }
366
367 fn schema_keys(schema_path: &Path) -> Result<HashSet<String>, Error> {
368 let schema = load_resolved_schema(schema_path)?;
369 let props = schema
370 .get("properties")
371 .and_then(|v| v.as_mapping())
372 .ok_or_else(|| Error::Validation("schema missing 'properties' section".to_string()))?;
373 let mut keys = HashSet::new();
374 Self::collect_schema_keys(props, "", &mut keys);
375 Ok(keys)
376 }
377
378 fn collect_schema_keys(props: &serde_yaml::Mapping, prefix: &str, keys: &mut HashSet<String>) {
379 for (k, v) in props {
380 if let Some(name) = k.as_str() {
381 let full_key = if prefix.is_empty() {
382 name.to_string()
383 } else {
384 format!("{}.{}", prefix, name)
385 };
386 if let Some(sub_props) = v.get("properties").and_then(|p| p.as_mapping()) {
389 Self::collect_schema_keys(sub_props, &full_key, keys);
390 } else {
391 keys.insert(full_key);
392 }
393 }
394 }
395 }
396
397 fn validate_entries(&self) -> Result<(), Error> {
400 let canonical_keys: HashSet<&str> = self.inventory.keys().map(String::as_str).collect();
401
402 for (key, entry) in &self.inventory {
403 match entry {
404 KnownEntry::Full(f) => {
405 if f.test_support.used_by.is_empty() {
406 return Err(Error::Validation(format!(
407 "full key '{}': used_by must be non-empty",
408 key
409 )));
410 }
411 if f.description.len() > 50 {
412 return Err(Error::Validation(format!(
413 "full key '{}': description exceeds 50 chars ({} chars)",
414 key,
415 f.description.len()
416 )));
417 }
418 Self::validate_additional_yaml_paths(key, &f.test_support.additional_yaml_paths, &canonical_keys)?;
419 }
420 KnownEntry::Partial(p) => {
421 if p.test_support.used_by.is_empty() {
422 return Err(Error::Validation(format!(
423 "partial key '{}': used_by must be non-empty",
424 key
425 )));
426 }
427 if p.description.len() > 50 {
428 return Err(Error::Validation(format!(
429 "partial key '{}': description exceeds 50 chars ({} chars)",
430 key,
431 p.description.len()
432 )));
433 }
434 Self::validate_additional_yaml_paths(key, &p.test_support.additional_yaml_paths, &canonical_keys)?;
435 }
436 KnownEntry::Unsupported(u) => {
437 if u.description.len() > 50 {
438 return Err(Error::Validation(format!(
439 "unsupported key '{}': description exceeds 50 chars ({} chars)",
440 key,
441 u.description.len()
442 )));
443 }
444 if u.planned && u.issue.is_none() {
445 return Err(Error::Validation(format!(
446 "unsupported key '{}': planned requires an issue",
447 key
448 )));
449 }
450 }
451 KnownEntry::Unknown(u) => {
452 if let Some(desc) = &u.description {
453 if desc.len() > 50 {
454 return Err(Error::Validation(format!(
455 "unknown key '{}': description exceeds 50 chars ({} chars)",
456 key,
457 desc.len()
458 )));
459 }
460 }
461 }
462 }
463 }
464 Ok(())
465 }
466
467 fn validate_additional_yaml_paths(
468 key: &str, paths: &[String], canonical_keys: &HashSet<&str>,
469 ) -> Result<(), Error> {
470 let mut seen: HashSet<&str> = HashSet::new();
471 for path in paths {
472 if !seen.insert(path.as_str()) {
473 return Err(Error::Validation(format!(
474 "key '{}': duplicate additional_yaml_path '{}'",
475 key, path
476 )));
477 }
478 if path.contains('.') {
479 return Err(Error::Validation(format!(
480 "key '{}': additional_yaml_path '{}' contains a dot. \
481 Dotted aliases land at a different nesting depth in the YAML tree, \
482 which cannot be represented as a serde field alias on the generated \
483 struct. Supporting dotted aliases requires a post-deserialization \
484 merge step or custom Deserialize impl.",
485 key, path
486 )));
487 }
488 if canonical_keys.contains(path.as_str()) {
489 return Err(Error::Validation(format!(
490 "key '{}': additional_yaml_path '{}' collides with a canonical \
491 overlay key. Two fields would deserialize from the same YAML key.",
492 key, path
493 )));
494 }
495 }
496 Ok(())
497 }
498}
499
500fn read_yaml(path: &Path) -> Result<serde_yaml::Value, Error> {
502 let contents = std::fs::read_to_string(path).map_err(|e| Error::Io((path.into(), e)))?;
503 serde_yaml::from_str(&contents).map_err(Error::Yaml)
504}
505
506pub fn load_resolved_schema(schema_path: &Path) -> Result<serde_yaml::Value, Error> {
518 let schema_dir = schema_path.parent().unwrap_or_else(|| Path::new("."));
519 let mut doc = read_yaml(schema_path)?;
520 resolve_refs(&mut doc, schema_dir)?;
521 Ok(doc)
522}
523
524fn resolve_refs(value: &mut serde_yaml::Value, schema_dir: &Path) -> Result<(), Error> {
527 if let Some(map) = value.as_mapping_mut() {
528 if let Some(ref_path) = map.get("$ref").and_then(|v| v.as_str()) {
529 let ref_file = schema_dir.join(ref_path);
530 let mut ref_doc = read_yaml(&ref_file)?;
531 resolve_refs(&mut ref_doc, schema_dir)?;
532 *value = ref_doc;
533 return Ok(());
534 }
535 for (_k, v) in map.iter_mut() {
536 resolve_refs(v, schema_dir)?;
537 }
538 }
539 Ok(())
540}
541
542const VALIDATION_RULES: &str = "\n\
543 \n\
544 Rules that must hold in schema_overlay.yaml:\n\
545 - Every core_schema.yaml key appears in exactly one section (known / ignored).\n\
546 - No key appears in more than one section.\n\
547 - Sections appear in order: known, ignored.\n\
548 - Keys within each section are sorted alphabetically.\n\
549 - full entries: pipelines non-empty, used_by non-empty, description <= 50 chars.\n\
550 - partial entries: pipelines non-empty, used_by non-empty, description <= 50 chars, documentation required.\n\
551 - unsupported entries: pipelines non-empty, description <= 50 chars, planned+issue consistent.\n\
552 - unknown entries: description <= 50 chars (when present).\n\
553 - additional_yaml_paths: no duplicates within a single entry, no dots, no collisions with canonical keys.\n\
554 Fix: edit lib/datadog-agent/config/schema/schema_overlay.yaml.";
555
556#[derive(Debug)]
558pub enum Error {
559 Io((PathBuf, std::io::Error)),
560 Yaml(serde_yaml::Error),
561 Validation(String),
562}
563
564impl std::fmt::Display for Error {
565 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
566 match self {
567 Error::Io(e) => write!(f, "Error reading {}: {}", e.0.display(), e.1),
568 Error::Yaml(e) => write!(f, "YAML parse error in overlay: {e}"),
569 Error::Validation(s) => write!(f, "schema_overlay.yaml validation failed: {s}{VALIDATION_RULES}"),
570 }
571 }
572}
573
574impl std::error::Error for Error {
575 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
576 match self {
577 Error::Io(e) => Some(&e.1),
578 Error::Yaml(e) => Some(e),
579 Error::Validation(_) => None,
580 }
581 }
582}
583
584#[cfg(test)]
585mod tests {
586 use super::*;
587
588 #[test]
589 fn overlay_loads() {
590 let test_files = Files {
591 schema: Path::new(env!("CARGO_MANIFEST_DIR"))
592 .join("test")
593 .join("fake_schema.yaml"),
594 overlay: Path::new(env!("CARGO_MANIFEST_DIR"))
595 .join("test")
596 .join("fake_overlay.yaml"),
597 };
598 let validated = SchemaOverlay::load(test_files).unwrap();
599 assert_eq!(validated.inventory.len(), 18);
600 }
601
602 #[test]
603 fn pipeline_affinity_cross_cutting() {
604 let yaml = "pipelines: [cross_cutting]";
605 #[derive(Deserialize)]
606 struct W {
607 pipelines: PipelineAffinity,
608 }
609 let w: W = serde_yaml::from_str(yaml).unwrap();
610 assert!(matches!(w.pipelines, PipelineAffinity::CrossCutting));
611 }
612
613 #[test]
614 fn pipeline_affinity_multi() {
615 let yaml = "pipelines: [dogstatsd, traces]";
616 #[derive(Deserialize)]
617 struct W {
618 pipelines: PipelineAffinity,
619 }
620 let w: W = serde_yaml::from_str(yaml).unwrap();
621 if let PipelineAffinity::Pipelines(ps) = w.pipelines {
622 assert_eq!(ps.len(), 2);
623 assert!(matches!(ps[0], Pipeline::DogStatsD));
624 assert!(matches!(ps[1], Pipeline::Traces));
625 } else {
626 panic!("expected Pipelines");
627 }
628 }
629
630 #[test]
631 fn pipeline_affinity_cross_cutting_must_be_alone() {
632 let yaml = "pipelines: [cross_cutting, dogstatsd]";
633 #[derive(Deserialize)]
634 #[allow(dead_code)]
635 struct W {
636 pipelines: PipelineAffinity,
637 }
638 assert!(serde_yaml::from_str::<W>(yaml).is_err());
639 }
640
641 fn load_from_strs(schema: &str, overlay: &str) -> Result<SchemaOverlay, Error> {
642 let dir = tempfile::tempdir().unwrap();
643 let schema_path = dir.path().join("schema.yaml");
644 let overlay_path = dir.path().join("overlay.yaml");
645 std::fs::write(&schema_path, schema).unwrap();
646 std::fs::write(&overlay_path, overlay).unwrap();
647 SchemaOverlay::load(Files {
648 schema: schema_path,
649 overlay: overlay_path,
650 })
651 }
652
653 #[test]
654 fn validation_rejects_schema_key_missing_from_overlay() {
655 let schema = "\
656properties:
657 key_a:
658 type: string
659 key_b:
660 type: string
661";
662 let overlay = "\
663inventory:
664 key_a:
665 support: full
666 pipelines: [cross_cutting]
667 description: \"Key A\"
668 test_support:
669 used_by: [ForwarderConfiguration]
670excluded: {}
671";
672 let err = load_from_strs(schema, overlay).unwrap_err();
673 assert!(
674 err.to_string().contains("schema key 'key_b' is not covered"),
675 "unexpected error: {err}"
676 );
677 }
678
679 #[test]
680 fn validation_rejects_overlay_key_absent_from_schema() {
681 let schema = "\
682properties:
683 key_a:
684 type: string
685";
686 let overlay = "\
687inventory:
688 key_a:
689 support: full
690 pipelines: [cross_cutting]
691 description: \"Key A\"
692 test_support:
693 used_by: [ForwarderConfiguration]
694excluded:
695 key_b: \"not in schema\"
696";
697 let err = load_from_strs(schema, overlay).unwrap_err();
698 assert!(
699 err.to_string().contains("overlay key 'key_b' is not present"),
700 "unexpected error: {err}"
701 );
702 }
703
704 #[test]
705 fn validation_rejects_key_in_two_sections() {
706 let schema = "\
707properties:
708 key_a:
709 type: string
710 key_b:
711 type: string
712";
713 let overlay = "\
714inventory:
715 key_a:
716 support: full
717 pipelines: [cross_cutting]
718 description: \"Key A\"
719 test_support:
720 used_by: [ForwarderConfiguration]
721excluded:
722 key_a: \"duplicate\"
723 key_b: \"ok\"
724";
725 let err = load_from_strs(schema, overlay).unwrap_err();
726 assert!(
727 err.to_string()
728 .contains("key 'key_a' appears in more than one overlay section"),
729 "unexpected error: {err}"
730 );
731 }
732
733 #[test]
735 fn schema_ref_is_resolved_and_keys_namespaced() {
736 let dir = tempfile::tempdir().unwrap();
737 std::fs::write(
738 dir.path().join("sub.yaml"),
739 "properties:\n enabled:\n type: boolean\n",
740 )
741 .unwrap();
742 let schema_path = dir.path().join("schema.yaml");
743 std::fs::write(&schema_path, "properties:\n feature:\n $ref: sub.yaml\n").unwrap();
744
745 let keys = SchemaOverlay::schema_keys(&schema_path).unwrap();
746 assert_eq!(
747 keys,
748 HashSet::from(["feature.enabled".to_string()]),
749 "unexpected keys: {keys:?}"
750 );
751 }
752
753 #[test]
756 fn missing_schema_ref_reports_io_error() {
757 let dir = tempfile::tempdir().unwrap();
758 let schema_path = dir.path().join("schema.yaml");
759 std::fs::write(&schema_path, "properties:\n feature:\n $ref: does_not_exist.yaml\n").unwrap();
760
761 let err = SchemaOverlay::schema_keys(&schema_path).unwrap_err();
762 assert!(matches!(err, Error::Io(_)), "expected Io error, got: {err}");
763 assert!(
764 err.to_string().contains("does_not_exist.yaml"),
765 "error should name the missing file: {err}"
766 );
767 }
768
769 #[test]
770 fn validation_rejects_unsorted_inventory_keys() {
771 let schema = "\
772properties:
773 key_a:
774 type: string
775 key_b:
776 type: string
777";
778 let overlay = "\
779inventory:
780 key_b:
781 support: full
782 pipelines: [cross_cutting]
783 description: \"Key B\"
784 test_support:
785 used_by: [ForwarderConfiguration]
786 key_a:
787 support: full
788 pipelines: [cross_cutting]
789 description: \"Key A\"
790 test_support:
791 used_by: [ForwarderConfiguration]
792excluded: {}
793";
794 let err = load_from_strs(schema, overlay).unwrap_err();
795 assert!(
796 err.to_string().contains("out of alphabetical order"),
797 "unexpected error: {err}"
798 );
799 }
800}