datadog_agent_apm/semantics/
registry.rs

1//! A `Registry` is the parsed, validated in-memory form of one complete semantic-mappings document for traces.
2//!
3//! The document comes from either:
4//!
5//! - the embedded `mappings.json`; or
6//! - one `APM_SEMANTIC_CORE_DD` Remote Configuration file.
7//!
8//! A `Registry` contains:
9//!
10//! - `version` and `metadata.content_hash` labels;
11//! - a local fingerprint of the original JSON bytes; and
12//! - a map from each supported `Concept` to its ordered fallback attributes.
13
14use std::{
15    borrow::Cow,
16    fmt,
17    sync::{Arc, LazyLock},
18};
19
20use saluki_common::collections::FastHashMap;
21use saluki_error::{generic_error, GenericError};
22use serde::{Deserialize, Deserializer};
23use serde_json::Value;
24
25use super::Concept;
26
27/// Provenance of an attribute convention.
28#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
29#[serde(rename_all = "lowercase")]
30pub enum Provider {
31    /// Datadog attribute conventions.
32    Datadog,
33    /// OpenTelemetry attribute conventions.
34    Otel,
35}
36
37/// The expected type of an attribute's value at lookup time.
38///
39/// The lookup is type-strict: an attribute registered as `Int64` is only read
40/// via the accessor's `get_int64` method. String-typed registrations allow the
41/// lookup functions to parse the string into the requested numeric type.
42#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
43#[serde(rename_all = "lowercase")]
44pub enum ValueType {
45    /// String-valued attributes.
46    String,
47    /// Signed integer attributes.
48    Int64,
49    /// Floating-point attributes.
50    Float64,
51}
52
53/// A predicate that must match before a fallback tag can be used.
54///
55/// A condition reads an attribute by exact key—no fallback chaining—and
56/// holds up to two predicate fields, all of which must hold (logical AND). A
57/// condition with no predicates set always matches.
58#[derive(Debug, Clone, Default, Deserialize)]
59pub struct Condition {
60    /// Exact attribute name to test, without looking up other names for the same concept.
61    ///
62    /// An absent or `null` JSON field becomes an empty name, matching upstream decoding.
63    #[serde(default, deserialize_with = "null_as_default")]
64    pub attribute: String,
65    /// When set, requires the attribute's presence (or absence) to match.
66    #[serde(default)]
67    pub present: Option<bool>,
68    /// When set, requires the attribute's string value to equal this.
69    #[serde(default)]
70    pub eq: Option<String>,
71}
72
73/// One entry in a concept's fallback precedence list.
74#[derive(Debug, Clone, Deserialize)]
75pub struct TagInfo {
76    /// Attribute name to read. An absent or `null` JSON field becomes an empty name.
77    #[serde(default, deserialize_with = "null_as_default")]
78    pub name: String,
79    /// The source of this attribute convention.
80    pub provider: Provider,
81    /// The convention version, or an empty string when unspecified.
82    #[serde(default, deserialize_with = "null_as_default")]
83    pub version: String,
84    /// The stored attribute type.
85    #[serde(rename = "type")]
86    pub value_type: ValueType,
87    /// Conditions that must all match before this attribute can supply the concept's value.
88    ///
89    /// An absent or `null` list is empty; a `null` element has no predicates and always matches.
90    #[serde(default, deserialize_with = "null_elements_as_default")]
91    pub when: Vec<Condition>,
92}
93
94/// Treats JSON `null` as the Rust type's default, matching upstream decoding of zero-valued fields.
95fn null_as_default<'de, D, T>(deserializer: D) -> Result<T, D::Error>
96where
97    D: Deserializer<'de>,
98    T: Deserialize<'de> + Default,
99{
100    Option::<T>::deserialize(deserializer).map(Option::unwrap_or_default)
101}
102
103/// Treats a JSON `null` list as empty and `null` elements as defaults, matching upstream decoding.
104fn null_elements_as_default<'de, D, T>(deserializer: D) -> Result<Vec<T>, D::Error>
105where
106    D: Deserializer<'de>,
107    T: Deserialize<'de> + Default,
108{
109    let elements = Option::<Vec<Option<T>>>::deserialize(deserializer)?;
110    Ok(elements.into_iter().flatten().map(Option::unwrap_or_default).collect())
111}
112
113/// Maps semantic concepts to ordered attribute names, types, and conditions for lookups.
114///
115/// For example, the HTTP status-code concept can read either `http.response.status_code` or `http.status_code`.
116/// A registry holds one complete set of mappings; remote registries replace rather than extend the embedded set.
117#[derive(Clone)]
118pub struct Registry {
119    version: String,
120    content_hash: String,
121    fingerprint: u64,
122    mappings: FastHashMap<Concept, Vec<TagInfo>>,
123}
124
125impl Registry {
126    /// Parses a registry JSON string, rejecting malformed or unsupported entries.
127    ///
128    /// # Errors
129    ///
130    /// Uses the same validation as [`Self::from_slice`]. Errors may quote payload keys and must not be sent to the
131    /// remote endpoint.
132    pub fn from_json(json: &str) -> Result<Self, GenericError> {
133        Self::from_slice(json.as_bytes())
134    }
135
136    /// Parses registry JSON, ignoring unknown concepts but validating mappings for known ones.
137    ///
138    /// Use this for embedded mappings: invalid mappings for known concepts indicate a build defect. Concepts without a
139    /// [`Concept`] variant are skipped without validation because nothing can look them up. This allows upstream to add
140    /// concepts without a code change. Remote Configuration decoding also skips invalid mappings and fallbacks.
141    ///
142    /// # Errors
143    ///
144    /// Returns an error for invalid JSON, a missing or empty `concepts` object, or a missing or empty
145    /// `metadata.content_hash` string. For known concepts, also rejects malformed mappings and fallbacks with invalid
146    /// fields, including unknown providers or value types. Errors may quote payload keys; do not send them to the
147    /// remote endpoint.
148    pub fn from_slice(json: &[u8]) -> Result<Self, GenericError> {
149        let (registry, mut skipped) = Self::from_slice_permissive(json)?;
150        skipped.unknown_concepts.clear();
151        if !skipped.is_empty() {
152            return Err(generic_error!(
153                "Registry JSON has parts the registry cannot represent: {}",
154                skipped
155            ));
156        }
157        Ok(registry)
158    }
159
160    /// Parses remote registry JSON while retaining the entries this binary understands.
161    ///
162    /// Remote mappings may contain concepts or conventions added after this binary was built. Unknown concepts,
163    /// malformed mappings, and fallbacks with invalid fields (including unknown providers or value types) are skipped
164    /// and recorded in [`SkipReport`]. The result is accepted even if every entry was skipped; missing mappings are
165    /// not filled from the embedded registry. The report is for local logging only.
166    ///
167    /// A `null` mapping or absent or `null` `fallbacks` gives an empty fallback list. See [`TagInfo`] and [`Condition`]
168    /// for field defaults, and [`Registry::version`] for version handling.
169    ///
170    /// The [fingerprint](Registry::fingerprint) includes skipped entries.
171    ///
172    /// # Errors
173    ///
174    /// Returns an error for invalid JSON, a missing or empty `concepts` object, or a missing or empty
175    /// `metadata.content_hash` string. These requirements apply before skipping entries. Errors never quote the
176    /// payload, so they can be reported to the remote endpoint.
177    pub(crate) fn from_slice_permissive(json: &[u8]) -> Result<(Self, SkipReport), GenericError> {
178        let document: Value = serde_json::from_slice(json).map_err(malformed_error)?;
179        let concepts = document.get("concepts").and_then(Value::as_object);
180        let content_hash = document
181            .pointer("/metadata/content_hash")
182            .and_then(Value::as_str)
183            .unwrap_or_default();
184        check_required(concepts.is_some_and(|c| !c.is_empty()), content_hash)?;
185
186        let mut skipped = SkipReport::default();
187        let mut mappings = FastHashMap::default();
188        for (key, mapping) in concepts.into_iter().flatten() {
189            let Some(concept) = Concept::from_name(key) else {
190                skipped.unknown_concepts.push(key.clone());
191                continue;
192            };
193            let entries: &[Value] = match mapping {
194                // Go decodes a `null` mapping to an empty fallback list.
195                Value::Null => &[],
196                Value::Object(map) => match map.get("fallbacks") {
197                    None | Some(Value::Null) => &[],
198                    Some(Value::Array(entries)) => entries.as_slice(),
199                    _ => {
200                        skipped.malformed_concepts.push(key.clone());
201                        continue;
202                    }
203                },
204                _ => {
205                    skipped.malformed_concepts.push(key.clone());
206                    continue;
207                }
208            };
209            let mut fallbacks = Vec::with_capacity(entries.len());
210            for (position, entry) in entries.iter().enumerate() {
211                match TagInfo::deserialize(entry) {
212                    Ok(tag) => fallbacks.push(tag),
213                    Err(_) => skipped.fallbacks.push((key.clone(), position)),
214                }
215            }
216            mappings.insert(concept, fallbacks);
217        }
218
219        let registry = Self {
220            version: document
221                .get("version")
222                .and_then(Value::as_str)
223                .unwrap_or_default()
224                .to_owned(),
225            content_hash: content_hash.to_owned(),
226            fingerprint: fingerprint_for(json),
227            mappings,
228        };
229        Ok((registry, skipped))
230    }
231
232    /// Returns the registry's `metadata.content_hash` label for logs and status.
233    ///
234    /// This value is not computed or verified here; different mappings can carry the same label. Use
235    /// [`Self::fingerprint`] to detect changes.
236    pub fn content_hash(&self) -> &str {
237        &self.content_hash
238    }
239
240    /// Returns a non-cryptographic hash of the original JSON bytes for change detection within this process.
241    ///
242    /// Compare fingerprints to decide whether to rebuild derived data, such as peer tag keys for APM stats.
243    /// Formatting, metadata, and skipped entries also affect the hash. A collision can leave derived data stale;
244    /// this is not an integrity check.
245    pub fn fingerprint(&self) -> u64 {
246        self.fingerprint
247    }
248
249    /// Returns the ordered list of attribute fallbacks for a concept, or
250    /// `None` if the concept has no registered mappings.
251    pub fn get_attribute_precedence(&self, concept: Concept) -> Option<&[TagInfo]> {
252        self.mappings.get(&concept).map(Vec::as_slice)
253    }
254
255    /// Returns the document's `version` label, or `""` if absent, `null`, or not a string.
256    ///
257    /// This label is informational; it does not control parsing or update selection.
258    pub fn version(&self) -> &str {
259        &self.version
260    }
261}
262
263/// Records the concept keys and fallback positions skipped by [`Registry::from_slice_permissive`].
264///
265/// Its [`Display`](fmt::Display) reports counts and a bounded list of escaped, shortened keys.
266/// It quotes payload keys: log it locally, but never report it to the remote endpoint.
267#[derive(Debug, Default, PartialEq, Eq)]
268pub(crate) struct SkipReport {
269    /// Concept keys that don't correspond to a known [`Concept`].
270    unknown_concepts: Vec<String>,
271
272    /// Keys of concepts with malformed mappings or `fallbacks`.
273    malformed_concepts: Vec<String>,
274
275    /// Unsupported fallbacks, as (concept key, zero-based position).
276    fallbacks: Vec<(String, usize)>,
277}
278
279impl SkipReport {
280    /// Returns `true` if nothing was skipped.
281    pub(crate) fn is_empty(&self) -> bool {
282        self.unknown_concepts.is_empty() && self.malformed_concepts.is_empty() && self.fallbacks.is_empty()
283    }
284}
285
286impl fmt::Display for SkipReport {
287    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
288        if self.is_empty() {
289            return f.write_str("nothing skipped");
290        }
291        let fallbacks: Vec<String> = self
292            .fallbacks
293            .iter()
294            .take(MAX_LISTED_KEYS)
295            .map(|(key, position)| format!("{:?}[{position}]", shorten_key(key)))
296            .collect();
297        let quoted = |keys: &[String]| -> Vec<String> {
298            keys.iter()
299                .take(MAX_LISTED_KEYS)
300                .map(|key| format!("{:?}", shorten_key(key)))
301                .collect()
302        };
303        let sections: [(&str, usize, Vec<String>); 3] = [
304            (
305                "unknown concepts",
306                self.unknown_concepts.len(),
307                quoted(&self.unknown_concepts),
308            ),
309            (
310                "malformed concepts",
311                self.malformed_concepts.len(),
312                quoted(&self.malformed_concepts),
313            ),
314            ("unrepresentable fallbacks", self.fallbacks.len(), fallbacks),
315        ];
316        let mut separator = "";
317        for (label, count, listed) in sections.into_iter().filter(|(_, count, _)| *count > 0) {
318            let more = if count > listed.len() { ", ..." } else { "" };
319            write!(f, "{separator}{label}: {count} ({}{more})", listed.join(", "))?;
320            separator = "; ";
321        }
322        Ok(())
323    }
324}
325
326/// Caps the keys listed in each skip-report section.
327const MAX_LISTED_KEYS: usize = 10;
328
329/// Caps the characters shown from each key in a skip report.
330const MAX_KEY_CHARS: usize = 64;
331
332fn shorten_key(key: &str) -> Cow<'_, str> {
333    if key.chars().count() <= MAX_KEY_CHARS {
334        return Cow::Borrowed(key);
335    }
336    let mut shortened: String = key.chars().take(MAX_KEY_CHARS).collect();
337    shortened.push('…');
338    Cow::Owned(shortened)
339}
340
341/// Reports a JSON error by position without quoting the payload.
342fn malformed_error(e: serde_json::Error) -> GenericError {
343    generic_error!(
344        "Registry JSON is malformed at line {}, column {}.",
345        e.line(),
346        e.column()
347    )
348}
349
350/// Rejects documents without concepts or a content-hash label, matching upstream registry validation.
351fn check_required(has_concepts: bool, content_hash: &str) -> Result<(), GenericError> {
352    if !has_concepts {
353        return Err(generic_error!("Registry JSON contains no concepts."));
354    }
355    if content_hash.is_empty() {
356        return Err(generic_error!("Registry JSON is missing `metadata.content_hash`."));
357    }
358    Ok(())
359}
360
361// Do not edit `mappings.json` by hand: `make update-apm-semantics` overwrites it from upstream and records the source
362// commit in `_version.txt`.
363const MAPPINGS_JSON: &str = include_str!("mappings.json");
364
365/// Hashes a raw registry document for change detection.
366fn fingerprint_for(json: &[u8]) -> u64 {
367    use std::hash::Hasher as _;
368
369    let mut hasher = std::collections::hash_map::DefaultHasher::new();
370    hasher.write(json);
371    hasher.finish()
372}
373
374/// The default semantic mappings compiled into the binary from `mappings.json`.
375///
376/// Parsed once on first access. Components use [`SemanticRegistryProvider`](super::SemanticRegistryProvider) rather
377/// than reading this static directly, so they can use remote mappings when available.
378///
379/// # Panics
380///
381/// Panics on first access if the embedded mappings are invalid or unsupported, which indicates a build defect.
382pub static EMBEDDED_REGISTRY: LazyLock<Arc<Registry>> = LazyLock::new(|| {
383    Arc::new(Registry::from_json(MAPPINGS_JSON).expect("embedded semantic mappings.json failed to load"))
384});
385
386/// Wraps `concepts` in a registry document with the required metadata.
387#[cfg(any(test, feature = "test-util"))]
388pub fn registry_json(concepts: &str) -> String {
389    format!(r#"{{"version":"test","metadata":{{"content_hash":"test"}},"concepts":{concepts}}}"#)
390}
391
392#[cfg(test)]
393mod tests {
394    use super::*;
395
396    #[test]
397    fn from_json_accepts_when_conditions() {
398        let with_when = registry_json(
399            r#"{
400                "http.status_code": {
401                    "canonical": "http.status_code",
402                    "fallbacks": [
403                        {"name": "rpc.response.status_code", "provider": "otel", "type": "int64",
404                         "when": [{"attribute": "rpc.system", "eq": "grpc"}]}
405                    ]
406                }
407            }"#,
408        );
409        let registry = Registry::from_json(&with_when).expect("registry with a `when` clause should parse");
410        let tags = registry
411            .get_attribute_precedence(Concept::HttpStatusCode)
412            .expect("concept missing");
413        assert_eq!(tags.len(), 1);
414        assert_eq!(tags[0].when.len(), 1);
415        assert_eq!(tags[0].when[0].attribute, "rpc.system");
416        assert_eq!(tags[0].when[0].eq.as_deref(), Some("grpc"));
417        assert_eq!(tags[0].when[0].present, None);
418    }
419
420    #[test]
421    fn embedded_mappings_use_when_for_grpc_fallback() {
422        // The `rpc.response.status_code` fallbacks are gated on the span being a
423        // gRPC span; the registry must preserve those conditions.
424        let registry = &EMBEDDED_REGISTRY;
425        let tags = registry
426            .get_attribute_precedence(Concept::RpcGrpcStatusCode)
427            .expect("rpc.grpc.status_code concept missing");
428        let gated = tags.iter().filter(|t| !t.when.is_empty()).count();
429        assert!(gated >= 4, "expected at least four when-gated fallbacks, found {gated}");
430    }
431
432    #[test]
433    fn every_concept_variant_is_registered() {
434        for concept in Concept::ALL {
435            assert!(
436                EMBEDDED_REGISTRY.get_attribute_precedence(*concept).is_some(),
437                "concept {:?} (\"{}\") has no entry in mappings.json",
438                concept,
439                concept.as_str(),
440            );
441        }
442    }
443
444    #[test]
445    fn http_status_code_has_int_and_string_fallbacks() {
446        // Guards against regressions of the exact bug this module was written for.
447        let registry = &EMBEDDED_REGISTRY;
448        let tags = registry
449            .get_attribute_precedence(Concept::HttpStatusCode)
450            .expect("http.status_code concept missing");
451
452        let has_int_status = tags
453            .iter()
454            .any(|t| t.name == "http.status_code" && t.value_type == ValueType::Int64);
455        let has_str_status = tags
456            .iter()
457            .any(|t| t.name == "http.status_code" && t.value_type == ValueType::String);
458        let has_int_response = tags
459            .iter()
460            .any(|t| t.name == "http.response.status_code" && t.value_type == ValueType::Int64);
461        let has_str_response = tags
462            .iter()
463            .any(|t| t.name == "http.response.status_code" && t.value_type == ValueType::String);
464
465        assert!(has_int_status && has_str_status && has_int_response && has_str_response);
466    }
467
468    #[test]
469    fn from_json_ignores_unknown_concepts() {
470        let json = registry_json(
471            r#"{
472                "this.is.not.a.real.concept": {"canonical": "x", "fallbacks": []},
473                "db.statement": {"fallbacks": [{"name": "db.statement", "provider": "datadog", "type": "string"}]}
474            }"#,
475        );
476        let registry = Registry::from_json(&json).expect("unknown concepts should be ignored");
477        assert_eq!(registry.mappings.len(), 1);
478        assert_eq!(
479            registry.get_attribute_precedence(Concept::DbStatement).map(<[_]>::len),
480            Some(1)
481        );
482    }
483
484    // Additive upstream updates need no Rust changes: unknown concepts are skipped, and supported fallbacks are kept.
485    #[test]
486    fn from_slice_loads_the_embedded_mappings_extended_with_additions() {
487        let mut document: Value = serde_json::from_str(MAPPINGS_JSON).expect("embedded mappings should be JSON");
488        let concepts = document["concepts"]
489            .as_object_mut()
490            .expect("concepts should be an object");
491        concepts.insert(
492            "future.concept".to_owned(),
493            serde_json::json!({
494                "canonical": "future.concept",
495                "fallbacks": [{"name": "future.concept", "provider": "otel", "type": "string"}]
496            }),
497        );
498        let added_fallback = serde_json::json!({"name": "added.http.method", "provider": "otel", "type": "string"});
499        concepts["http.method"]["fallbacks"]
500            .as_array_mut()
501            .expect("http.method should have fallbacks")
502            .push(added_fallback);
503        let extended = document.to_string();
504
505        let registry = Registry::from_slice(extended.as_bytes()).expect("additions should not break loading");
506        for concept in Concept::ALL {
507            assert!(
508                registry.get_attribute_precedence(*concept).is_some(),
509                "concept {concept:?} should be present"
510            );
511        }
512        let added = registry
513            .get_attribute_precedence(Concept::HttpMethod)
514            .expect("http.method should be present")
515            .iter()
516            .any(|tag| tag.name == "added.http.method");
517        assert!(added, "an added fallback for a known concept should be kept");
518
519        let (_, skipped) = Registry::from_slice_permissive(extended.as_bytes()).expect("payload should load");
520        assert!(skipped.unknown_concepts.contains(&"future.concept".to_owned()));
521        assert!(skipped.malformed_concepts.is_empty() && skipped.fallbacks.is_empty());
522    }
523
524    #[test]
525    fn from_slice_rejects_an_unusable_fallback_in_the_extended_embedded_mappings() {
526        let mut document: Value = serde_json::from_str(MAPPINGS_JSON).expect("embedded mappings should be JSON");
527        document["concepts"]["http.method"]["fallbacks"]
528            .as_array_mut()
529            .expect("http.method should have fallbacks")
530            .push(serde_json::json!({"name": "future.provider", "provider": "vendor", "type": "string"}));
531        document["concepts"]["future.concept"] = serde_json::json!({"fallbacks": []});
532
533        let error = Registry::from_slice(document.to_string().as_bytes())
534            .err()
535            .expect("an unrepresentable fallback should be rejected");
536        assert!(
537            error
538                .to_string()
539                .starts_with("Registry JSON has parts the registry cannot represent: unrepresentable fallbacks: 1"),
540            "{error}"
541        );
542    }
543
544    fn parse_strict(json: &str) -> Result<Registry, GenericError> {
545        Registry::from_json(json)
546    }
547
548    fn parse_permissive(json: &str) -> Result<Registry, GenericError> {
549        Registry::from_slice_permissive(json.as_bytes()).map(|(registry, _)| registry)
550    }
551
552    type Parser = fn(&str) -> Result<Registry, GenericError>;
553
554    const PARSERS: [(&str, Parser); 2] = [("strict", parse_strict), ("permissive", parse_permissive)];
555
556    // Ported from upstream `TestNewRegistryFromJSON_MalformedJSON`:
557    // https://github.com/DataDog/datadog-agent/blob/17ecddf4e3e83ccbb0e68aeb99461d1a1d902927/pkg/trace/semantics/registry_update_test.go#L27
558    #[test]
559    fn parsers_reject_malformed_input_by_position() {
560        for (name, parse) in PARSERS {
561            let error = parse("not json").err().expect("malformed input should be rejected");
562            assert_eq!(
563                error.to_string(),
564                "Registry JSON is malformed at line 1, column 2.",
565                "{name}"
566            );
567        }
568    }
569
570    #[test]
571    fn from_json_rejects_unusable_parts_the_permissive_parser_skips() {
572        let json = registry_json(
573            r#"{
574                "http.status_code": {"fallbacks": [{"name": "http.status_code", "provider": "X", "type": "string"}]},
575                "http.method": "not an object"
576            }"#,
577        );
578        let error = Registry::from_json(&json)
579            .err()
580            .expect("unrepresentable parts should be rejected");
581        assert_eq!(
582            error.to_string(),
583            r#"Registry JSON has parts the registry cannot represent: malformed concepts: 1 ("http.method"); "#
584                .to_owned()
585                + r#"unrepresentable fallbacks: 1 ("http.status_code"[0])"#
586        );
587    }
588
589    #[test]
590    fn from_json_rejects_absent_concepts_key_as_empty() {
591        // Upstream treats an absent `concepts` key as an empty map.
592        let json = r#"{"version":"0.1.0","metadata":{"content_hash":"hash-a"}}"#;
593        let error = Registry::from_json(json)
594            .err()
595            .expect("absent concepts key should be rejected");
596        assert_eq!(error.to_string(), "Registry JSON contains no concepts.");
597    }
598
599    // Ported from upstream `TestNewRegistryFromJSON_ValidJSON`:
600    // https://github.com/DataDog/datadog-agent/blob/17ecddf4e3e83ccbb0e68aeb99461d1a1d902927/pkg/trace/semantics/registry_update_test.go#L17
601    #[test]
602    fn from_slice_parses_the_embedded_mappings() {
603        let registry = Registry::from_slice(MAPPINGS_JSON.as_bytes()).expect("embedded mappings should parse");
604        for concept in Concept::ALL {
605            assert!(
606                registry.get_attribute_precedence(*concept).is_some(),
607                "concept {concept:?} should be present"
608            );
609        }
610    }
611
612    // Ported from upstream `TestNewRegistryFromJSON_EmptyConcepts`:
613    // https://github.com/DataDog/datadog-agent/blob/17ecddf4e3e83ccbb0e68aeb99461d1a1d902927/pkg/trace/semantics/registry_update_test.go#L32
614    #[test]
615    fn parsers_reject_empty_concepts() {
616        let json = r#"{"version":"0.1.0","metadata":{"content_hash":"hash-a"},"concepts":{}}"#;
617        for (name, parse) in PARSERS {
618            let error = parse(json).err().expect("empty concepts should be rejected");
619            assert_eq!(error.to_string(), "Registry JSON contains no concepts.", "{name}");
620        }
621    }
622
623    // Ported from upstream `TestNewRegistryFromJSON_MissingContentHash`:
624    // https://github.com/DataDog/datadog-agent/blob/17ecddf4e3e83ccbb0e68aeb99461d1a1d902927/pkg/trace/semantics/registry_update_test.go#L37
625    #[test]
626    fn parsers_reject_missing_content_hash() {
627        let json = r#"{"version":"0.1.0","concepts":{"db.statement":{"canonical":"db.statement","fallbacks":[
628            {"name":"db.statement","provider":"datadog","type":"string"}]}}}"#;
629        for (name, parse) in PARSERS {
630            let error = parse(json).err().expect("missing content hash should be rejected");
631            assert_eq!(
632                error.to_string(),
633                "Registry JSON is missing `metadata.content_hash`.",
634                "{name}"
635            );
636        }
637    }
638
639    #[test]
640    fn from_json_rejects_empty_content_hash() {
641        let json = r#"{"version":"0.1.0","metadata":{"content_hash":""},"concepts":{"db.statement":{
642            "canonical":"db.statement","fallbacks":[{"name":"db.statement","provider":"datadog","type":"string"}]}}}"#;
643        let error = Registry::from_json(json)
644            .err()
645            .expect("empty content hash should be rejected");
646        assert_eq!(error.to_string(), "Registry JSON is missing `metadata.content_hash`.");
647    }
648
649    #[test]
650    fn content_hash_is_the_declared_label() {
651        let document: Value = serde_json::from_str(MAPPINGS_JSON).expect("embedded mappings should be JSON");
652        let declared = document["metadata"]["content_hash"]
653            .as_str()
654            .expect("embedded mappings should declare a content hash");
655        assert_eq!(EMBEDDED_REGISTRY.content_hash(), declared);
656    }
657
658    /// An upstream fixture; placeholder concepts `alpha` and `beta` are skipped as unknown.
659    const FINGERPRINT_BASE_JSON: &str = r#"{
660  "version":"1.0.0",
661  "git_commit":"commit-a",
662  "timestamp":"2026-01-01T00:00:00Z",
663  "metadata":{"content_hash":"same-declared-hash"},
664  "concepts":{
665    "alpha":{"canonical":"ignored.alpha","fallbacks":[
666      {"name":"alpha.one","provider":"datadog","version":"1.0","type":"string"},
667      {"name":"alpha.two","provider":"otel","type":"float64",
668       "when":[{"attribute":"span.kind","present":true,"eq":"client"}]}
669    ]},
670    "beta":{"canonical":"ignored.beta","fallbacks":[{"name":"beta.one","provider":"datadog","type":"string"}]}
671  }
672}"#;
673
674    #[track_caller]
675    fn fingerprint_of(json: &str) -> u64 {
676        let (registry, skipped) = Registry::from_slice_permissive(json.as_bytes()).expect("registry should parse");
677        assert_eq!(registry.content_hash(), "same-declared-hash");
678        assert_eq!(skipped.unknown_concepts, ["alpha", "beta"]);
679        registry.fingerprint()
680    }
681
682    // Ported from upstream `TestFingerprintChangesWithParsedMappings`:
683    // https://github.com/DataDog/datadog-agent/blob/17ecddf4e3e83ccbb0e68aeb99461d1a1d902927/pkg/trace/semantics/registry_update_test.go#L64
684    #[test]
685    fn fingerprint_changes_with_parsed_mappings() {
686        let base = fingerprint_of(FINGERPRINT_BASE_JSON);
687        let cases = [
688            (
689                "fallback name",
690                FINGERPRINT_BASE_JSON.replacen(r#""name":"alpha.one""#, r#""name":"alpha.changed""#, 1),
691            ),
692            (
693                "condition",
694                FINGERPRINT_BASE_JSON.replacen(r#""eq":"client""#, r#""eq":"server""#, 1),
695            ),
696        ];
697        for (name, json) in cases {
698            assert_ne!(
699                json, FINGERPRINT_BASE_JSON,
700                "{name}: the replacement must change the payload"
701            );
702            assert_ne!(fingerprint_of(&json), base, "{name}");
703        }
704    }
705
706    // Ported from upstream `TestFingerprintChangesWithJSONPresentationAndMetadata`:
707    // https://github.com/DataDog/datadog-agent/blob/17ecddf4e3e83ccbb0e68aeb99461d1a1d902927/pkg/trace/semantics/registry_update_test.go#L78
708    #[test]
709    fn fingerprint_changes_with_json_presentation_and_metadata() {
710        let base = fingerprint_of(FINGERPRINT_BASE_JSON);
711
712        let reformatted = FINGERPRINT_BASE_JSON.replace("  ", "    ");
713        assert_ne!(fingerprint_of(&reformatted), base);
714
715        let metadata_changed =
716            FINGERPRINT_BASE_JSON.replacen(r#""git_commit":"commit-a""#, r#""git_commit":"commit-b""#, 1);
717        assert_ne!(fingerprint_of(&metadata_changed), base);
718    }
719
720    // Ported from upstream `TestFingerprintDeterministicAcrossLoads`:
721    // https://github.com/DataDog/datadog-agent/blob/17ecddf4e3e83ccbb0e68aeb99461d1a1d902927/pkg/trace/semantics/registry_update_test.go#L91
722    #[test]
723    fn fingerprint_is_deterministic_across_loads() {
724        let want = fingerprint_of(FINGERPRINT_BASE_JSON);
725        for _ in 0..100 {
726            assert_eq!(fingerprint_of(FINGERPRINT_BASE_JSON), want);
727        }
728    }
729
730    // Ported from upstream `TestRegistryEqual_IdenticalPayloadBytesRegardlessOfSource`:
731    // https://github.com/DataDog/datadog-agent/blob/17ecddf4e3e83ccbb0e68aeb99461d1a1d902927/pkg/trace/semantics/registry_update_test.go#L111
732    #[test]
733    fn fingerprint_matches_for_identical_payload_bytes() {
734        let (remote, skipped) =
735            Registry::from_slice_permissive(MAPPINGS_JSON.as_bytes()).expect("embedded mappings should parse");
736        assert!(
737            skipped.malformed_concepts.is_empty() && skipped.fallbacks.is_empty(),
738            "{skipped}"
739        );
740        assert_eq!(remote.fingerprint(), EMBEDDED_REGISTRY.fingerprint());
741    }
742
743    // Ported from upstream `TestRegistryEqual_DifferentPayloadBytesWhen*` tests:
744    // https://github.com/DataDog/datadog-agent/blob/17ecddf4e3e83ccbb0e68aeb99461d1a1d902927/pkg/trace/semantics/registry_update_test.go#L120-L142
745    #[test]
746    fn fingerprint_differs_when_payload_bytes_change() {
747        const DB_STATEMENT: &str = concat!(
748            r#"{"db.statement":{"canonical":"db.statement","fallbacks":["#,
749            r#"{"name":"db.statement","provider":"datadog","type":"string"}]}}"#
750        );
751        const HTTP_METHOD: &str = concat!(
752            r#"{"http.method":{"canonical":"http.method","fallbacks":["#,
753            r#"{"name":"http.method","provider":"otel","type":"string"}]}}"#
754        );
755        let payload = |version: &str, content_hash: &str, concepts: &str| {
756            format!(r#"{{"version":"{version}","metadata":{{"content_hash":"{content_hash}"}},"concepts":{concepts}}}"#)
757        };
758        let fingerprint = |json: &str| Registry::from_json(json).expect("registry should parse").fingerprint();
759
760        let base = payload("1.0.0", "hash-a", DB_STATEMENT);
761        let cases = [
762            ("version", payload("2.0.0", "hash-a", DB_STATEMENT)),
763            (
764                "mappings under the same declared hash",
765                payload("1.0.0", "hash-a", HTTP_METHOD),
766            ),
767            (
768                "declared hash over the same mappings",
769                payload("1.0.0", "hash-b", DB_STATEMENT),
770            ),
771        ];
772        for (name, changed) in cases {
773            assert_ne!(fingerprint(&changed), fingerprint(&base), "{name}");
774        }
775    }
776
777    #[test]
778    fn permissive_skips_unknown_concepts_and_keeps_known_ones() {
779        let json = registry_json(
780            r#"{
781                "alpha": {"canonical": "alpha", "fallbacks": [{"name": "a", "provider": "otel", "type": "string"}]},
782                "db.statement": {"canonical": "db.statement", "fallbacks": [
783                    {"name": "db.statement", "provider": "datadog", "type": "string"}]},
784                "zeta": {"canonical": "zeta", "fallbacks": []}
785            }"#,
786        );
787        let (registry, skipped) = Registry::from_slice_permissive(json.as_bytes()).expect("known concepts should load");
788        let tags = registry
789            .get_attribute_precedence(Concept::DbStatement)
790            .expect("known concept should be kept");
791        assert_eq!(tags.len(), 1);
792        assert_eq!(tags[0].name, "db.statement");
793        assert_eq!(registry.mappings.len(), 1);
794        assert_eq!(skipped.unknown_concepts, ["alpha", "zeta"]);
795        assert!(skipped.malformed_concepts.is_empty() && skipped.fallbacks.is_empty());
796    }
797
798    #[test]
799    fn permissive_accepts_a_payload_whose_concepts_are_all_unknown() {
800        // Upstream rejects only an empty `concepts` map; unknown keys are not a rejection.
801        let json = registry_json(r#"{"alpha": {"canonical": "alpha", "fallbacks": []}}"#);
802        let (registry, skipped) =
803            Registry::from_slice_permissive(json.as_bytes()).expect("unknown concepts are not a rejection");
804        assert!(registry.mappings.is_empty());
805        assert_eq!(skipped.unknown_concepts, ["alpha"]);
806    }
807
808    #[test]
809    fn permissive_skips_unrepresentable_fallbacks_and_keeps_the_rest() {
810        let json = registry_json(
811            r#"{"http.status_code": {"canonical": "http.status_code", "fallbacks": [
812                {"name": "unknown.provider", "provider": "vendor", "type": "string"},
813                {"name": "kept.first", "provider": "otel", "type": "int64"},
814                {"name": "unknown.type", "provider": "otel", "type": "bool"},
815                {"name": "absent.type", "provider": "otel"},
816                {"name": 5, "provider": "otel", "type": "string"},
817                {"name": "bad.when", "provider": "otel", "type": "string", "when": [{"attribute": 5}]},
818                "not an object",
819                {"name": "kept.second", "provider": "datadog", "type": "string", "future_field": true}
820            ]}}"#,
821        );
822        let (registry, skipped) = Registry::from_slice_permissive(json.as_bytes()).expect("payload should load");
823        let names: Vec<&str> = registry
824            .get_attribute_precedence(Concept::HttpStatusCode)
825            .expect("concept should be kept")
826            .iter()
827            .map(|tag| tag.name.as_str())
828            .collect();
829        assert_eq!(names, ["kept.first", "kept.second"]);
830        let positions: Vec<usize> = skipped.fallbacks.iter().map(|(_, position)| *position).collect();
831        assert_eq!(positions, [0, 2, 3, 4, 5, 6]);
832        assert!(skipped.fallbacks.iter().all(|(key, _)| key == "http.status_code"));
833        assert!(skipped.unknown_concepts.is_empty() && skipped.malformed_concepts.is_empty());
834    }
835
836    #[test]
837    fn parsers_treat_absent_fallbacks_as_empty() {
838        let json = registry_json(r#"{"db.statement": {"canonical": "db.statement"}}"#);
839        for (name, parse) in PARSERS {
840            let registry = parse(&json).expect("absent fallbacks should parse");
841            assert_eq!(
842                registry.get_attribute_precedence(Concept::DbStatement).map(<[_]>::len),
843                Some(0),
844                "{name}"
845            );
846        }
847    }
848
849    #[test]
850    fn parsers_treat_null_fallbacks_as_empty() {
851        let json = registry_json(r#"{"db.statement": {"canonical": "db.statement", "fallbacks": null}}"#);
852        for (name, parse) in PARSERS {
853            let registry = parse(&json).expect("null fallbacks should parse");
854            assert_eq!(
855                registry.get_attribute_precedence(Concept::DbStatement).map(<[_]>::len),
856                Some(0),
857                "{name}"
858            );
859        }
860    }
861
862    #[test]
863    fn permissive_skips_malformed_concepts() {
864        let json = registry_json(
865            r#"{
866                "db.statement": "not an object",
867                "http.method": {"canonical": "http.method", "fallbacks": {"name": "http.method"}},
868                "http.route": {"canonical": "http.route", "fallbacks": [
869                    {"name": "http.route", "provider": "otel", "type": "string"}]}
870            }"#,
871        );
872        let (registry, skipped) = Registry::from_slice_permissive(json.as_bytes()).expect("payload should load");
873        assert!(registry.get_attribute_precedence(Concept::DbStatement).is_none());
874        assert!(registry.get_attribute_precedence(Concept::HttpMethod).is_none());
875        assert_eq!(
876            registry.get_attribute_precedence(Concept::HttpRoute).map(<[_]>::len),
877            Some(1)
878        );
879        assert_eq!(skipped.malformed_concepts, ["db.statement", "http.method"]);
880    }
881
882    #[test]
883    fn permissive_reads_version_and_content_hash() {
884        let json = r#"{"version":"2.1.0","metadata":{"content_hash":"hash-a"},"concepts":{"alpha":{}}}"#;
885        let (registry, _) = Registry::from_slice_permissive(json.as_bytes()).expect("payload should load");
886        assert_eq!(registry.version(), "2.1.0");
887        assert_eq!(registry.content_hash(), "hash-a");
888
889        let non_string_version = r#"{"version":2,"metadata":{"content_hash":"hash-a"},"concepts":{"alpha":{}}}"#;
890        let (registry, _) =
891            Registry::from_slice_permissive(non_string_version.as_bytes()).expect("a non-string version is not fatal");
892        assert_eq!(registry.version(), "");
893    }
894
895    #[test]
896    fn permissive_errors_never_quote_skipped_parts() {
897        let without_hash = r#"{"version":"0.1.0","concepts":{"secret.concept":{"fallbacks":[
898            {"name":"secret.name","provider":"secret-provider","type":"string"}]}}}"#;
899        let empty_hash = r#"{"metadata":{"content_hash":""},"concepts":{"secret.concept":{}}}"#;
900        for json in [without_hash, empty_hash] {
901            let error = Registry::from_slice_permissive(json.as_bytes())
902                .err()
903                .expect("missing content hash should be rejected");
904            assert_eq!(error.to_string(), "Registry JSON is missing `metadata.content_hash`.");
905        }
906
907        let malformed = r#"{"concepts":{"secret.concept":{"fallbacks":[secret]}}}"#;
908        let error = Registry::from_slice_permissive(malformed.as_bytes())
909            .err()
910            .expect("malformed JSON should be rejected");
911        assert_eq!(error.to_string(), "Registry JSON is malformed at line 1, column 45.");
912    }
913
914    #[test]
915    fn skip_report_display_names_skipped_parts() {
916        let json = registry_json(
917            r#"{
918                "alpha": {},
919                "beta": {},
920                "db.statement": {"fallbacks": [{"name": "x", "provider": "vendor", "type": "string"}]},
921                "http.method": "not an object",
922                "http.route": {"fallbacks": [
923                    {"name": "http.route", "provider": "otel", "type": "string"},
924                    {"name": "http.route", "provider": "otel"}]}
925            }"#,
926        );
927        let (_, skipped) = Registry::from_slice_permissive(json.as_bytes()).expect("payload should load");
928        assert_eq!(
929            skipped.to_string(),
930            r#"unknown concepts: 2 ("alpha", "beta"); malformed concepts: 1 ("http.method"); "#.to_owned()
931                + r#"unrepresentable fallbacks: 2 ("db.statement"[0], "http.route"[1])"#
932        );
933        assert_eq!(SkipReport::default().to_string(), "nothing skipped");
934    }
935
936    #[test]
937    fn parsers_treat_null_mapping_as_empty() {
938        let json = registry_json(r#"{"db.statement": null}"#);
939        for (name, parse) in PARSERS {
940            let registry = parse(&json).expect("null mapping should parse");
941            assert_eq!(
942                registry.get_attribute_precedence(Concept::DbStatement).map(<[_]>::len),
943                Some(0),
944                "{name}"
945            );
946        }
947    }
948
949    // Go's `encoding/json` leaves a `null` slice element at its zero value, a condition with no predicates.
950    #[test]
951    fn parsers_read_null_condition_as_empty() {
952        let json = registry_json(
953            r#"{"db.statement": {"fallbacks": [
954                {"name": "db.statement", "provider": "datadog", "type": "string", "when": [null]}]}}"#,
955        );
956        for (name, parse) in PARSERS {
957            let registry = parse(&json).expect("a null condition should parse");
958            let tags = registry
959                .get_attribute_precedence(Concept::DbStatement)
960                .expect("concept should be kept");
961            assert_eq!(tags.len(), 1, "{name}");
962            let [condition] = tags[0].when.as_slice() else {
963                panic!("{name}: expected one condition, got {:?}", tags[0].when);
964            };
965            assert_eq!(condition.attribute, "", "{name}");
966            assert_eq!(condition.present, None, "{name}");
967            assert_eq!(condition.eq, None, "{name}");
968        }
969    }
970
971    #[test]
972    fn parsers_read_null_version_and_when_as_defaults() {
973        let json = registry_json(
974            r#"{"db.statement": {"fallbacks": [
975                {"name": "db.statement", "provider": "datadog", "type": "string", "version": null, "when": null}]}}"#,
976        );
977        for (name, parse) in PARSERS {
978            let registry = parse(&json).expect("null version and when should parse");
979            let tags = registry
980                .get_attribute_precedence(Concept::DbStatement)
981                .expect("concept should be kept");
982            assert_eq!(tags.len(), 1, "{name}");
983            assert_eq!(tags[0].version, "", "{name}");
984            assert!(tags[0].when.is_empty(), "{name}");
985        }
986    }
987
988    #[test]
989    fn fallback_missing_or_null_name_defaults_to_empty_string() {
990        // Go's `encoding/json` leaves an absent or `null` name at its zero value.
991        let json = registry_json(
992            r#"{"db.statement": {"fallbacks": [
993                {"provider": "datadog", "type": "string"},
994                {"name": null, "provider": "otel", "type": "string"}
995            ]}}"#,
996        );
997        let (registry, skipped) = Registry::from_slice_permissive(json.as_bytes()).expect("payload should load");
998        assert!(skipped.fallbacks.is_empty(), "{skipped}");
999        let tags = registry
1000            .get_attribute_precedence(Concept::DbStatement)
1001            .expect("concept should be kept");
1002        assert_eq!(tags.len(), 2, "both fallbacks should be kept rather than skipped");
1003        assert!(tags.iter().all(|tag| tag.name.is_empty()));
1004    }
1005
1006    #[test]
1007    fn skip_report_display_shortens_long_keys_on_a_char_boundary() {
1008        // A multi-byte character sits right at the cutoff; truncating by byte count would panic.
1009        let long_key: String = "é".repeat(70);
1010        let json = registry_json(&format!(r#"{{"{long_key}": {{}}}}"#));
1011        let (_, skipped) = Registry::from_slice_permissive(json.as_bytes()).expect("payload should load");
1012        let shortened: String = long_key.chars().take(64).chain(std::iter::once('…')).collect();
1013        assert_eq!(skipped.to_string(), format!("unknown concepts: 1 ({shortened:?})"));
1014    }
1015
1016    #[test]
1017    fn parsers_read_absent_or_null_condition_attribute_as_empty() {
1018        let json = registry_json(
1019            r#"{"db.statement": {"fallbacks": [
1020                {"name": "db.statement", "provider": "datadog", "type": "string",
1021                 "when": [{"eq": "grpc"}, {"attribute": null, "present": true}]}]}}"#,
1022        );
1023        for (name, parse) in PARSERS {
1024            let registry = parse(&json).expect("conditions without an attribute should parse");
1025            let when = &registry
1026                .get_attribute_precedence(Concept::DbStatement)
1027                .expect("concept should be kept")[0]
1028                .when;
1029            assert_eq!(when.len(), 2, "{name}");
1030            assert_eq!(when[0].attribute, "", "{name}");
1031            assert_eq!(when[0].eq.as_deref(), Some("grpc"), "{name}");
1032            assert_eq!(when[1].attribute, "", "{name}");
1033            assert_eq!(when[1].present, Some(true), "{name}");
1034        }
1035    }
1036
1037    #[test]
1038    fn skip_report_display_escapes_keys_and_caps_each_list() {
1039        let unknown: Vec<String> = (0..12).map(|i| format!(r#""k{i:02}": {{}}"#)).collect();
1040        let bad_entries = [r#"{"provider": "vendor"}"#; 11].join(", ");
1041        let json = registry_json(&format!(
1042            r#"{{"a\nbreak": {{}}, {}, "db.statement": {{"fallbacks": [{bad_entries}]}}}}"#,
1043            unknown.join(", ")
1044        ));
1045        let (_, skipped) = Registry::from_slice_permissive(json.as_bytes()).expect("payload should load");
1046        let listed_unknown: Vec<String> = std::iter::once(r#""a\nbreak""#.to_owned())
1047            .chain((0..9).map(|i| format!(r#""k{i:02}""#)))
1048            .collect();
1049        let listed_fallbacks: Vec<String> = (0..10).map(|i| format!(r#""db.statement"[{i}]"#)).collect();
1050        assert_eq!(
1051            skipped.to_string(),
1052            format!(
1053                "unknown concepts: 13 ({}, ...); unrepresentable fallbacks: 11 ({}, ...)",
1054                listed_unknown.join(", "),
1055                listed_fallbacks.join(", ")
1056            )
1057        );
1058    }
1059}
datadog_agent_apm/semantics/registry.rs

datadog_agent_apm/semantics/
registry.rs
