dd_sds/scanner/regex_rule/
config.rs

1use crate::proximity_keywords::compile_keywords_proximity_config;
2use crate::scanner::config::RuleConfig;
3use crate::scanner::metrics::RuleMetrics;
4use crate::scanner::regex_rule::compiled::RegexCompiledRule;
5use crate::scanner::regex_rule::regex_store::get_memoized_regex;
6use crate::validation::{
7    RegexPatternCaptureGroupsValidationError, validate_and_create_regex,
8    validate_named_capture_group_minimum_length,
9};
10use crate::{CompiledRule, CreateScannerError, Labels};
11use regex_automata::util::captures::GroupInfo;
12use serde::{Deserialize, Serialize};
13use serde_with::DefaultOnNull;
14use serde_with::serde_as;
15use std::sync::Arc;
16use strum::{AsRefStr, EnumIter};
17
18pub const DEFAULT_KEYWORD_LOOKAHEAD: usize = 30;
19
20#[serde_as]
21#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
22pub struct RegexRuleConfig {
23    pub pattern: String,
24    pub proximity_keywords: Option<ProximityKeywordsConfig>,
25    pub validator: Option<SecondaryValidator>,
26    #[serde_as(deserialize_as = "DefaultOnNull")]
27    #[serde(default)]
28    pub labels: Labels,
29    pub pattern_capture_groups: Option<Vec<String>>,
30}
31
32impl RegexRuleConfig {
33    pub fn new(pattern: &str) -> Self {
34        #[allow(deprecated)]
35        Self {
36            pattern: pattern.to_owned(),
37            proximity_keywords: None,
38            validator: None,
39            labels: Labels::default(),
40            pattern_capture_groups: None,
41        }
42    }
43
44    pub fn with_pattern(&self, pattern: &str) -> Self {
45        self.mutate_clone(|x| x.pattern = pattern.to_string())
46    }
47
48    pub fn with_proximity_keywords(&self, proximity_keywords: ProximityKeywordsConfig) -> Self {
49        self.mutate_clone(|x| x.proximity_keywords = Some(proximity_keywords))
50    }
51
52    pub fn with_labels(&self, labels: Labels) -> Self {
53        self.mutate_clone(|x| x.labels = labels)
54    }
55
56    pub fn with_pattern_capture_groups(&self, pattern_capture_groups: Vec<String>) -> Self {
57        self.mutate_clone(|x| x.pattern_capture_groups = Some(pattern_capture_groups))
58    }
59
60    pub fn with_pattern_capture_group(&self, pattern_capture_group: &str) -> Self {
61        self.mutate_clone(|x| match x.pattern_capture_groups {
62            Some(ref mut pattern_capture_groups) => {
63                pattern_capture_groups.push(pattern_capture_group.to_string());
64            }
65            None => {
66                x.pattern_capture_groups = Some(vec![pattern_capture_group.to_string()]);
67            }
68        })
69    }
70
71    pub fn build(&self) -> Arc<dyn RuleConfig> {
72        Arc::new(self.clone())
73    }
74
75    fn mutate_clone(&self, modify: impl FnOnce(&mut Self)) -> Self {
76        let mut clone = self.clone();
77        modify(&mut clone);
78        clone
79    }
80
81    pub fn with_included_keywords(
82        &self,
83        keywords: impl IntoIterator<Item = impl AsRef<str>>,
84    ) -> Self {
85        let mut this = self.clone();
86        let mut config = self.get_or_create_proximity_keywords_config();
87        config.included_keywords = keywords
88            .into_iter()
89            .map(|x| x.as_ref().to_string())
90            .collect::<Vec<_>>();
91        this.proximity_keywords = Some(config);
92        this
93    }
94
95    pub fn with_excluded_keywords(
96        &self,
97        keywords: impl IntoIterator<Item = impl AsRef<str>>,
98    ) -> Self {
99        let mut this = self.clone();
100        let mut config = self.get_or_create_proximity_keywords_config();
101        config.excluded_keywords = keywords
102            .into_iter()
103            .map(|x| x.as_ref().to_string())
104            .collect::<Vec<_>>();
105        this.proximity_keywords = Some(config);
106        this
107    }
108
109    pub fn with_validator(&self, validator: Option<SecondaryValidator>) -> Self {
110        let mut this = self.clone();
111        this.validator = validator;
112        this
113    }
114
115    fn get_or_create_proximity_keywords_config(&self) -> ProximityKeywordsConfig {
116        self.proximity_keywords
117            .clone()
118            .unwrap_or_else(|| ProximityKeywordsConfig {
119                look_ahead_character_count: DEFAULT_KEYWORD_LOOKAHEAD,
120                included_keywords: vec![],
121                excluded_keywords: vec![],
122            })
123    }
124}
125
126fn is_pattern_capture_groups_valid(
127    pattern: &str,
128    pattern_capture_groups: &Option<Vec<String>>,
129    group_info: &GroupInfo,
130) -> Result<(), RegexPatternCaptureGroupsValidationError> {
131    if pattern_capture_groups.is_none() {
132        return Ok(());
133    }
134    let pattern_capture_groups = pattern_capture_groups.as_ref().unwrap();
135    if pattern_capture_groups.len() != 1 {
136        // We currently only allow one capture group
137        return Err(
138            RegexPatternCaptureGroupsValidationError::TooManyCaptureGroups(
139                pattern_capture_groups.len(),
140            ),
141        );
142    }
143    let pattern_capture_group = pattern_capture_groups.first().unwrap();
144    if !group_info
145        .all_names()
146        .filter(|(_, _, name)| name.is_some())
147        .map(|(_, _, name)| name.unwrap())
148        .any(|name| name == pattern_capture_group)
149    {
150        return Err(
151            RegexPatternCaptureGroupsValidationError::CaptureGroupNotPresent(
152                pattern_capture_group.clone(),
153            ),
154        );
155    }
156    // At this point, the capture group is in the regex, and there is exactly one.
157    // Currently, it must be called `sds_match`.
158    if pattern_capture_group != "sds_match" {
159        return Err(RegexPatternCaptureGroupsValidationError::TargetedCaptureGroupMustBeSdsMatch);
160    }
161    validate_named_capture_group_minimum_length(pattern, pattern_capture_group)?;
162    Ok(())
163}
164
165impl RuleConfig for RegexRuleConfig {
166    fn convert_to_compiled_rule(
167        &self,
168        rule_index: usize,
169        scanner_labels: Labels,
170    ) -> Result<Box<dyn CompiledRule>, CreateScannerError> {
171        let regex = get_memoized_regex(&self.pattern, validate_and_create_regex)?;
172
173        let rule_labels = scanner_labels.clone_with_labels(self.labels.clone());
174
175        let (included_keywords, excluded_keywords) = self
176            .proximity_keywords
177            .as_ref()
178            .map(|config| compile_keywords_proximity_config(config, &rule_labels))
179            .unwrap_or(Ok((None, None)))?;
180
181        is_pattern_capture_groups_valid(
182            &self.pattern,
183            &self.pattern_capture_groups,
184            regex.group_info(),
185        )?;
186
187        Ok(Box::new(RegexCompiledRule {
188            rule_index,
189            regex,
190            included_keywords,
191            excluded_keywords,
192            validator: self.validator.clone().map(|x| x.compile()),
193            metrics: RuleMetrics::new(&rule_labels),
194            pattern_capture_groups: self.pattern_capture_groups.clone(),
195        }))
196    }
197
198    fn as_regex_rule(&self) -> Option<&RegexRuleConfig> {
199        Some(self)
200    }
201}
202
203#[serde_as]
204#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
205pub struct ProximityKeywordsConfig {
206    pub look_ahead_character_count: usize,
207
208    #[serde_as(deserialize_as = "DefaultOnNull")]
209    #[serde(default)]
210    pub included_keywords: Vec<String>,
211
212    #[serde_as(deserialize_as = "DefaultOnNull")]
213    #[serde(default)]
214    pub excluded_keywords: Vec<String>,
215}
216
217#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, EnumIter, AsRefStr)]
218#[serde(tag = "type")]
219pub enum SecondaryValidator {
220    AbaRtnChecksum,
221    AtlassianTokenChecksum,
222    AustralianMedicareChecksum,
223    AustralianTfnChecksum,
224    AustrianSSNChecksum,
225    BelgiumNationalRegisterChecksum,
226    BrazilianCnpjChecksum,
227    BrazilianCpfChecksum,
228    BtcChecksum,
229    BulgarianEGNChecksum,
230    ChineseIdChecksum,
231    CoordinationNumberChecksum,
232    CzechPersonalIdentificationNumberChecksum,
233    CzechTaxIdentificationNumberChecksum,
234    DutchBsnChecksum,
235    DutchPassportChecksum,
236    EntropyCheck,
237    EstoniaPersonalCodeChecksum,
238    EthereumChecksum,
239    FinnishHetuChecksum,
240    FranceNifChecksum,
241    FranceSsnChecksum,
242    GermanIdsChecksum,
243    GermanSvnrChecksum,
244    GithubTokenChecksum,
245    GreeceAmkaChecksum,
246    GreekTinChecksum,
247    HungarianTinChecksum,
248    IbanChecker,
249    IrishPpsChecksum,
250    ItalianNationalIdChecksum,
251    JwtClaimsValidator { config: JwtClaimsValidatorConfig },
252    JwtExpirationChecker,
253    LatviaNationalIdChecksum,
254    LithuanianPersonalIdentificationNumberChecksum,
255    LuhnChecksum,
256    LuxembourgIndividualNINChecksum,
257    Mod11_10checksum,
258    Mod11_2checksum,
259    Mod1271_36Checksum,
260    Mod27_26checksum,
261    Mod37_2checksum,
262    Mod37_36checksum,
263    Mod661_26checksum,
264    Mod97_10checksum,
265    MoneroAddress,
266    NhsCheckDigit,
267    NirChecksum,
268    NonHexChecker,
269    NonHexPlusTokenEfficiencyChecker,
270    PolishNationalIdChecksum,
271    PolishNipChecksum,
272    PortugueseTaxIdChecksum,
273    RodneCisloNumberChecksum,
274    RomanianPersonalNumericCode,
275    SingaporeNricChecksum,
276    SloveniaTinChecksum,
277    SlovenianPINChecksum,
278    SpanishDniChecksum,
279    SpanishNussChecksum,
280    SwedenPINChecksum,
281    TokenEfficiencyCheck,
282    UkNinoFormatCheck,
283    UkTrnChecksum,
284    UsDeaChecksum,
285    UsNpiChecksum,
286    VerhoeffChecksum,
287    VinChecksum,
288}
289
290#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
291#[serde(tag = "type", content = "config")]
292pub enum ClaimRequirement {
293    /// Just check that the claim exists
294    Present,
295    /// Check that the claim exists and is not expired
296    NotExpired,
297    /// Check that the claim exists and has an exact value
298    ExactValue(String),
299    /// Check that the claim exists and matches a regex pattern
300    RegexMatch(String),
301}
302
303#[derive(Serialize, Deserialize, Default, Clone, Debug, PartialEq)]
304pub struct JwtClaimsValidatorConfig {
305    #[serde(default)]
306    pub required_headers: std::collections::BTreeMap<String, ClaimRequirement>,
307    #[serde(default)]
308    pub required_claims: std::collections::BTreeMap<String, ClaimRequirement>,
309}
310
311#[cfg(test)]
312mod test {
313    use crate::{AwsType, CustomHttpConfig, MatchAction, MatchValidationType, RootRuleConfig};
314    use std::collections::BTreeMap;
315    use strum::IntoEnumIterator;
316
317    use super::*;
318
319    #[test]
320    fn should_override_pattern() {
321        let rule_config = RegexRuleConfig::new("123").with_pattern("456");
322        assert_eq!(rule_config.pattern, "456");
323    }
324
325    #[test]
326    #[allow(deprecated)]
327    fn should_have_default() {
328        let rule_config = RegexRuleConfig::new("123");
329        assert_eq!(
330            rule_config,
331            RegexRuleConfig {
332                pattern: "123".to_string(),
333                proximity_keywords: None,
334                validator: None,
335                labels: Labels::empty(),
336                pattern_capture_groups: None,
337            }
338        );
339    }
340
341    #[test]
342    fn should_use_capture_group() {
343        let rule_config = RegexRuleConfig::new("hey (?<capture_group>world)")
344            .with_pattern_capture_groups(vec!["capture_group".to_string()]);
345        assert_eq!(
346            rule_config,
347            RegexRuleConfig {
348                pattern: "hey (?<capture_group>world)".to_string(),
349                proximity_keywords: None,
350                validator: None,
351                labels: Labels::empty(),
352                pattern_capture_groups: Some(vec!["capture_group".to_string()]),
353            }
354        );
355    }
356
357    #[test]
358    fn match_action_should_default_to_none_on_deserialization() {
359        let config: RootRuleConfig<RegexRuleConfig> =
360            serde_json::from_str(r#"{"pattern":"hello"}"#).unwrap();
361        assert_eq!(config.match_action, MatchAction::None);
362    }
363
364    #[test]
365    fn proximity_keywords_should_have_default() {
366        let json_config = r#"{"look_ahead_character_count": 0}"#;
367        let test: ProximityKeywordsConfig = serde_json::from_str(json_config).unwrap();
368        assert_eq!(
369            test,
370            ProximityKeywordsConfig {
371                look_ahead_character_count: 0,
372                included_keywords: vec![],
373                excluded_keywords: vec![]
374            }
375        );
376
377        let json_config = r#"{"look_ahead_character_count": 0, "excluded_keywords": null, "included_keywords": null}"#;
378        let test: ProximityKeywordsConfig = serde_json::from_str(json_config).unwrap();
379        assert_eq!(
380            test,
381            ProximityKeywordsConfig {
382                look_ahead_character_count: 0,
383                included_keywords: vec![],
384                excluded_keywords: vec![]
385            }
386        );
387    }
388
389    #[test]
390    #[allow(deprecated)]
391    fn test_third_party_active_checker() {
392        // Test setting only the new field
393        let http_config = CustomHttpConfig::default().with_endpoint("http://test.com".to_string());
394        let validation_type = MatchValidationType::CustomHttp(http_config.clone());
395        let rule_config = RootRuleConfig::new(RegexRuleConfig::new("123"))
396            .third_party_active_checker(validation_type.clone());
397
398        assert_eq!(
399            rule_config.third_party_active_checker,
400            Some(validation_type.clone())
401        );
402        assert_eq!(rule_config.match_validation_type, None);
403        assert_eq!(
404            rule_config.get_third_party_active_checker(),
405            Some(&validation_type)
406        );
407
408        // Test setting via deprecated field updates both
409        let aws_type = AwsType::AwsId;
410        let validation_type2 = MatchValidationType::Aws(aws_type);
411        let rule_config = RootRuleConfig::new(RegexRuleConfig::new("123"))
412            .third_party_active_checker(validation_type2.clone());
413
414        assert_eq!(
415            rule_config.third_party_active_checker,
416            Some(validation_type2.clone())
417        );
418        assert_eq!(
419            rule_config.get_third_party_active_checker(),
420            Some(&validation_type2)
421        );
422
423        // Test that get_match_validation_type prioritizes third_party_active_checker
424        let rule_config = RootRuleConfig::new(RegexRuleConfig::new("123"))
425            .third_party_active_checker(MatchValidationType::CustomHttp(http_config.clone()));
426
427        assert_eq!(
428            rule_config.get_third_party_active_checker(),
429            Some(&MatchValidationType::CustomHttp(http_config.clone()))
430        );
431    }
432
433    #[test]
434    fn test_secondary_validator_enum_iter() {
435        // Test that we can iterate over all SecondaryValidator variants
436        let validators: Vec<SecondaryValidator> = SecondaryValidator::iter().collect();
437        // Verify some variants
438        assert!(validators.contains(&SecondaryValidator::GithubTokenChecksum));
439        assert!(validators.contains(&SecondaryValidator::JwtExpirationChecker));
440    }
441
442    #[test]
443    fn test_secondary_validator_are_sorted() {
444        let validator_names: Vec<String> = SecondaryValidator::iter()
445            .map(|a| a.as_ref().to_string())
446            .collect();
447        let mut sorted_validator_names = validator_names.clone();
448        sorted_validator_names.sort();
449        assert_eq!(
450            sorted_validator_names, validator_names,
451            "Secondary validators should be sorted by alphabetical order, but it's not the case, expected order:"
452        );
453    }
454
455    // The order has to be stable to pass linter checks. Otherwise, each instantiation will change the file
456    #[test]
457    fn test_jwt_claims_validator_config_serialization_order() {
458        // Create a config with claims in non-alphabetical order
459        let mut required_claims = BTreeMap::new();
460        required_claims.insert("zzz".to_string(), ClaimRequirement::Present);
461        required_claims.insert("exp".to_string(), ClaimRequirement::NotExpired);
462        required_claims.insert(
463            "aaa".to_string(),
464            ClaimRequirement::ExactValue("test".to_string()),
465        );
466        required_claims.insert(
467            "mmm".to_string(),
468            ClaimRequirement::RegexMatch(r"^test.*".to_string()),
469        );
470
471        let config = JwtClaimsValidatorConfig {
472            required_claims,
473            required_headers: std::collections::BTreeMap::new(),
474        };
475
476        // Serialize multiple times to ensure stable order
477        let serialized1 = serde_json::to_string(&config).unwrap();
478        let serialized2 = serde_json::to_string(&config).unwrap();
479
480        // Both serializations should be identical
481        assert_eq!(serialized1, serialized2, "Serialization should be stable");
482
483        // Keys should be in alphabetical order
484        assert!(serialized1.find("aaa").unwrap() < serialized1.find("exp").unwrap());
485        assert!(serialized1.find("exp").unwrap() < serialized1.find("mmm").unwrap());
486        assert!(serialized1.find("mmm").unwrap() < serialized1.find("zzz").unwrap());
487    }
488
489    #[test]
490    fn test_capture_groups_validation() {
491        let test_cases: Vec<(
492            &str,
493            Vec<String>,
494            Result<(), RegexPatternCaptureGroupsValidationError>,
495        )> = vec![
496            (
497                "hello (?<sds_match>world)",
498                vec!["sds_match".to_string()],
499                Ok(()),
500            ),
501            (
502                "hello (?<capture_group>world)",
503                vec!["capture_group".to_string()],
504                Err(RegexPatternCaptureGroupsValidationError::TargetedCaptureGroupMustBeSdsMatch),
505            ),
506            (
507                "hello (?<sds_match>world) and (?<another_group>world)",
508                vec!["sds_match".to_string()],
509                Ok(()),
510            ),
511            (
512                "hello (?<capture_grou>world)",
513                vec!["capture_group".to_string()],
514                Err(
515                    RegexPatternCaptureGroupsValidationError::CaptureGroupNotPresent(
516                        "capture_group".to_string(),
517                    ),
518                ),
519            ),
520            (
521                "hello (?<sds_match>d*)",
522                vec!["sds_match".to_string()],
523                Err(RegexPatternCaptureGroupsValidationError::CaptureGroupMatchesEmptyString),
524            ),
525            (
526                "hello (?<sds_match>world)",
527                vec!["sds_match".to_string(), "sds_match2".to_string()],
528                Err(RegexPatternCaptureGroupsValidationError::TooManyCaptureGroups(2)),
529            ),
530        ];
531        for (pattern, capture_groups, expected_result) in test_cases {
532            let rule_config =
533                RegexRuleConfig::new(pattern).with_pattern_capture_groups(capture_groups);
534            assert_eq!(
535                is_pattern_capture_groups_valid(
536                    &rule_config.pattern,
537                    &rule_config.pattern_capture_groups,
538                    &get_memoized_regex(pattern, validate_and_create_regex)
539                        .unwrap()
540                        .group_info()
541                ),
542                expected_result
543            );
544        }
545    }
546}
dd_sds/scanner/regex_rule/config.rs

dd_sds/scanner/regex_rule/
config.rs
