dd_sds/scanner/regex_rule/
config.rs

1use crate::proximity_keywords::compile_keywords_proximity_config;
2use crate::scanner::config::RuleConfig;
3use crate::scanner::metrics::RuleMetrics;
4use crate::scanner::regex_rule::compiled::RegexCompiledRule;
5use crate::scanner::regex_rule::regex_store::get_memoized_regex;
6use crate::validation::{
7    RegexPatternCaptureGroupsValidationError, validate_and_create_regex,
8    validate_named_capture_group_minimum_length,
9};
10use crate::{CompiledRule, CreateScannerError, Labels};
11use regex_automata::util::captures::GroupInfo;
12use serde::{Deserialize, Serialize};
13use serde_with::DefaultOnNull;
14use serde_with::serde_as;
15use std::sync::Arc;
16use strum::{AsRefStr, EnumIter};
17
18pub const DEFAULT_KEYWORD_LOOKAHEAD: usize = 30;
19
20#[serde_as]
21#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
22pub struct RegexRuleConfig {
23    pub pattern: String,
24    pub proximity_keywords: Option<ProximityKeywordsConfig>,
25    pub validator: Option<SecondaryValidator>,
26    #[serde_as(deserialize_as = "DefaultOnNull")]
27    #[serde(default)]
28    pub labels: Labels,
29    pub pattern_capture_groups: Option<Vec<String>>,
30}
31
32impl RegexRuleConfig {
33    pub fn new(pattern: &str) -> Self {
34        #[allow(deprecated)]
35        Self {
36            pattern: pattern.to_owned(),
37            proximity_keywords: None,
38            validator: None,
39            labels: Labels::default(),
40            pattern_capture_groups: None,
41        }
42    }
43
44    pub fn with_pattern(&self, pattern: &str) -> Self {
45        self.mutate_clone(|x| x.pattern = pattern.to_string())
46    }
47
48    pub fn with_proximity_keywords(&self, proximity_keywords: ProximityKeywordsConfig) -> Self {
49        self.mutate_clone(|x| x.proximity_keywords = Some(proximity_keywords))
50    }
51
52    pub fn with_labels(&self, labels: Labels) -> Self {
53        self.mutate_clone(|x| x.labels = labels)
54    }
55
56    pub fn with_pattern_capture_groups(&self, pattern_capture_groups: Vec<String>) -> Self {
57        self.mutate_clone(|x| x.pattern_capture_groups = Some(pattern_capture_groups))
58    }
59
60    pub fn with_pattern_capture_group(&self, pattern_capture_group: &str) -> Self {
61        self.mutate_clone(|x| match x.pattern_capture_groups {
62            Some(ref mut pattern_capture_groups) => {
63                pattern_capture_groups.push(pattern_capture_group.to_string());
64            }
65            None => {
66                x.pattern_capture_groups = Some(vec![pattern_capture_group.to_string()]);
67            }
68        })
69    }
70
71    pub fn build(&self) -> Arc<dyn RuleConfig> {
72        Arc::new(self.clone())
73    }
74
75    fn mutate_clone(&self, modify: impl FnOnce(&mut Self)) -> Self {
76        let mut clone = self.clone();
77        modify(&mut clone);
78        clone
79    }
80
81    pub fn with_included_keywords(
82        &self,
83        keywords: impl IntoIterator<Item = impl AsRef<str>>,
84    ) -> Self {
85        let mut this = self.clone();
86        let mut config = self.get_or_create_proximity_keywords_config();
87        config.included_keywords = keywords
88            .into_iter()
89            .map(|x| x.as_ref().to_string())
90            .collect::<Vec<_>>();
91        this.proximity_keywords = Some(config);
92        this
93    }
94
95    pub fn with_excluded_keywords(
96        &self,
97        keywords: impl IntoIterator<Item = impl AsRef<str>>,
98    ) -> Self {
99        let mut this = self.clone();
100        let mut config = self.get_or_create_proximity_keywords_config();
101        config.excluded_keywords = keywords
102            .into_iter()
103            .map(|x| x.as_ref().to_string())
104            .collect::<Vec<_>>();
105        this.proximity_keywords = Some(config);
106        this
107    }
108
109    pub fn with_validator(&self, validator: Option<SecondaryValidator>) -> Self {
110        let mut this = self.clone();
111        this.validator = validator;
112        this
113    }
114
115    fn get_or_create_proximity_keywords_config(&self) -> ProximityKeywordsConfig {
116        self.proximity_keywords
117            .clone()
118            .unwrap_or_else(|| ProximityKeywordsConfig {
119                look_ahead_character_count: DEFAULT_KEYWORD_LOOKAHEAD,
120                included_keywords: vec![],
121                excluded_keywords: vec![],
122            })
123    }
124}
125
126fn is_pattern_capture_groups_valid(
127    pattern: &str,
128    pattern_capture_groups: &Option<Vec<String>>,
129    group_info: &GroupInfo,
130) -> Result<(), RegexPatternCaptureGroupsValidationError> {
131    if pattern_capture_groups.is_none() {
132        return Ok(());
133    }
134    let pattern_capture_groups = pattern_capture_groups.as_ref().unwrap();
135    if pattern_capture_groups.len() != 1 {
136        // We currently only allow one capture group
137        return Err(
138            RegexPatternCaptureGroupsValidationError::TooManyCaptureGroups(
139                pattern_capture_groups.len(),
140            ),
141        );
142    }
143    let pattern_capture_group = pattern_capture_groups.first().unwrap();
144    if !group_info
145        .all_names()
146        .filter(|(_, _, name)| name.is_some())
147        .map(|(_, _, name)| name.unwrap())
148        .any(|name| name == pattern_capture_group)
149    {
150        return Err(
151            RegexPatternCaptureGroupsValidationError::CaptureGroupNotPresent(
152                pattern_capture_group.clone(),
153            ),
154        );
155    }
156    // At this point, the capture group is in the regex, and there is exactly one.
157    // Currently, it must be called `sds_match`.
158    if pattern_capture_group != "sds_match" {
159        return Err(RegexPatternCaptureGroupsValidationError::TargetedCaptureGroupMustBeSdsMatch);
160    }
161    validate_named_capture_group_minimum_length(pattern, pattern_capture_group)?;
162    Ok(())
163}
164
165impl RuleConfig for RegexRuleConfig {
166    fn convert_to_compiled_rule(
167        &self,
168        rule_index: usize,
169        scanner_labels: Labels,
170    ) -> Result<Box<dyn CompiledRule>, CreateScannerError> {
171        let regex = get_memoized_regex(&self.pattern, validate_and_create_regex)?;
172
173        let rule_labels = scanner_labels.clone_with_labels(self.labels.clone());
174
175        let (included_keywords, excluded_keywords) = self
176            .proximity_keywords
177            .as_ref()
178            .map(|config| compile_keywords_proximity_config(config, &rule_labels))
179            .unwrap_or(Ok((None, None)))?;
180
181        is_pattern_capture_groups_valid(
182            &self.pattern,
183            &self.pattern_capture_groups,
184            regex.group_info(),
185        )?;
186
187        Ok(Box::new(RegexCompiledRule {
188            rule_index,
189            regex,
190            included_keywords,
191            excluded_keywords,
192            validator: self.validator.clone().map(|x| x.compile()),
193            metrics: RuleMetrics::new(&rule_labels),
194            pattern_capture_groups: self.pattern_capture_groups.clone(),
195        }))
196    }
197
198    fn as_regex_rule(&self) -> Option<&RegexRuleConfig> {
199        Some(self)
200    }
201}
202
203#[serde_as]
204#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
205pub struct ProximityKeywordsConfig {
206    pub look_ahead_character_count: usize,
207
208    #[serde_as(deserialize_as = "DefaultOnNull")]
209    #[serde(default)]
210    pub included_keywords: Vec<String>,
211
212    #[serde_as(deserialize_as = "DefaultOnNull")]
213    #[serde(default)]
214    pub excluded_keywords: Vec<String>,
215}
216
217#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, EnumIter, AsRefStr)]
218#[serde(tag = "type")]
219pub enum SecondaryValidator {
220    AbaRtnChecksum,
221    AtlassianTokenChecksum,
222    AustralianMedicareChecksum,
223    AustralianTfnChecksum,
224    AustrianSSNChecksum,
225    BelgiumNationalRegisterChecksum,
226    BrazilianCnpjChecksum,
227    BrazilianCpfChecksum,
228    BtcChecksum,
229    BulgarianEGNChecksum,
230    ChineseIdChecksum,
231    CoordinationNumberChecksum,
232    CzechPersonalIdentificationNumberChecksum,
233    CzechTaxIdentificationNumberChecksum,
234    DutchBsnChecksum,
235    DutchPassportChecksum,
236    EntropyCheck,
237    EstoniaPersonalCodeChecksum,
238    EthereumChecksum,
239    FinnishHetuChecksum,
240    FranceNifChecksum,
241    FranceSsnChecksum,
242    GermanIdsChecksum,
243    GermanSvnrChecksum,
244    GithubTokenChecksum,
245    GreeceAmkaChecksum,
246    GreekTinChecksum,
247    HungarianTinChecksum,
248    IbanChecker,
249    IrishPpsChecksum,
250    ItalianNationalIdChecksum,
251    JapanCorporateNumberChecksum,
252    JapanDrivingLicenseChecksum,
253    JapanMyNumberChecksum,
254    JwtClaimsValidator {
255        config: JwtClaimsValidatorConfig,
256    },
257    JwtExpirationChecker,
258    LatviaNationalIdChecksum,
259    LithuanianPersonalIdentificationNumberChecksum,
260    LuhnChecksum,
261    LuxembourgIndividualNINChecksum,
262    Mod11_10checksum,
263    Mod11_2checksum,
264    Mod1271_36Checksum,
265    Mod27_26checksum,
266    Mod37_2checksum,
267    Mod37_36checksum,
268    Mod661_26checksum,
269    Mod97_10checksum,
270    MoneroAddress,
271    NhsCheckDigit,
272    NirChecksum,
273    NonHexChecker,
274    #[cfg(feature = "token-efficiency-checker")]
275    NonHexPlusTokenEfficiencyChecker,
276    PolishNationalIdChecksum,
277    PolishNipChecksum,
278    PortugueseTaxIdChecksum,
279    RodneCisloNumberChecksum,
280    RomanianPersonalNumericCode,
281    SingaporeNricChecksum,
282    SloveniaTinChecksum,
283    SlovenianPINChecksum,
284    SpanishDniChecksum,
285    SpanishNussChecksum,
286    SwedenPINChecksum,
287    ThaiNationalIdChecksum,
288    #[cfg(feature = "token-efficiency-checker")]
289    TokenEfficiencyCheck,
290    UkNinoFormatCheck,
291    UkTrnChecksum,
292    UsDeaChecksum,
293    UsNpiChecksum,
294    VerhoeffChecksum,
295    VinChecksum,
296}
297
298#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
299#[serde(tag = "type", content = "config")]
300pub enum ClaimRequirement {
301    /// Just check that the claim exists
302    Present,
303    /// Check that the claim exists and is not expired
304    NotExpired,
305    /// Check that the claim exists and has an exact value
306    ExactValue(String),
307    /// Check that the claim exists and matches a regex pattern
308    RegexMatch(String),
309}
310
311#[derive(Serialize, Deserialize, Default, Clone, Debug, PartialEq)]
312pub struct JwtClaimsValidatorConfig {
313    #[serde(default)]
314    pub required_headers: std::collections::BTreeMap<String, ClaimRequirement>,
315    #[serde(default)]
316    pub required_claims: std::collections::BTreeMap<String, ClaimRequirement>,
317}
318
319#[cfg(test)]
320mod test {
321    use crate::{AwsType, CustomHttpConfig, MatchAction, MatchValidationType, RootRuleConfig};
322    use std::collections::BTreeMap;
323    use strum::IntoEnumIterator;
324
325    use super::*;
326
327    #[test]
328    fn should_override_pattern() {
329        let rule_config = RegexRuleConfig::new("123").with_pattern("456");
330        assert_eq!(rule_config.pattern, "456");
331    }
332
333    #[test]
334    #[allow(deprecated)]
335    fn should_have_default() {
336        let rule_config = RegexRuleConfig::new("123");
337        assert_eq!(
338            rule_config,
339            RegexRuleConfig {
340                pattern: "123".to_string(),
341                proximity_keywords: None,
342                validator: None,
343                labels: Labels::empty(),
344                pattern_capture_groups: None,
345            }
346        );
347    }
348
349    #[test]
350    fn should_use_capture_group() {
351        let rule_config = RegexRuleConfig::new("hey (?<capture_group>world)")
352            .with_pattern_capture_groups(vec!["capture_group".to_string()]);
353        assert_eq!(
354            rule_config,
355            RegexRuleConfig {
356                pattern: "hey (?<capture_group>world)".to_string(),
357                proximity_keywords: None,
358                validator: None,
359                labels: Labels::empty(),
360                pattern_capture_groups: Some(vec!["capture_group".to_string()]),
361            }
362        );
363    }
364
365    #[test]
366    fn match_action_should_default_to_none_on_deserialization() {
367        let config: RootRuleConfig<RegexRuleConfig> =
368            serde_json::from_str(r#"{"pattern":"hello"}"#).unwrap();
369        assert_eq!(config.match_action, MatchAction::None);
370    }
371
372    #[test]
373    fn proximity_keywords_should_have_default() {
374        let json_config = r#"{"look_ahead_character_count": 0}"#;
375        let test: ProximityKeywordsConfig = serde_json::from_str(json_config).unwrap();
376        assert_eq!(
377            test,
378            ProximityKeywordsConfig {
379                look_ahead_character_count: 0,
380                included_keywords: vec![],
381                excluded_keywords: vec![]
382            }
383        );
384
385        let json_config = r#"{"look_ahead_character_count": 0, "excluded_keywords": null, "included_keywords": null}"#;
386        let test: ProximityKeywordsConfig = serde_json::from_str(json_config).unwrap();
387        assert_eq!(
388            test,
389            ProximityKeywordsConfig {
390                look_ahead_character_count: 0,
391                included_keywords: vec![],
392                excluded_keywords: vec![]
393            }
394        );
395    }
396
397    #[test]
398    #[allow(deprecated)]
399    fn test_third_party_active_checker() {
400        // Test setting only the new field
401        let http_config = CustomHttpConfig::default().with_endpoint("http://test.com".to_string());
402        let validation_type = MatchValidationType::CustomHttp(http_config.clone());
403        let rule_config = RootRuleConfig::new(RegexRuleConfig::new("123"))
404            .third_party_active_checker(validation_type.clone());
405
406        assert_eq!(
407            rule_config.third_party_active_checker,
408            Some(validation_type.clone())
409        );
410        assert_eq!(rule_config.match_validation_type, None);
411        assert_eq!(
412            rule_config.get_third_party_active_checker(),
413            Some(&validation_type)
414        );
415
416        // Test setting via deprecated field updates both
417        let aws_type = AwsType::AwsId;
418        let validation_type2 = MatchValidationType::Aws(aws_type);
419        let rule_config = RootRuleConfig::new(RegexRuleConfig::new("123"))
420            .third_party_active_checker(validation_type2.clone());
421
422        assert_eq!(
423            rule_config.third_party_active_checker,
424            Some(validation_type2.clone())
425        );
426        assert_eq!(
427            rule_config.get_third_party_active_checker(),
428            Some(&validation_type2)
429        );
430
431        // Test that get_match_validation_type prioritizes third_party_active_checker
432        let rule_config = RootRuleConfig::new(RegexRuleConfig::new("123"))
433            .third_party_active_checker(MatchValidationType::CustomHttp(http_config.clone()));
434
435        assert_eq!(
436            rule_config.get_third_party_active_checker(),
437            Some(&MatchValidationType::CustomHttp(http_config.clone()))
438        );
439    }
440
441    #[test]
442    fn test_secondary_validator_enum_iter() {
443        // Test that we can iterate over all SecondaryValidator variants
444        let validators: Vec<SecondaryValidator> = SecondaryValidator::iter().collect();
445        // Verify some variants
446        assert!(validators.contains(&SecondaryValidator::GithubTokenChecksum));
447        assert!(validators.contains(&SecondaryValidator::JwtExpirationChecker));
448    }
449
450    #[test]
451    fn test_secondary_validator_are_sorted() {
452        let validator_names: Vec<String> = SecondaryValidator::iter()
453            .map(|a| a.as_ref().to_string())
454            .collect();
455        let mut sorted_validator_names = validator_names.clone();
456        sorted_validator_names.sort();
457        assert_eq!(
458            sorted_validator_names, validator_names,
459            "Secondary validators should be sorted by alphabetical order, but it's not the case, expected order:"
460        );
461    }
462
463    // The order has to be stable to pass linter checks. Otherwise, each instantiation will change the file
464    #[test]
465    fn test_jwt_claims_validator_config_serialization_order() {
466        // Create a config with claims in non-alphabetical order
467        let mut required_claims = BTreeMap::new();
468        required_claims.insert("zzz".to_string(), ClaimRequirement::Present);
469        required_claims.insert("exp".to_string(), ClaimRequirement::NotExpired);
470        required_claims.insert(
471            "aaa".to_string(),
472            ClaimRequirement::ExactValue("test".to_string()),
473        );
474        required_claims.insert(
475            "mmm".to_string(),
476            ClaimRequirement::RegexMatch(r"^test.*".to_string()),
477        );
478
479        let config = JwtClaimsValidatorConfig {
480            required_claims,
481            required_headers: std::collections::BTreeMap::new(),
482        };
483
484        // Serialize multiple times to ensure stable order
485        let serialized1 = serde_json::to_string(&config).unwrap();
486        let serialized2 = serde_json::to_string(&config).unwrap();
487
488        // Both serializations should be identical
489        assert_eq!(serialized1, serialized2, "Serialization should be stable");
490
491        // Keys should be in alphabetical order
492        assert!(serialized1.find("aaa").unwrap() < serialized1.find("exp").unwrap());
493        assert!(serialized1.find("exp").unwrap() < serialized1.find("mmm").unwrap());
494        assert!(serialized1.find("mmm").unwrap() < serialized1.find("zzz").unwrap());
495    }
496
497    #[test]
498    fn test_capture_groups_validation() {
499        let test_cases: Vec<(
500            &str,
501            Vec<String>,
502            Result<(), RegexPatternCaptureGroupsValidationError>,
503        )> = vec![
504            (
505                "hello (?<sds_match>world)",
506                vec!["sds_match".to_string()],
507                Ok(()),
508            ),
509            (
510                "hello (?<capture_group>world)",
511                vec!["capture_group".to_string()],
512                Err(RegexPatternCaptureGroupsValidationError::TargetedCaptureGroupMustBeSdsMatch),
513            ),
514            (
515                "hello (?<sds_match>world) and (?<another_group>world)",
516                vec!["sds_match".to_string()],
517                Ok(()),
518            ),
519            (
520                "hello (?<capture_grou>world)",
521                vec!["capture_group".to_string()],
522                Err(
523                    RegexPatternCaptureGroupsValidationError::CaptureGroupNotPresent(
524                        "capture_group".to_string(),
525                    ),
526                ),
527            ),
528            (
529                "hello (?<sds_match>d*)",
530                vec!["sds_match".to_string()],
531                Err(RegexPatternCaptureGroupsValidationError::CaptureGroupMatchesEmptyString),
532            ),
533            (
534                "hello (?<sds_match>world)",
535                vec!["sds_match".to_string(), "sds_match2".to_string()],
536                Err(RegexPatternCaptureGroupsValidationError::TooManyCaptureGroups(2)),
537            ),
538        ];
539        for (pattern, capture_groups, expected_result) in test_cases {
540            let rule_config =
541                RegexRuleConfig::new(pattern).with_pattern_capture_groups(capture_groups);
542            assert_eq!(
543                is_pattern_capture_groups_valid(
544                    &rule_config.pattern,
545                    &rule_config.pattern_capture_groups,
546                    &get_memoized_regex(pattern, validate_and_create_regex)
547                        .unwrap()
548                        .group_info()
549                ),
550                expected_result
551            );
552        }
553    }
554}
dd_sds/scanner/regex_rule/config.rs

dd_sds/scanner/regex_rule/
config.rs
