dd_sds/scanner/
mod.rs

1use crate::encoding::Encoding;
2use crate::event::Event;
3use std::future::Future;
4
5use crate::match_validation::{
6    config::InternalMatchValidationType, config::MatchValidationType, match_status::MatchStatus,
7    match_validator::MatchValidator,
8};
9
10#[cfg(feature = "third-party-active-checkers")]
11use error::MatchValidatorCreationError;
12
13use self::metrics::ScannerMetrics;
14#[cfg(feature = "third-party-active-checkers")]
15use crate::match_validation::match_validator::RAYON_THREAD_POOL;
16use crate::observability::labels::Labels;
17use crate::rule_match::{InternalRuleMatch, RuleMatch};
18use crate::scanner::config::RuleConfig;
19use crate::scanner::internal_rule_match_set::InternalRuleMatchSet;
20use crate::scanner::regex_rule::compiled::RegexCompiledRule;
21use crate::scanner::regex_rule::{RegexCaches, access_regex_caches};
22use crate::scanner::scope::Scope;
23pub use crate::scanner::shared_data::SharedData;
24use crate::scanner::suppression::{CompiledSuppressions, SuppressionValidationError, Suppressions};
25use crate::scoped_ruleset::{ContentVisitor, ExclusionCheck, ScopedRuleSet};
26pub use crate::secondary_validation::Validator;
27use crate::stats::GLOBAL_STATS;
28use crate::tokio::TOKIO_RUNTIME;
29use crate::{CreateScannerError, EncodeIndices, MatchAction, Path, ScannerError};
30use ::metrics::counter;
31use ahash::AHashMap;
32use futures::executor::block_on;
33use serde::{Deserialize, Serialize};
34use serde_with::serde_as;
35use std::ops::Deref;
36use std::pin::Pin;
37use std::sync::Arc;
38use std::time::{Duration, Instant};
39use tokio::task::JoinHandle;
40use tokio::time::timeout;
41
42pub mod config;
43pub mod debug_scan;
44pub mod error;
45pub mod metrics;
46pub mod regex_rule;
47pub mod scope;
48pub mod shared_data;
49pub mod shared_pool;
50pub mod suppression;
51
52mod internal_rule_match_set;
53#[cfg(test)]
54mod test;
55
56#[derive(Clone)]
57pub struct StringMatch {
58    pub start: usize,
59    pub end: usize,
60    // The keyword that was used to match this rule. Optional, only some rules may set this value.
61    pub keyword: Option<String>,
62}
63
64pub trait MatchEmitter<T = ()> {
65    fn emit(&mut self, string_match: StringMatch) -> T;
66}
67
68// This implements MatchEmitter for mutable closures (so you can use a closure instead of a custom
69// struct that implements MatchEmitter)
70impl<F, T> MatchEmitter<T> for F
71where
72    F: FnMut(StringMatch) -> T,
73{
74    fn emit(&mut self, string_match: StringMatch) -> T {
75        // This just calls the closure (itself)
76        (self)(string_match)
77    }
78}
79
80/// The precedence of a rule. Catchall is the lowest precedence, Specific is the highest precedence.
81/// The default precedence is Specific.
82/// For rules that:
83/// - Have the same mutation priority
84/// - Match at the same index
85/// - Match the same number of characters
86///
87/// Then the rule with the highest precedence will be used.
88#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Copy, Default)]
89pub enum Precedence {
90    Catchall,
91    Generic,
92    #[default]
93    Specific,
94}
95
96#[serde_as]
97#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
98pub struct RootRuleConfig<T> {
99    #[serde(default)]
100    pub match_action: MatchAction,
101    #[serde(default)]
102    pub scope: Scope,
103    #[deprecated(note = "Use `third_party_active_checker` instead")]
104    match_validation_type: Option<MatchValidationType>,
105    third_party_active_checker: Option<MatchValidationType>,
106    suppressions: Option<Suppressions>,
107    #[serde(default)]
108    precedence: Precedence,
109    #[serde(default)]
110    pub is_supporting_rule: bool,
111    /// Raw `"key:value"` strings, matching the format used by standard rule definitions
112    /// (e.g. `sensitive_data:travis_ci_access_token`). Not parsed into a map since no
113    /// producer of these values does so either; consumers parse the entries they need.
114    #[serde(default)]
115    pub tags: Vec<String>,
116    #[serde(flatten)]
117    pub inner: T,
118}
119
120impl<T> RootRuleConfig<T>
121where
122    T: RuleConfig + 'static,
123{
124    pub fn new_dyn(inner: T) -> RootRuleConfig<Arc<dyn RuleConfig>> {
125        RootRuleConfig::new(Arc::new(inner) as Arc<dyn RuleConfig>)
126    }
127
128    pub fn into_dyn(self) -> RootRuleConfig<Arc<dyn RuleConfig>> {
129        self.map_inner(|x| Arc::new(x) as Arc<dyn RuleConfig>)
130    }
131}
132
133impl<T> RootRuleConfig<T> {
134    pub fn new(inner: T) -> Self {
135        #[allow(deprecated)]
136        Self {
137            match_action: MatchAction::None,
138            scope: Scope::all(),
139            match_validation_type: None,
140            third_party_active_checker: None,
141            suppressions: None,
142            precedence: Precedence::default(),
143            is_supporting_rule: false,
144            tags: Vec::new(),
145            inner,
146        }
147    }
148
149    pub fn map_inner<U>(self, func: impl FnOnce(T) -> U) -> RootRuleConfig<U> {
150        #[allow(deprecated)]
151        RootRuleConfig {
152            match_action: self.match_action,
153            scope: self.scope,
154            match_validation_type: self.match_validation_type,
155            third_party_active_checker: self.third_party_active_checker,
156            suppressions: self.suppressions,
157            precedence: self.precedence,
158            is_supporting_rule: self.is_supporting_rule,
159            tags: self.tags,
160            inner: func(self.inner),
161        }
162    }
163
164    pub fn match_action(mut self, action: MatchAction) -> Self {
165        self.match_action = action;
166        self
167    }
168
169    pub fn precedence(mut self, precedence: Precedence) -> Self {
170        self.precedence = precedence;
171        self
172    }
173
174    pub fn scope(mut self, scope: Scope) -> Self {
175        self.scope = scope;
176        self
177    }
178
179    pub fn third_party_active_checker(
180        mut self,
181        match_validation_type: MatchValidationType,
182    ) -> Self {
183        self.third_party_active_checker = Some(match_validation_type);
184        self
185    }
186
187    pub fn suppressions(mut self, suppressions: Suppressions) -> Self {
188        self.suppressions = Some(suppressions);
189        self
190    }
191
192    pub fn is_supporting_rule(mut self, value: bool) -> Self {
193        self.is_supporting_rule = value;
194        self
195    }
196
197    pub fn tags(mut self, tags: Vec<String>) -> Self {
198        self.tags = tags;
199        self
200    }
201
202    pub fn get_suppressions(&self) -> Option<&Suppressions> {
203        self.suppressions.as_ref()
204    }
205
206    fn get_third_party_active_checker(&self) -> Option<&MatchValidationType> {
207        #[allow(deprecated)]
208        self.third_party_active_checker
209            .as_ref()
210            .or(self.match_validation_type.as_ref())
211    }
212}
213
214impl<T> Deref for RootRuleConfig<T> {
215    type Target = T;
216
217    fn deref(&self) -> &Self::Target {
218        &self.inner
219    }
220}
221pub struct RootCompiledRule {
222    pub inner: Box<dyn CompiledRule>,
223    pub scope: Scope,
224    pub match_action: MatchAction,
225    pub match_validation_type: Option<MatchValidationType>,
226    pub suppressions: Option<CompiledSuppressions>,
227    pub precedence: Precedence,
228    pub is_supporting_rule: bool,
229    /// Precomputed `"{sensitive_data_category}/{sensitive_data}"` tag value derived from
230    /// the rule's `tags`, used to tag `scanning.match_count`. `None` when the rule has no
231    /// `sensitive_data` tag.
232    pub sds_rule_name: Option<String>,
233}
234
235impl RootCompiledRule {
236    pub fn internal_match_validation_type(&self) -> Option<InternalMatchValidationType> {
237        self.match_validation_type
238            .as_ref()
239            .map(|x| x.get_internal_match_validation_type())
240    }
241}
242
243impl Deref for RootCompiledRule {
244    type Target = dyn CompiledRule;
245
246    fn deref(&self) -> &Self::Target {
247        self.inner.as_ref()
248    }
249}
250
251pub struct StringMatchesCtx<'a> {
252    rule_index: usize,
253    pub regex_caches: &'a mut RegexCaches,
254    pub exclusion_check: &'a ExclusionCheck<'a>,
255    pub excluded_matches: &'a mut AHashMap<String, String>,
256    pub match_emitter: &'a mut dyn MatchEmitter,
257    pub wildcard_indices: Option<&'a Vec<(usize, usize)>>,
258    pub enable_debug_observability: bool,
259
260    // Shared Data
261    pub per_string_data: &'a mut SharedData,
262    pub per_scanner_data: &'a SharedData,
263    pub per_event_data: &'a mut SharedData,
264    pub event_id: Option<&'a str>,
265
266    // Per-scan metadata supplied by the caller via `ScanOptions::scan_metadata`.
267    pub scan_metadata: &'a AHashMap<String, String>,
268}
269
270impl StringMatchesCtx<'_> {
271    /// If a `get_string_matches` implementation needs to do any async processing (e.g. I/O),
272    /// this function can be used to return an "async job" to find matches. The return value
273    /// of `process_async` should be returned from the `get_string_matches` function. The future
274    /// passed into this function will be spawned and executed immediately without blocking
275    /// other `get_string_matches` calls. This means all the async jobs will run concurrently.
276    ///
277    /// The `ctx` available to async jobs is more restrictive than the normal `ctx` available in
278    /// `get_string_matches`. The only thing you can do is return matches. If other data is needed,
279    /// it should be accessed before `process_async` is called.
280    pub fn process_async(
281        &self,
282        func: impl for<'a> FnOnce(
283            &'a mut AsyncStringMatchesCtx,
284        )
285            -> Pin<Box<dyn Future<Output = Result<(), ScannerError>> + Send + 'a>>
286        + Send
287        + 'static,
288    ) -> RuleResult {
289        let rule_index = self.rule_index;
290
291        // The future is spawned onto the tokio runtime immediately so it starts running
292        // in the background
293        let fut = TOKIO_RUNTIME.spawn(async move {
294            let start = Instant::now();
295            let mut ctx = AsyncStringMatchesCtx {
296                rule_matches: vec![],
297            };
298            (func)(&mut ctx).await?;
299            let io_duration = start.elapsed();
300
301            Ok(AsyncRuleInfo {
302                rule_index,
303                rule_matches: ctx.rule_matches,
304                io_duration,
305            })
306        });
307
308        Ok(RuleStatus::Pending(fut))
309    }
310}
311
312pub struct AsyncStringMatchesCtx {
313    rule_matches: Vec<StringMatch>,
314}
315
316impl AsyncStringMatchesCtx {
317    pub fn emit_match(&mut self, string_match: StringMatch) {
318        self.rule_matches.push(string_match);
319    }
320}
321
322#[must_use]
323pub enum RuleStatus {
324    Done,
325    Pending(PendingRuleResult),
326}
327
328// pub type PendingRuleResult = BoxFuture<'static, Result<AsyncRuleInfo, ScannerError>>;
329pub type PendingRuleResult = JoinHandle<Result<AsyncRuleInfo, ScannerError>>;
330
331pub struct PendingRuleJob {
332    fut: PendingRuleResult,
333    path: Path<'static>,
334}
335
336pub struct AsyncRuleInfo {
337    rule_index: usize,
338    rule_matches: Vec<StringMatch>,
339    io_duration: Duration,
340}
341
342/// A rule result that cannot be async
343pub type RuleResult = Result<RuleStatus, ScannerError>;
344
345// This is the public trait that is used to define the behavior of a compiled rule.
346pub trait CompiledRule: Send + Sync {
347    fn init_per_scanner_data(&self, _per_scanner_data: &mut SharedData) {
348        // by default, no per-scanner data is initialized
349    }
350
351    fn init_per_string_data(&self, _labels: &Labels, _per_string_data: &mut SharedData) {
352        // by default, no per-string data is initialized
353    }
354
355    fn init_per_event_data(&self, _per_event_data: &mut SharedData) {
356        // by default, no per-event data is initialized
357    }
358
359    /// Called once per unblocked rule after successful traversal, before awaiting pending jobs.
360    /// This also runs when the rule had no eligible strings, so its state may be absent.
361    /// Rules can publish event-wide state here to jobs waiting for all strings to be visited.
362    /// This is not called when traversal fails; dropping per-event state must release any waiters.
363    fn finish_per_event_data(&self, _per_event_data: &mut SharedData) {}
364
365    fn get_string_matches(
366        &self,
367        content: &str,
368        path: &Path,
369        ctx: &mut StringMatchesCtx<'_>,
370    ) -> RuleResult;
371
372    // Whether a match from this rule should be excluded (marked as a false-positive)
373    // if the content of this match was found in a match from an excluded scope
374    fn should_exclude_multipass_v0(&self) -> bool {
375        // default is to NOT use Multi-pass V0
376        false
377    }
378
379    fn on_excluded_match_multipass_v0(
380        &self,
381        _path: &Path,
382        _excluded_path: &str,
383        _enable_debug_observability: bool,
384    ) {
385        // default is to do nothing
386    }
387
388    fn as_regex_rule(&self) -> Option<&RegexCompiledRule> {
389        None
390    }
391
392    fn as_regex_rule_mut(&mut self) -> Option<&mut RegexCompiledRule> {
393        None
394    }
395
396    fn allow_scanner_to_exclude_namespace(&self) -> bool {
397        true
398    }
399}
400
401impl<T> RuleConfig for Box<T>
402where
403    T: RuleConfig + ?Sized,
404{
405    fn convert_to_compiled_rule(
406        &self,
407        rule_index: usize,
408        labels: Labels,
409    ) -> Result<Box<dyn CompiledRule>, CreateScannerError> {
410        self.as_ref().convert_to_compiled_rule(rule_index, labels)
411    }
412}
413
414#[derive(Debug, PartialEq, Clone)]
415struct ScannerFeatures {
416    pub add_implicit_index_wildcards: bool,
417    pub multipass_v0_enabled: bool,
418    pub return_matches: bool,
419    pub enable_debug_observability: bool,
420}
421
422impl Default for ScannerFeatures {
423    fn default() -> Self {
424        Self {
425            add_implicit_index_wildcards: false,
426            multipass_v0_enabled: true,
427            return_matches: false,
428            enable_debug_observability: false,
429        }
430    }
431}
432
433pub struct ScanOptions {
434    // The blocked_rules_idx parameter is a list of rule indices that should be skipped for this scan.
435    // this list shall be small (<10), so a linear search is acceptable otherwise performance will be impacted.
436    pub blocked_rules_idx: Vec<usize>,
437    // The wildcarded_indices parameter is a map containing a list of tuples of (start, end) indices that should be treated as wildcards (for the message key only) per path.
438    pub wildcarded_indices: AHashMap<Path<'static>, Vec<(usize, usize)>>,
439    // Whether to validate matches using third-party validators (e.g., checksum validation for credit cards).
440    // When enabled, the scanner automatically collects match content needed for validation.
441    pub validate_matches: bool,
442    // Arbitrary key-value metadata passed through to each rule's `get_string_matches` call via
443    // `StringMatchesCtx::scan_metadata`. Rules may use this to receive per-scan context (e.g.
444    // an org identifier) without requiring changes to the `CompiledRule` trait signature.
445    pub scan_metadata: AHashMap<String, String>,
446}
447
448impl Default for ScanOptions {
449    fn default() -> Self {
450        Self {
451            blocked_rules_idx: vec![],
452            wildcarded_indices: AHashMap::new(),
453            validate_matches: false,
454            scan_metadata: AHashMap::new(),
455        }
456    }
457}
458
459pub struct ScanOptionBuilder {
460    blocked_rules_idx: Vec<usize>,
461    wildcarded_indices: AHashMap<Path<'static>, Vec<(usize, usize)>>,
462    validate_matches: bool,
463    scan_metadata: AHashMap<String, String>,
464}
465
466impl ScanOptionBuilder {
467    pub fn new() -> Self {
468        Self {
469            blocked_rules_idx: vec![],
470            wildcarded_indices: AHashMap::new(),
471            validate_matches: false,
472            scan_metadata: AHashMap::new(),
473        }
474    }
475
476    pub fn with_blocked_rules_idx(mut self, blocked_rules_idx: Vec<usize>) -> Self {
477        self.blocked_rules_idx = blocked_rules_idx;
478        self
479    }
480
481    pub fn with_wildcarded_indices(
482        mut self,
483        wildcarded_indices: AHashMap<Path<'static>, Vec<(usize, usize)>>,
484    ) -> Self {
485        self.wildcarded_indices = wildcarded_indices;
486        self
487    }
488
489    pub fn with_validate_matching(mut self, validate_matches: bool) -> Self {
490        self.validate_matches = validate_matches;
491        self
492    }
493
494    pub fn with_scan_metadata(mut self, scan_metadata: AHashMap<String, String>) -> Self {
495        self.scan_metadata = scan_metadata;
496        self
497    }
498
499    pub fn build(self) -> ScanOptions {
500        ScanOptions {
501            blocked_rules_idx: self.blocked_rules_idx,
502            wildcarded_indices: self.wildcarded_indices,
503            validate_matches: self.validate_matches,
504            scan_metadata: self.scan_metadata,
505        }
506    }
507}
508
509pub struct Scanner {
510    rules: Vec<RootCompiledRule>,
511    scoped_ruleset: ScopedRuleSet,
512    scanner_features: ScannerFeatures,
513    metrics: ScannerMetrics,
514    labels: Labels,
515    // Without third-party active checkers this map is always empty and never read, but it is kept
516    // in the struct so the type stays identical across build configurations.
517    #[cfg_attr(not(feature = "third-party-active-checkers"), allow(dead_code))]
518    match_validators_per_type: AHashMap<InternalMatchValidationType, Box<dyn MatchValidator>>,
519    per_scanner_data: SharedData,
520    async_scan_timeout: Duration,
521}
522
523impl Scanner {
524    pub fn builder(rules: &[RootRuleConfig<Arc<dyn RuleConfig>>]) -> ScannerBuilder<'_> {
525        ScannerBuilder::new(rules)
526    }
527
528    // This function scans the given event with the rules configured in the scanner.
529    // The event parameter is a mutable reference to the event that should be scanned (implemented the Event trait).
530    // The return value is a list of RuleMatch objects, which contain information about the matches that were found.
531    // This version uses default scan options (no validation, no blocked rules, no wildcarded indices).
532    pub fn scan<E: Event>(&self, event: &mut E) -> Result<Vec<RuleMatch>, ScannerError> {
533        self.scan_with_options(event, ScanOptions::default())
534    }
535
536    // This function scans the given event with the rules configured in the scanner.
537    // The event parameter is a mutable reference to the event that should be scanned (implemented the Event trait).
538    // The options parameter allows customizing the scan behavior (validation, blocked rules, etc.).
539    // The return value is a list of RuleMatch objects, which contain information about the matches that were found.
540    pub fn scan_with_options<E: Event>(
541        &self,
542        event: &mut E,
543        options: ScanOptions,
544    ) -> Result<Vec<RuleMatch>, ScannerError> {
545        let start = Instant::now();
546        let validate = options.validate_matches;
547        // Collect matches inside block_on, then run finalize_matches (which uses rayon) outside of
548        // it to avoid re-entrancy between the futures LocalPool executor and RAYON_THREAD_POOL.
549        let result = block_on(self.internal_scan_collect(event, options));
550        match result {
551            Ok((mut rule_matches, io_duration)) => {
552                self.finalize_matches(&mut rule_matches, validate);
553                self.record_metrics(&rule_matches, start, Some(io_duration));
554                Ok(rule_matches)
555            }
556            Err(e) => {
557                self.record_metrics(&[], start, None);
558                Err(e)
559            }
560        }
561    }
562
563    // This function scans the given event with the rules configured in the scanner.
564    // The event parameter is a mutable reference to the event that should be scanned (implemented the Event trait).
565    // The return value is a list of RuleMatch objects, which contain information about the matches that were found.
566    pub async fn scan_async<E: Event>(
567        &self,
568        event: &mut E,
569    ) -> Result<Vec<RuleMatch>, ScannerError> {
570        self.scan_async_with_options(event, ScanOptions::default())
571            .await
572    }
573
574    pub async fn scan_async_with_options<E: Event>(
575        &self,
576        event: &mut E,
577        options: ScanOptions,
578    ) -> Result<Vec<RuleMatch>, ScannerError> {
579        let start = Instant::now();
580        let validate = options.validate_matches;
581        let fut = self.internal_scan_collect(event, options);
582
583        // The sleep from the timeout requires being in a tokio context
584        // The guard needs to be dropped before await since the guard is !Send
585        let timeout_result = {
586            let _tokio_guard = TOKIO_RUNTIME.enter();
587            timeout(self.async_scan_timeout, fut)
588        };
589
590        let result = timeout_result.await.unwrap_or(Err(ScannerError::Transient(
591            "Async scan timeout".to_string(),
592        )));
593
594        match result {
595            Ok((mut rule_matches, io_duration)) => {
596                self.finalize_matches(&mut rule_matches, validate);
597                self.record_metrics(&rule_matches, start, Some(io_duration));
598                Ok(rule_matches)
599            }
600            Err(e) => {
601                self.record_metrics(&[], start, None);
602                Err(e)
603            }
604        }
605    }
606
607    fn record_metrics(
608        &self,
609        output_rule_matches: &[RuleMatch],
610        start: Instant,
611        io_duration: Option<Duration>,
612    ) {
613        // Add number of scanned events
614        self.metrics.num_scanned_events.increment(1);
615        // Add number of matches, tagged per rule so `sds_rule_name` can break down match counts
616        // by the rule's sensitive_data_category/sensitive_data tags.
617        let mut match_counts_by_rule: AHashMap<usize, u64> = AHashMap::new();
618        for rule_match in output_rule_matches {
619            *match_counts_by_rule
620                .entry(rule_match.rule_index)
621                .or_default() += 1;
622        }
623        for (rule_index, count) in match_counts_by_rule {
624            match self.rules[rule_index].sds_rule_name.as_deref() {
625                Some(sds_rule_name) => {
626                    let labels = self.labels.clone_with_labels(Labels::new(&[(
627                        "sds_rule_name",
628                        sds_rule_name.to_string(),
629                    )]));
630                    counter!("scanning.match_count", labels).increment(count);
631                }
632                None => self.metrics.match_count.increment(count),
633            }
634        }
635
636        if let Some(io_duration) = io_duration {
637            let total_duration = start.elapsed();
638            let cpu_duration = total_duration.saturating_sub(io_duration);
639            self.metrics
640                .cpu_duration
641                .increment(cpu_duration.as_nanos() as u64);
642        }
643    }
644
645    fn process_rule_matches<E: Event>(
646        &self,
647        event: &mut E,
648        rule_matches: InternalRuleMatchSet<E::Encoding>,
649        excluded_matches: AHashMap<String, String>,
650        output_rule_matches: &mut Vec<RuleMatch>,
651        need_match_content: bool,
652    ) {
653        if rule_matches.is_empty() {
654            return;
655        }
656        access_regex_caches(|regex_caches| {
657            for (path, mut rule_matches) in rule_matches.into_iter() {
658                // All rule matches in each inner list are for a single path, so they can be processed independently.
659                event.visit_string_mut(&path, |content| {
660                    // calculate_indices requires that matches are sorted by start index
661                    rule_matches.sort_unstable_by_key(|rule_match| rule_match.utf8_start);
662
663                    <<E as Event>::Encoding>::calculate_indices(
664                        content,
665                        rule_matches.iter_mut().map(
666                            |rule_match: &mut InternalRuleMatch<E::Encoding>| EncodeIndices {
667                                utf8_start: rule_match.utf8_start,
668                                utf8_end: rule_match.utf8_end,
669                                custom_start: &mut rule_match.custom_start,
670                                custom_end: &mut rule_match.custom_end,
671                            },
672                        ),
673                    );
674
675                    if self.scanner_features.multipass_v0_enabled {
676                        // Now that the `excluded_matches` set is fully populated, filter out any matches
677                        // that are the same as excluded matches (also known as "Multi-pass V0")
678                        rule_matches.retain(|rule_match| {
679                            if self.rules[rule_match.rule_index]
680                                .inner
681                                .should_exclude_multipass_v0()
682                            {
683                                let match_content =
684                                    &content[rule_match.utf8_start..rule_match.utf8_end];
685                                let excluded_path = excluded_matches.get(match_content);
686                                if let Some(excluded_path) = excluded_path {
687                                    self.rules[rule_match.rule_index]
688                                        .on_excluded_match_multipass_v0(
689                                            &path,
690                                            excluded_path,
691                                            self.scanner_features.enable_debug_observability,
692                                        );
693                                }
694                                excluded_path.is_none()
695                            } else {
696                                true
697                            }
698                        });
699                    }
700
701                    self.suppress_matches::<E::Encoding>(&mut rule_matches, content, regex_caches);
702
703                    self.sort_and_remove_overlapping_rules::<E::Encoding>(&mut rule_matches);
704
705                    let will_mutate = rule_matches.iter().any(|rule_match| {
706                        self.rules[rule_match.rule_index].match_action.is_mutating()
707                    });
708
709                    self.apply_match_actions(
710                        content,
711                        &path,
712                        rule_matches,
713                        output_rule_matches,
714                        need_match_content,
715                    );
716
717                    will_mutate
718                });
719            }
720        });
721    }
722
723    async fn internal_scan_collect<E: Event>(
724        &self,
725        event: &mut E,
726        options: ScanOptions,
727    ) -> Result<(Vec<RuleMatch>, Duration), ScannerError> {
728        // If validation is requested, we need to collect match content even if the scanner
729        // wasn't originally configured to return matches
730        let need_match_content = self.scanner_features.return_matches || options.validate_matches;
731        // All matches, after some (but not all) false-positives have been removed.
732        let mut rule_matches = InternalRuleMatchSet::new();
733        let mut excluded_matches = AHashMap::new();
734        let mut async_jobs = vec![];
735        let mut per_event_data = SharedData::new();
736
737        access_regex_caches(|regex_caches| {
738            self.scoped_ruleset.visit_string_rule_combinations(
739                event,
740                ScannerContentVisitor {
741                    scanner: self,
742                    regex_caches,
743                    rule_matches: &mut rule_matches,
744                    blocked_rules: &options.blocked_rules_idx,
745                    excluded_matches: &mut excluded_matches,
746                    per_event_data: &mut per_event_data,
747                    wildcarded_indexes: &options.wildcarded_indices,
748                    async_jobs: &mut async_jobs,
749                    event_id: event.get_id().map(|s| s.to_string()),
750                    scan_metadata: &options.scan_metadata,
751                },
752            )
753        })?;
754
755        for (rule_index, rule) in self.rules.iter().enumerate() {
756            if !options.blocked_rules_idx.contains(&rule_index) {
757                rule.finish_per_event_data(&mut per_event_data);
758            }
759        }
760        // Release completion senders before awaiting jobs, including unused per-event state.
761        drop(per_event_data);
762
763        // The async jobs were already spawned on the tokio runtime, so the
764        // results just need to be collected
765        let mut total_io_duration = Duration::ZERO;
766        for job in async_jobs {
767            let rule_info = job.fut.await.unwrap()?;
768            total_io_duration += rule_info.io_duration;
769            rule_matches.push_async_matches(
770                &job.path,
771                rule_info
772                    .rule_matches
773                    .into_iter()
774                    .map(|x| InternalRuleMatch::new(rule_info.rule_index, x)),
775            );
776        }
777
778        let mut output_rule_matches = vec![];
779
780        self.process_rule_matches(
781            event,
782            rule_matches,
783            excluded_matches,
784            &mut output_rule_matches,
785            need_match_content,
786        );
787
788        Ok((output_rule_matches, total_io_duration))
789    }
790
791    pub fn suppress_matches<E: Encoding>(
792        &self,
793        rule_matches: &mut Vec<InternalRuleMatch<E>>,
794        content: &str,
795        regex_caches: &mut RegexCaches,
796    ) {
797        rule_matches.retain(|rule_match| {
798            if let Some(suppressions) = &self.rules[rule_match.rule_index].suppressions {
799                let match_should_be_suppressed = suppressions.should_match_be_suppressed(
800                    &content[rule_match.utf8_start..rule_match.utf8_end],
801                    regex_caches,
802                );
803
804                if match_should_be_suppressed {
805                    self.metrics.suppressed_match_count.increment(1);
806                }
807                !match_should_be_suppressed
808            } else {
809                true
810            }
811        });
812    }
813
814    #[cfg(feature = "third-party-active-checkers")]
815    pub fn validate_matches(&self, rule_matches: &mut Vec<RuleMatch>) {
816        // Create MatchValidatorRuleMatch per match_validator_type to pass it to each match_validator
817        let mut match_validator_rule_match_per_type = AHashMap::new();
818
819        let mut validated_rule_matches = vec![];
820
821        for mut rule_match in rule_matches.drain(..) {
822            let rule = &self.rules[rule_match.rule_index];
823            if let Some(match_validation_type) = rule.internal_match_validation_type() {
824                match_validator_rule_match_per_type
825                    .entry(match_validation_type)
826                    .or_insert_with(Vec::new)
827                    .push(rule_match)
828            } else {
829                // There is no match validator for this rule, so mark it as not available.
830                rule_match.match_status.merge(MatchStatus::NotAvailable);
831                validated_rule_matches.push(rule_match);
832            }
833        }
834
835        // Skip the pool hop when there's nothing to validate.
836        if !match_validator_rule_match_per_type.is_empty() {
837            let run_validation = || {
838                use rayon::prelude::*;
839
840                match_validator_rule_match_per_type.par_iter_mut().for_each(
841                    |(match_validation_type, matches_per_type)| {
842                        let match_validator =
843                            self.match_validators_per_type.get(match_validation_type);
844                        if let Some(match_validator) = match_validator {
845                            match_validator
846                                .as_ref()
847                                .validate(matches_per_type, &self.rules)
848                        }
849                    },
850                );
851            };
852
853            // TODO(SDSP-450): move validation onto the async TOKIO_RUNTIME. It is I/O-bound
854            // (blocking HTTP to third-party checkers), so a Rayon (CPU-bound) pool is a poor
855            // fit and forces the workaround below. Async validation would remove it entirely.
856            //
857            // If we're already on a Rayon worker (e.g. the caller scans from its own parallel
858            // iterator), calling `install` here blocks the worker, and Rayon keeps it busy by
859            // stealing more of the caller's jobs — which scan and validate and steal again,
860            // recursing until the stack overflows. Running `install` on a separate OS thread
861            // parks this worker instead, breaking the recursion; validation still runs in
862            // parallel on RAYON_THREAD_POOL.
863            if rayon::current_thread_index().is_some() {
864                std::thread::scope(|s| {
865                    s.spawn(|| RAYON_THREAD_POOL.install(run_validation));
866                });
867            } else {
868                RAYON_THREAD_POOL.install(run_validation);
869            }
870        }
871
872        // Refill the rule_matches with the validated matches
873        for (_, mut matches) in match_validator_rule_match_per_type {
874            validated_rule_matches.append(&mut matches);
875        }
876
877        // Sort rule_matches by start index
878        validated_rule_matches.sort_by_key(|rule_match| rule_match.start_index);
879        *rule_matches = validated_rule_matches;
880    }
881
882    // Runs optional validation and drops supporting-rule matches from the output.
883    // Must be called OUTSIDE of any futures executor (e.g. block_on) because validate_matches
884    // uses RAYON_THREAD_POOL internally; running rayon inside block_on causes an EnterError panic
885    // when the calling thread re-enters the LocalPool executor context.
886    fn finalize_matches(&self, rule_matches: &mut Vec<RuleMatch>, validate: bool) {
887        #[cfg(feature = "third-party-active-checkers")]
888        if validate {
889            self.validate_matches(rule_matches);
890        }
891        #[cfg(not(feature = "third-party-active-checkers"))]
892        let _ = validate;
893        // Supporting rules exist only to provide template variables to CustomHttpV2 validators of
894        // other rules. Their matches must not appear in the final output. They are retained until
895        // after validate_matches so that match pairing can reference their match values.
896        rule_matches.retain(|rule_match| !self.rules[rule_match.rule_index].is_supporting_rule);
897    }
898
899    /// Apply mutations from actions, and shift indices to match the mutated values.
900    /// This assumes the matches are all from the content given, and are sorted by start index.
901    fn apply_match_actions<E: Encoding>(
902        &self,
903        content: &mut String,
904        path: &Path<'static>,
905        rule_matches: Vec<InternalRuleMatch<E>>,
906        output_rule_matches: &mut Vec<RuleMatch>,
907        need_match_content: bool,
908    ) {
909        let mut utf8_byte_delta: isize = 0;
910        let mut custom_index_delta: <E>::IndexShift = <E>::zero_shift();
911
912        for rule_match in rule_matches {
913            output_rule_matches.push(self.apply_match_actions_for_string::<E>(
914                content,
915                path.clone(),
916                rule_match,
917                &mut utf8_byte_delta,
918                &mut custom_index_delta,
919                need_match_content,
920            ));
921        }
922    }
923
924    /// This will be called once for each match of a single string. The rules must be passed in in order of the start index. Mutating rules must not overlap.
925    fn apply_match_actions_for_string<E: Encoding>(
926        &self,
927        content: &mut String,
928        path: Path<'static>,
929        rule_match: InternalRuleMatch<E>,
930        // The current difference in length between the original and mutated string
931        utf8_byte_delta: &mut isize,
932
933        // The difference between the custom index on the original string and the mutated string
934        custom_index_delta: &mut <E>::IndexShift,
935        need_match_content: bool,
936    ) -> RuleMatch {
937        let rule = &self.rules[rule_match.rule_index];
938
939        let custom_start =
940            (<E>::get_index(&rule_match.custom_start, rule_match.utf8_start) as isize
941                + <E>::get_shift(custom_index_delta, *utf8_byte_delta)) as usize;
942
943        let mut matched_content_copy = None;
944
945        if need_match_content {
946            // This copies part of the is_mutating block but is seperate since can't mix compilation condition and code condition
947            let mutated_utf8_match_start =
948                (rule_match.utf8_start as isize + *utf8_byte_delta) as usize;
949            let mutated_utf8_match_end = (rule_match.utf8_end as isize + *utf8_byte_delta) as usize;
950
951            // Matches for mutating rules must have valid indices
952            debug_assert!(content.is_char_boundary(mutated_utf8_match_start));
953            debug_assert!(content.is_char_boundary(mutated_utf8_match_end));
954
955            let matched_content = &content[mutated_utf8_match_start..mutated_utf8_match_end];
956            matched_content_copy = Some(matched_content.to_string());
957        }
958
959        if rule.match_action.is_mutating() {
960            let mutated_utf8_match_start =
961                (rule_match.utf8_start as isize + *utf8_byte_delta) as usize;
962            let mutated_utf8_match_end = (rule_match.utf8_end as isize + *utf8_byte_delta) as usize;
963
964            // Matches for mutating rules must have valid indices
965            debug_assert!(content.is_char_boundary(mutated_utf8_match_start));
966            debug_assert!(content.is_char_boundary(mutated_utf8_match_end));
967
968            let matched_content = &content[mutated_utf8_match_start..mutated_utf8_match_end];
969            if let Some(replacement) = rule.match_action.get_replacement(matched_content) {
970                let before_replacement = &matched_content[replacement.start..replacement.end];
971
972                // update indices to match the new mutated content
973                <E>::adjust_shift(
974                    custom_index_delta,
975                    before_replacement,
976                    &replacement.replacement,
977                );
978                *utf8_byte_delta +=
979                    replacement.replacement.len() as isize - before_replacement.len() as isize;
980
981                let replacement_start = mutated_utf8_match_start + replacement.start;
982                let replacement_end = mutated_utf8_match_start + replacement.end;
983                content.replace_range(replacement_start..replacement_end, &replacement.replacement);
984            }
985        }
986
987        let shift_offset = <E>::get_shift(custom_index_delta, *utf8_byte_delta);
988        let custom_end = (<E>::get_index(&rule_match.custom_end, rule_match.utf8_end) as isize
989            + shift_offset) as usize;
990
991        let rule = &self.rules[rule_match.rule_index];
992
993        let match_status: MatchStatus = if rule.match_validation_type.is_some() {
994            MatchStatus::NotChecked
995        } else {
996            MatchStatus::NotAvailable
997        };
998
999        RuleMatch {
1000            rule_index: rule_match.rule_index,
1001            path,
1002            replacement_type: rule.match_action.replacement_type(),
1003            start_index: custom_start,
1004            end_index_exclusive: custom_end,
1005            shift_offset,
1006            match_value: matched_content_copy,
1007            match_status,
1008            keyword: rule_match.keyword,
1009        }
1010    }
1011
1012    fn sort_and_remove_overlapping_rules<E: Encoding>(
1013        &self,
1014        rule_matches: &mut Vec<InternalRuleMatch<E>>,
1015    ) {
1016        // Some of the scanner code relies on the behavior here, such as the sort order and removal of overlapping mutating rules.
1017        // Be very careful if this function is modified.
1018
1019        rule_matches.sort_unstable_by(|a, b| {
1020            // Mutating rules are a higher priority (earlier in the list)
1021            let ord = self.rules[a.rule_index]
1022                .match_action
1023                .is_mutating()
1024                .cmp(&self.rules[b.rule_index].match_action.is_mutating())
1025                .reverse();
1026
1027            // Earlier start offset
1028            let ord = ord.then(a.utf8_start.cmp(&b.utf8_start));
1029
1030            // Longer matches
1031            let ord = ord.then(a.len().cmp(&b.len()).reverse());
1032
1033            // Matches with higher precedence come first
1034            let ord = ord.then(
1035                self.rules[a.rule_index]
1036                    .precedence
1037                    .cmp(&self.rules[b.rule_index].precedence)
1038                    .reverse(),
1039            );
1040
1041            // Matches from earlier rules
1042            let ord = ord.then(a.rule_index.cmp(&b.rule_index));
1043
1044            // swap the order of everything so matches can be efficiently popped off the back as they are processed
1045            ord.reverse()
1046        });
1047
1048        let mut retained_rules: Vec<InternalRuleMatch<E>> = vec![];
1049
1050        'rule_matches: while let Some(rule_match) = rule_matches.pop() {
1051            if self.rules[rule_match.rule_index].match_action.is_mutating() {
1052                // Mutating rules are kept only if they don't overlap with a previous rule.
1053                if let Some(last) = retained_rules.last()
1054                    && last.utf8_end > rule_match.utf8_start
1055                {
1056                    continue;
1057                }
1058            } else {
1059                // Only retain if it doesn't overlap with any other rule. Since mutating matches are sorted before non-mutated matches
1060                // this needs to check all retained matches (instead of just the last one)
1061                for retained_rule in &retained_rules {
1062                    if retained_rule.utf8_start < rule_match.utf8_end
1063                        && retained_rule.utf8_end > rule_match.utf8_start
1064                    {
1065                        continue 'rule_matches;
1066                    }
1067                }
1068            };
1069            retained_rules.push(rule_match);
1070        }
1071
1072        // ensure rules are sorted by start index (other parts of the library required this to function correctly)
1073        retained_rules.sort_unstable_by_key(|rule_match| rule_match.utf8_start);
1074
1075        *rule_matches = retained_rules;
1076    }
1077}
1078
1079impl Drop for Scanner {
1080    fn drop(&mut self) {
1081        let stats = &*GLOBAL_STATS;
1082        stats.scanner_deletions.increment(1);
1083        stats.decrement_total_scanners();
1084    }
1085}
1086
1087#[derive(Default)]
1088pub struct ScannerBuilder<'a> {
1089    rules: &'a [RootRuleConfig<Arc<dyn RuleConfig>>],
1090    labels: Labels,
1091    scanner_features: ScannerFeatures,
1092    async_scan_timeout: Duration,
1093}
1094
1095impl ScannerBuilder<'_> {
1096    pub fn new(rules: &[RootRuleConfig<Arc<dyn RuleConfig>>]) -> ScannerBuilder<'_> {
1097        ScannerBuilder {
1098            rules,
1099            labels: Labels::empty(),
1100            scanner_features: ScannerFeatures::default(),
1101            async_scan_timeout: Duration::from_secs(60 * 5),
1102        }
1103    }
1104
1105    pub fn labels(mut self, labels: Labels) -> Self {
1106        self.labels = labels;
1107        self
1108    }
1109
1110    pub fn with_async_scan_timeout(mut self, duration: Duration) -> Self {
1111        self.async_scan_timeout = duration;
1112        self
1113    }
1114
1115    pub fn with_implicit_wildcard_indexes_for_scopes(mut self, value: bool) -> Self {
1116        self.scanner_features.add_implicit_index_wildcards = value;
1117        self
1118    }
1119
1120    pub fn with_return_matches(mut self, value: bool) -> Self {
1121        self.scanner_features.return_matches = value;
1122        self
1123    }
1124
1125    /// Enables/Disables the Multipass V0 feature. This defaults to TRUE.
1126    /// Multipass V0 saves matches from excluded scopes, and marks any identical
1127    /// matches in included scopes as a false positive.
1128    pub fn with_multipass_v0(mut self, value: bool) -> Self {
1129        self.scanner_features.multipass_v0_enabled = value;
1130        self
1131    }
1132
1133    /// Enables/Disables debug observability features. This defaults to FALSE.
1134    /// When enabled, metrics will include additional tags (such as `sds_namespace`)
1135    /// to help debug the source of matches.
1136    pub fn with_debug_observability(mut self, value: bool) -> Self {
1137        self.scanner_features.enable_debug_observability = value;
1138        self
1139    }
1140
1141    pub fn build(self) -> Result<Scanner, CreateScannerError> {
1142        // When third-party active checkers are compiled out, no validators are ever
1143        // created; the map stays empty and `mut` would be unused.
1144        #[cfg_attr(not(feature = "third-party-active-checkers"), allow(unused_mut))]
1145        let mut match_validators_per_type = AHashMap::new();
1146
1147        #[cfg(feature = "third-party-active-checkers")]
1148        for rule in self.rules.iter() {
1149            if let Some(match_validation_type) = &rule.get_third_party_active_checker()
1150                && match_validation_type.can_create_match_validator()
1151            {
1152                let internal_type = match_validation_type.get_internal_match_validation_type();
1153                let match_validator = match_validation_type.into_match_validator();
1154                if let Ok(match_validator) = match_validator {
1155                    if !match_validators_per_type.contains_key(&internal_type) {
1156                        match_validators_per_type.insert(internal_type, match_validator);
1157                    }
1158                } else {
1159                    return Err(CreateScannerError::InvalidMatchValidator(
1160                        MatchValidatorCreationError::InternalError,
1161                    ));
1162                }
1163            }
1164        }
1165
1166        let compiled_rules = self
1167            .rules
1168            .iter()
1169            .enumerate()
1170            .map(|(rule_index, config)| {
1171                if config.is_supporting_rule && config.match_action != MatchAction::None {
1172                    return Err(CreateScannerError::SupportingRuleHasMatchAction);
1173                }
1174                let inner = config.convert_to_compiled_rule(rule_index, self.labels.clone())?;
1175                config.match_action.validate()?;
1176                let compiled_suppressions = match &config.suppressions {
1177                    Some(s) => s.compile()?,
1178                    None => None,
1179                };
1180                Ok(RootCompiledRule {
1181                    inner,
1182                    scope: config.scope.clone(),
1183                    match_action: config.match_action.clone(),
1184                    match_validation_type: config.get_third_party_active_checker().cloned(),
1185                    suppressions: compiled_suppressions,
1186                    precedence: config.precedence,
1187                    is_supporting_rule: config.is_supporting_rule,
1188                    sds_rule_name: metrics::compute_sds_rule_name(&config.tags),
1189                })
1190            })
1191            .collect::<Result<Vec<RootCompiledRule>, CreateScannerError>>()?;
1192
1193        let mut per_scanner_data = SharedData::new();
1194
1195        compiled_rules.iter().for_each(|rule| {
1196            rule.init_per_scanner_data(&mut per_scanner_data);
1197        });
1198
1199        let scoped_ruleset = ScopedRuleSet::new(
1200            &compiled_rules
1201                .iter()
1202                .map(|rule| rule.scope.clone())
1203                .collect::<Vec<_>>(),
1204        )
1205        .with_implicit_index_wildcards(self.scanner_features.add_implicit_index_wildcards);
1206
1207        {
1208            let stats = &*GLOBAL_STATS;
1209            stats.scanner_creations.increment(1);
1210            stats.increment_total_scanners();
1211        }
1212
1213        Ok(Scanner {
1214            rules: compiled_rules,
1215            scoped_ruleset,
1216            scanner_features: self.scanner_features,
1217            metrics: ScannerMetrics::new(&self.labels),
1218            match_validators_per_type,
1219            labels: self.labels,
1220            per_scanner_data,
1221            async_scan_timeout: self.async_scan_timeout,
1222        })
1223    }
1224}
1225
1226struct ScannerContentVisitor<'a, E: Encoding> {
1227    scanner: &'a Scanner,
1228    regex_caches: &'a mut RegexCaches,
1229    rule_matches: &'a mut InternalRuleMatchSet<E>,
1230    // Rules that shall be skipped for this scan
1231    // This list shall be small (<10), so a linear search is acceptable
1232    blocked_rules: &'a Vec<usize>,
1233    excluded_matches: &'a mut AHashMap<String, String>,
1234    per_event_data: &'a mut SharedData,
1235    wildcarded_indexes: &'a AHashMap<Path<'static>, Vec<(usize, usize)>>,
1236    async_jobs: &'a mut Vec<PendingRuleJob>,
1237    event_id: Option<String>,
1238    scan_metadata: &'a AHashMap<String, String>,
1239}
1240
1241impl<'a, E: Encoding> ContentVisitor<'a> for ScannerContentVisitor<'a, E> {
1242    fn visit_content<'b>(
1243        &'b mut self,
1244        path: &Path<'a>,
1245        content: &str,
1246        mut rule_visitor: crate::scoped_ruleset::RuleIndexVisitor,
1247        exclusion_check: ExclusionCheck<'b>,
1248    ) -> Result<bool, ScannerError> {
1249        // matches for a single path
1250        let mut path_rules_matches = vec![];
1251
1252        // Create a map of per rule type data that can be shared between rules of the same type
1253        let mut per_string_data = SharedData::new();
1254        let wildcard_indices_per_path = self.wildcarded_indexes.get(path);
1255
1256        rule_visitor.visit_rule_indices(|rule_index| {
1257            if self.blocked_rules.contains(&rule_index) {
1258                return Ok(());
1259            }
1260            let rule = &self.scanner.rules[rule_index];
1261            {
1262                if rule.inner.allow_scanner_to_exclude_namespace() {
1263                    // check if the path is excluded
1264                    if exclusion_check.is_excluded(rule_index) {
1265                        return Ok(());
1266                    }
1267                }
1268                // creating the emitter is basically free, it will get mostly optimized away
1269                let mut emitter = |rule_match: StringMatch| {
1270                    // This should never happen, but to ensure no empty match is ever generated
1271                    // (which may cause an infinite loop), this will panic instead.
1272                    assert_ne!(
1273                        rule_match.start, rule_match.end,
1274                        "empty match detected on rule with index {rule_index}"
1275                    );
1276                    path_rules_matches.push(InternalRuleMatch::new(rule_index, rule_match));
1277                };
1278
1279                rule.init_per_string_data(&self.scanner.labels, &mut per_string_data);
1280
1281                // TODO: move this somewhere higher?
1282                rule.init_per_event_data(self.per_event_data);
1283
1284                let mut ctx = StringMatchesCtx {
1285                    rule_index,
1286                    regex_caches: self.regex_caches,
1287                    exclusion_check: &exclusion_check,
1288                    excluded_matches: self.excluded_matches,
1289                    match_emitter: &mut emitter,
1290                    wildcard_indices: wildcard_indices_per_path,
1291                    enable_debug_observability: self
1292                        .scanner
1293                        .scanner_features
1294                        .enable_debug_observability,
1295                    per_string_data: &mut per_string_data,
1296                    per_scanner_data: &self.scanner.per_scanner_data,
1297                    per_event_data: self.per_event_data,
1298                    event_id: self.event_id.as_deref(),
1299                    scan_metadata: self.scan_metadata,
1300                };
1301
1302                let async_status = rule.get_string_matches(content, path, &mut ctx)?;
1303
1304                match async_status {
1305                    RuleStatus::Done => {
1306                        // nothing to do
1307                    }
1308                    RuleStatus::Pending(fut) => {
1309                        self.async_jobs.push(PendingRuleJob {
1310                            fut,
1311                            path: path.into_static(),
1312                        });
1313                    }
1314                }
1315            }
1316            Ok(())
1317        })?;
1318
1319        // If there are any matches, the string will need to be accessed to check for false positives from
1320        // excluded matches, any to potentially mutate the string.
1321        // If there are any async jobs, this is also true since it's not known yet whether there
1322        // will be a match
1323        let needs_to_access_content = !path_rules_matches.is_empty() || !self.async_jobs.is_empty();
1324
1325        self.rule_matches
1326            .push_sync_matches(path, path_rules_matches);
1327
1328        Ok(needs_to_access_content)
1329    }
1330}
1331
1332// Calculates the next starting position for a regex match if a the previous match is a false positive
1333fn get_next_regex_start(content: &str, regex_match: (usize, usize)) -> Option<usize> {
1334    // The next valid UTF8 char after the start of the regex match is used
1335    if let Some((i, _)) = content[regex_match.0..].char_indices().nth(1) {
1336        Some(regex_match.0 + i)
1337    } else {
1338        // There are no more chars left in the string to scan
1339        None
1340    }
1341}
1342
1343fn is_false_positive_match(
1344    regex_match_range: (usize, usize),
1345    rule: &RegexCompiledRule,
1346    content: &str,
1347    check_excluded_keywords: bool,
1348) -> bool {
1349    if check_excluded_keywords
1350        && let Some(excluded_keywords) = &rule.excluded_keywords
1351        && excluded_keywords.is_false_positive_match(content, regex_match_range.0)
1352    {
1353        return true;
1354    }
1355
1356    if let Some(validator) = rule.validator.as_ref()
1357        && !validator.is_valid_match(&content[regex_match_range.0..regex_match_range.1])
1358    {
1359        return true;
1360    }
1361    false
1362}
dd_sds/scanner/mod.rs

dd_sds/scanner/
mod.rs
