Class: Datadog::CI::TestSuite

Inherits:
ConcurrentSpan show all
Defined in:
lib/datadog/ci/test_suite.rb,
sig/datadog/ci/test_suite.rbs

Overview

Represents a single test suite.

Read here on what test suite means: https://docs.datadoghq.com/continuous_integration/explorer/?tab=testruns#suite

This object can be shared between multiple threads.

Constant Summary

Constants inherited from Span

Span::EMPTY_TAGS, Span::ENVIRONMENT_RUNTIME_TAGS

Instance Attribute Summary

Attributes inherited from Span

#tracer_span

Instance Method Summary collapse

Methods inherited from ConcurrentSpan

#clear_internal_tag, #failed!, #get_tag, #passed!, #set_default_tags, #set_environment_runtime_tags, #set_internal_metric, #set_internal_tag, #set_internal_tags, #set_metric, #set_tag, #set_tags, #skipped!, #synchronize

Methods inherited from Span

#base_commit_sha, #clear_internal_tag, #clear_tag, #failed!, #failed?, #get_metric, #get_tag, #git_branch, #git_commit_message, #git_commit_sha, #git_repository_url, #git_tag, #id, #name, #original_git_commit_message, #original_git_commit_sha, #os_architecture, #os_platform, #os_version, #passed!, #passed?, #runtime_name, #runtime_version, #service, #set_default_tags, #set_environment_runtime_tags, #set_internal_metric, #set_internal_tag, #set_internal_tags, #set_metric, #set_parameters, #set_tag, #set_tags, #shared_tags, #skipped!, #skipped?, #source_file, #status, #test_tracing, #to_s, #trace_id, #type, #undefined?

Constructor Details

#initialize(tracer_span) ⇒ TestSuite

Returns a new instance of TestSuite.



16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/datadog/ci/test_suite.rb', line 16

def initialize(tracer_span)
  super

  @custom_impacted_files = []

  # counts how many times every test in this suite was executed with each status:
  #   { "MySuite.mytest.a:1" => { "pass" => 3, "fail" => 2 } }
  @execution_stats_per_test = {}

  # tracks final status for each test (the status that is reported after all retries):
  #   { "MySuite.mytest.a:1" => "pass" }
  @final_statuses_per_test = {}
  @tests_skipped_by_tia_count = 0
end

Instance Method Details

#add_impacted_files(file_paths) ⇒ void

This method returns an undefined value.

Adds files that Test Impact Analysis should consider capable of impacting every test in this suite.

In test-level skipping mode, custom impacted files must be added before the first test in this suite starts. In suite-level skipping mode, they may be added until the suite finishes.

This is useful for files shared by every test that Datadog's native Ruby coverage cannot observe, such as JavaScript test setup. Calls are incremental: every call adds files for the remainder of the suite.

Paths must resolve inside the Git repository. Relative paths must be relative to the repository root. Absolute paths are also accepted.

If paths are relative to the current working directory, convert them to absolute paths with +File.expand_path+ before calling this method. Submitted paths must be lexically normalized without redundant +.+ or +..+ components. Resolution does not access the filesystem. Paths outside the repository are ignored when the coverage event is serialized.

Examples:

Register JavaScript setup shared by an RSpec suite

before(:context) do
  Datadog::CI.active_test_suite&.add_impacted_files(
    ["app/frontend/test_setup.js"]
  )
end

Parameters:

  • file_paths (Array<String>) —

    custom paths that can impact this suite

Raises:

  • (RuntimeError) —

    if test-level skipping is active and a test in this suite has already started

  • (ArgumentError) —

    if file_paths is not an Array



64
65
66
67
68
69
70
71
72
 
# File 'lib/datadog/ci/test_suite.rb', line 64

def add_impacted_files(file_paths)
  raise ArgumentError, "file_paths must be an Array" unless file_paths.is_a?(Array)

  synchronize do
    ensure_custom_impacted_files_mutable!
    @custom_impacted_files.concat(file_paths)
  end
  nil
end

#all_executions_failed?(test_id) ⇒ Boolean

Parameters:

  • (String)

Returns:

  • (Boolean) 


144
145
146
147
148
149
 
# File 'lib/datadog/ci/test_suite.rb', line 144

def all_executions_failed?(test_id)
  synchronize do
    stats = @execution_stats_per_test[test_id]
    stats && stats[Ext::Test::Status::FAIL] > 0 && stats[Ext::Test::Status::PASS] == 0
  end
end

#all_executions_passed?(test_id) ⇒ Boolean

Parameters:

  • (String)

Returns:

  • (Boolean) 


152
153
154
155
156
157
 
# File 'lib/datadog/ci/test_suite.rb', line 152

def all_executions_passed?(test_id)
  synchronize do
    stats = @execution_stats_per_test[test_id]
    stats && stats[Ext::Test::Status::PASS] > 0 && stats[Ext::Test::Status::FAIL] == 0
  end
end

#any_passed? ⇒ Boolean

Returns:

  • (Boolean) 


127
128
129
130
131
132
133
 
# File 'lib/datadog/ci/test_suite.rb', line 127

def any_passed?
  synchronize do
    @execution_stats_per_test.any? do |_, stats|
      stats[Ext::Test::Status::PASS] > 0
    end
  end
end

#any_test_retry_passed?(test_id) ⇒ Boolean

Parameters:

  • (String)

Returns:

  • (Boolean) 


136
137
138
139
140
141
 
# File 'lib/datadog/ci/test_suite.rb', line 136

def any_test_retry_passed?(test_id)
  synchronize do
    stats = @execution_stats_per_test[test_id]
    stats && stats[Ext::Test::Status::PASS] > 0
  end
end

#datadog_skip_reason ⇒ String?

Returns:

  • (String, nil) 


185
186
187
 
# File 'lib/datadog/ci/test_suite.rb', line 185

def datadog_skip_reason
  get_tag(Ext::Test::TAG_SKIP_REASON)
end

#ensure_custom_impacted_files_mutable! ⇒ void

This method returns an undefined value.



206
207
208
209
210
211
212
 
# File 'lib/datadog/ci/test_suite.rb', line 206

def ensure_custom_impacted_files_mutable!
  return unless @custom_impacted_files.frozen?

  raise(
    "Custom impacted files must be added to a test suite before the first test in the suite starts"
  )
end

#expected_test_done!(test_name) ⇒ void

This method returns an undefined value.

Parameters:

  • test_name (String) 


176
177
178
179
180
181
182
 
# File 'lib/datadog/ci/test_suite.rb', line 176

def expected_test_done!(test_name)
  synchronize do
    @expected_tests_set.delete(test_name)

    finish if @expected_tests_set.empty?
  end
end

#finish ⇒ void

This method returns an undefined value.

Finishes this test suite.



90
91
92
93
94
95
96
97
98
99
 
# File 'lib/datadog/ci/test_suite.rb', line 90

def finish
  synchronize do
    # we try to derive test suite status from execution stats if no status was set explicitly
    set_status_from_stats! if undefined?

    test_tracing.deactivate_test_suite(name)

    super
  end
end

#incr_tests_skipped_by_tia_count ⇒ void

This method returns an undefined value.



107
108
109
 
# File 'lib/datadog/ci/test_suite.rb', line 107

def incr_tests_skipped_by_tia_count
  synchronize { @tests_skipped_by_tia_count += 1 }
end

#itr_unskippable? ⇒ Boolean

Returns:

  • (Boolean) 


200
201
202
 
# File 'lib/datadog/ci/test_suite.rb', line 200

def itr_unskippable?
  get_tag(Ext::Test::TAG_ITR_UNSKIPPABLE) == "true"
end

#lock_custom_impacted_files ⇒ Array<String>

Locks suite-level custom impacted files against further changes and returns them to Test Impact Analysis.

Returns:

  • (Array<String>) 


79
80
81
82
83
84
85
86
 
# File 'lib/datadog/ci/test_suite.rb', line 79

def lock_custom_impacted_files
  synchronize do
    unless @custom_impacted_files.frozen?
      @custom_impacted_files = @custom_impacted_files.uniq.freeze
    end
    @custom_impacted_files
  end
end

#record_test_final_status(test_id, final_status) ⇒ void

This method returns an undefined value.

Parameters:

  • test_id (String)
  • final_status (String) 


120
121
122
123
124
 
# File 'lib/datadog/ci/test_suite.rb', line 120

def record_test_final_status(test_id, final_status)
  synchronize do
    @final_statuses_per_test[test_id] = final_status
  end
end

#record_test_result(test_id, datadog_test_status) ⇒ void

This method returns an undefined value.

Parameters:

  • test_id (String)
  • datadog_test_status (String) 


112
113
114
115
116
117
 
# File 'lib/datadog/ci/test_suite.rb', line 112

def record_test_result(test_id, datadog_test_status)
  synchronize do
    @execution_stats_per_test[test_id] ||= Hash.new(0)
    @execution_stats_per_test[test_id][datadog_test_status] += 1
  end
end

#set_expected_tests!(expected_tests) ⇒ void

This method returns an undefined value.

Parameters:

  • tests (Enumerable[String]) 


167
168
169
170
171
172
173
 
# File 'lib/datadog/ci/test_suite.rb', line 167

def set_expected_tests!(expected_tests)
  synchronize do
    return if @expected_tests_set

    @expected_tests_set = Set.new(expected_tests)
  end
end

#set_status_from_stats! ⇒ void

This method returns an undefined value.



214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
 
# File 'lib/datadog/ci/test_suite.rb', line 214

def set_status_from_stats!
  synchronize do
    # count how many tests have each final status
    test_suite_stats = @final_statuses_per_test.each_with_object(Hash.new(0)) do |(_test_id, final_status), acc|
      acc[final_status] += 1
    end

    # test suite is considered failed if at least one test failed
    if test_suite_stats[Ext::Test::Status::FAIL] > 0
      failed!
    # if there are no failures and no passes, it is skipped
    elsif test_suite_stats[Ext::Test::Status::PASS] == 0
      skipped!
    # otherwise we consider it passed
    else
      passed!
    end
  end
end

#should_skip? ⇒ Boolean

Returns:

  • (Boolean) 


190
191
192
 
# File 'lib/datadog/ci/test_suite.rb', line 190

def should_skip?
  skipped_by_test_impact_analysis?
end

#skipped_by_test_impact_analysis? ⇒ Boolean

Returns:

  • (Boolean) 


195
196
197
 
# File 'lib/datadog/ci/test_suite.rb', line 195

def skipped_by_test_impact_analysis?
  get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
end

#test_executed?(test_id) ⇒ Boolean

Parameters:

  • test_id (String)

Returns:

  • (Boolean) 


160
161
162
163
164
 
# File 'lib/datadog/ci/test_suite.rb', line 160

def test_executed?(test_id)
  synchronize do
    @execution_stats_per_test.key?(test_id)
  end
end

#tests_skipped_by_tia_count ⇒ Integer

Returns:

  • (Integer) 


102
103
104
 
# File 'lib/datadog/ci/test_suite.rb', line 102

def tests_skipped_by_tia_count
  synchronize { @tests_skipped_by_tia_count }
end