Class: Datadog::CI::TestSuite

Inherits:
ConcurrentSpan show all
Defined in:
lib/datadog/ci/test_suite.rb,
sig/datadog/ci/test_suite.rbs

Overview

Represents a single test suite.

Read here on what test suite means: https://docs.datadoghq.com/continuous_integration/explorer/?tab=testruns#suite

This object can be shared between multiple threads.

Constant Summary

Constants inherited from Span

Span::ENVIRONMENT_RUNTIME_TAGS

Instance Attribute Summary

Attributes inherited from Span

#tracer_span

Instance Method Summary collapse

Methods inherited from ConcurrentSpan

#failed!, #get_tag, #passed!, #set_default_tags, #set_environment_runtime_tags, #set_metric, #set_tag, #set_tags, #skipped!, #synchronize

Methods inherited from Span

#base_commit_sha, #clear_tag, #failed!, #failed?, #get_metric, #get_tag, #git_branch, #git_commit_message, #git_commit_sha, #git_repository_url, #git_tag, #id, #name, #original_git_commit_message, #original_git_commit_sha, #os_architecture, #os_platform, #os_version, #passed!, #passed?, #runtime_name, #runtime_version, #service, #set_default_tags, #set_environment_runtime_tags, #set_metric, #set_parameters, #set_tag, #set_tags, #skipped!, #skipped?, #source_file, #status, #test_tracing, #to_s, #trace_id, #type, #undefined?

Constructor Details

#initialize(tracer_span) ⇒ TestSuite

Returns a new instance of TestSuite.



16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/datadog/ci/test_suite.rb', line 16

def initialize(tracer_span)
  super

  @custom_impacted_files = []

  # counts how many times every test in this suite was executed with each status:
  #   { "MySuite.mytest.a:1" => { "pass" => 3, "fail" => 2 } }
  @execution_stats_per_test = {}

  # tracks final status for each test (the status that is reported after all retries):
  #   { "MySuite.mytest.a:1" => "pass" }
  @final_statuses_per_test = {}
end

Instance Method Details

#add_impacted_files(file_paths) ⇒ void

This method returns an undefined value.

Adds files that Test Impact Analysis should consider capable of impacting every test in this suite.

In test-level skipping mode, custom impacted files must be added before the first test in this suite starts. In suite-level skipping mode, they may be added until the suite finishes.

This is useful for files shared by every test that Datadog's native Ruby coverage cannot observe, such as JavaScript test setup. Calls are incremental: every call adds files for the remainder of the suite.

Paths must resolve inside the Git repository. Relative paths must be relative to the repository root. Absolute paths are also accepted.

If paths are relative to the current working directory, convert them to absolute paths with +File.expand_path+ before calling this method. Submitted paths must be lexically normalized without redundant +.+ or +..+ components. Resolution does not access the filesystem. Paths outside the repository are ignored when the coverage event is serialized.

Examples:

Register JavaScript setup shared by an RSpec suite

before(:context) do
  Datadog::CI.active_test_suite&.add_impacted_files(
    ["app/frontend/test_setup.js"]
  )
end

Parameters:

  • file_paths (Array<String>) —

    custom paths that can impact this suite

Raises:

  • (RuntimeError) —

    if test-level skipping is active and a test in this suite has already started

  • (ArgumentError) —

    if file_paths is not an Array



63
64
65
66
67
68
69
70
71
 
# File 'lib/datadog/ci/test_suite.rb', line 63

def add_impacted_files(file_paths)
  raise ArgumentError, "file_paths must be an Array" unless file_paths.is_a?(Array)

  synchronize do
    ensure_custom_impacted_files_mutable!
    @custom_impacted_files.concat(file_paths)
  end
  nil
end

#all_executions_failed?(test_id) ⇒ Boolean

Parameters:

  • (String)

Returns:

  • (Boolean) 


133
134
135
136
137
138
 
# File 'lib/datadog/ci/test_suite.rb', line 133

def all_executions_failed?(test_id)
  synchronize do
    stats = @execution_stats_per_test[test_id]
    stats && stats[Ext::Test::Status::FAIL] > 0 && stats[Ext::Test::Status::PASS] == 0
  end
end

#all_executions_passed?(test_id) ⇒ Boolean

Parameters:

  • (String)

Returns:

  • (Boolean) 


141
142
143
144
145
146
 
# File 'lib/datadog/ci/test_suite.rb', line 141

def all_executions_passed?(test_id)
  synchronize do
    stats = @execution_stats_per_test[test_id]
    stats && stats[Ext::Test::Status::PASS] > 0 && stats[Ext::Test::Status::FAIL] == 0
  end
end

#any_passed? ⇒ Boolean

Returns:

  • (Boolean) 


116
117
118
119
120
121
122
 
# File 'lib/datadog/ci/test_suite.rb', line 116

def any_passed?
  synchronize do
    @execution_stats_per_test.any? do |_, stats|
      stats[Ext::Test::Status::PASS] > 0
    end
  end
end

#any_test_retry_passed?(test_id) ⇒ Boolean

Parameters:

  • (String)

Returns:

  • (Boolean) 


125
126
127
128
129
130
 
# File 'lib/datadog/ci/test_suite.rb', line 125

def any_test_retry_passed?(test_id)
  synchronize do
    stats = @execution_stats_per_test[test_id]
    stats && stats[Ext::Test::Status::PASS] > 0
  end
end

#datadog_skip_reason ⇒ String?

Returns:

  • (String, nil) 


174
175
176
 
# File 'lib/datadog/ci/test_suite.rb', line 174

def datadog_skip_reason
  get_tag(Ext::Test::TAG_SKIP_REASON)
end

#ensure_custom_impacted_files_mutable! ⇒ void

This method returns an undefined value.



195
196
197
198
199
200
201
 
# File 'lib/datadog/ci/test_suite.rb', line 195

def ensure_custom_impacted_files_mutable!
  return unless @custom_impacted_files.frozen?

  raise(
    "Custom impacted files must be added to a test suite before the first test in the suite starts"
  )
end

#expected_test_done!(test_name) ⇒ void

This method returns an undefined value.

Parameters:

  • test_name (String) 


165
166
167
168
169
170
171
 
# File 'lib/datadog/ci/test_suite.rb', line 165

def expected_test_done!(test_name)
  synchronize do
    @expected_tests_set.delete(test_name)

    finish if @expected_tests_set.empty?
  end
end

#finish ⇒ void

This method returns an undefined value.

Finishes this test suite.



89
90
91
92
93
94
95
96
97
98
 
# File 'lib/datadog/ci/test_suite.rb', line 89

def finish
  synchronize do
    # we try to derive test suite status from execution stats if no status was set explicitly
    set_status_from_stats! if undefined?

    test_tracing.deactivate_test_suite(name)

    super
  end
end

#itr_unskippable? ⇒ Boolean

Returns:

  • (Boolean) 


189
190
191
 
# File 'lib/datadog/ci/test_suite.rb', line 189

def itr_unskippable?
  get_tag(Ext::Test::TAG_ITR_UNSKIPPABLE) == "true"
end

#lock_custom_impacted_files ⇒ Array<String>

Locks suite-level custom impacted files against further changes and returns them to Test Impact Analysis.

Returns:

  • (Array<String>) 


78
79
80
81
82
83
84
85
 
# File 'lib/datadog/ci/test_suite.rb', line 78

def lock_custom_impacted_files
  synchronize do
    unless @custom_impacted_files.frozen?
      @custom_impacted_files = @custom_impacted_files.uniq.freeze
    end
    @custom_impacted_files
  end
end

#record_test_final_status(test_id, final_status) ⇒ void

This method returns an undefined value.

Parameters:

  • test_id (String)
  • final_status (String) 


109
110
111
112
113
 
# File 'lib/datadog/ci/test_suite.rb', line 109

def record_test_final_status(test_id, final_status)
  synchronize do
    @final_statuses_per_test[test_id] = final_status
  end
end

#record_test_result(test_id, datadog_test_status) ⇒ void

This method returns an undefined value.

Parameters:

  • test_id (String)
  • datadog_test_status (String) 


101
102
103
104
105
106
 
# File 'lib/datadog/ci/test_suite.rb', line 101

def record_test_result(test_id, datadog_test_status)
  synchronize do
    @execution_stats_per_test[test_id] ||= Hash.new(0)
    @execution_stats_per_test[test_id][datadog_test_status] += 1
  end
end

#set_expected_tests!(expected_tests) ⇒ void

This method returns an undefined value.

Parameters:

  • tests (Enumerable[String]) 


156
157
158
159
160
161
162
 
# File 'lib/datadog/ci/test_suite.rb', line 156

def set_expected_tests!(expected_tests)
  synchronize do
    return if @expected_tests_set

    @expected_tests_set = Set.new(expected_tests)
  end
end

#set_status_from_stats! ⇒ void

This method returns an undefined value.



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
 
# File 'lib/datadog/ci/test_suite.rb', line 203

def set_status_from_stats!
  synchronize do
    # count how many tests have each final status
    test_suite_stats = @final_statuses_per_test.each_with_object(Hash.new(0)) do |(_test_id, final_status), acc|
      acc[final_status] += 1
    end

    # test suite is considered failed if at least one test failed
    if test_suite_stats[Ext::Test::Status::FAIL] > 0
      failed!
    # if there are no failures and no passes, it is skipped
    elsif test_suite_stats[Ext::Test::Status::PASS] == 0
      skipped!
    # otherwise we consider it passed
    else
      passed!
    end
  end
end

#should_skip? ⇒ Boolean

Returns:

  • (Boolean) 


179
180
181
 
# File 'lib/datadog/ci/test_suite.rb', line 179

def should_skip?
  skipped_by_test_impact_analysis?
end

#skipped_by_test_impact_analysis? ⇒ Boolean

Returns:

  • (Boolean) 


184
185
186
 
# File 'lib/datadog/ci/test_suite.rb', line 184

def skipped_by_test_impact_analysis?
  get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
end

#test_executed?(test_id) ⇒ Boolean

Parameters:

  • test_id (String)

Returns:

  • (Boolean) 


149
150
151
152
153
 
# File 'lib/datadog/ci/test_suite.rb', line 149

def test_executed?(test_id)
  synchronize do
    @execution_stats_per_test.key?(test_id)
  end
end