From e23ab654f1cebc489a4447285e41c5d6b275a830 Mon Sep 17 00:00:00 2001 From: andrejs Date: Tue, 3 Nov 2026 11:47:59 +0000 Subject: [PATCH] Implement test case id support --- .../allure_cucumber/models/cucumber_model.rb | 17 ++-- .../lib/allure_cucumber/models/scenario.rb | 6 ++ .../unit/formatter_test_case_started_spec.rb | 24 ++++++ allure-rspec/lib/allure_rspec/formatter.rb | 1 + allure-rspec/spec/rspec_runner_helper.rb | 19 +++-- allure-rspec/spec/spec_helper.rb | 4 +- .../unit/formatter_example_started_spec.rb | 1 + allure-rspec/spec/unit/test_case_id_spec.rb | 84 +++++++++++++++++++ .../allure_ruby_commons/model/test_result.rb | 3 + .../lib/allure_ruby_commons/result_utils.rb | 10 +++ .../spec/unit/result_utils_spec.rb | 28 +++++++ .../spec/unit/test_result_spec.rb | 26 ++++++ 12 files changed, 207 insertions(+), 16 deletions(-) create mode 100644 allure-rspec/spec/unit/test_case_id_spec.rb diff --git a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb index 18fa86e2..f08d4d94 100644 --- a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb +++ b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb @@ -26,12 +26,10 @@ def test_result(test_case) description_html: scenario.description, history_id: scenario.id, full_name: scenario.name, + test_case_id: scenario.test_case_id, title_path: scenario.title_path, - labels: parser.labels, - links: parser.links, - parameters: parser.parameters, - status_details: parser.status_details, - environment: config.environment + environment: config.environment, + **test_result_metadata(parser) ) end @@ -70,6 +68,15 @@ def failure_details(result) attr_reader :ast_lookup, :config + def test_result_metadata(parser) + { + labels: parser.labels, + links: parser.links, + parameters: parser.parameters, + status_details: parser.status_details + } + end + # @param [Step] step # @return [Array] def step_attachments(step) diff --git a/allure-cucumber/lib/allure_cucumber/models/scenario.rb b/allure-cucumber/lib/allure_cucumber/models/scenario.rb index 1c4da62b..9e3c6b29 100644 --- a/allure-cucumber/lib/allure_cucumber/models/scenario.rb +++ b/allure-cucumber/lib/allure_cucumber/models/scenario.rb @@ -19,6 +19,12 @@ def id @id ||= Digest::MD5.hexdigest(test_case.inspect) end + # Stable scenario identity shared by all rows of an outline + # @return [String] + def test_case_id + @test_case_id ||= Allure::ResultUtils.test_case_id(location_file, scenario.name) + end + # Feature name scenario belongs to # @return [String] def feature_name diff --git a/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb b/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb index 5646bfe7..ffbfbf8a 100644 --- a/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb @@ -56,6 +56,7 @@ expect(arg.name).to eq(scenario) expect(arg.description).to eq("Simple scenario description") expect(arg.full_name).to eq(scenario) + expect(arg.test_case_id).to eq(Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature##{scenario}")) expect(arg.title_path).to eq(["#{test_tmp_dir}/features/test.feature", feature]) expect(arg.links).to be_empty expect(arg.parameters).to be_empty @@ -65,6 +66,27 @@ end end + it "distinguishes identical scenario names in different feature files" do + source = <<~FEATURE + Feature: Shared feature + + Scenario: Same scenario + Given a is 5 + FEATURE + expected = [ + Digest::MD5.hexdigest("#{test_tmp_dir}/api/features/test.feature#Same scenario"), + Digest::MD5.hexdigest("#{test_tmp_dir}/ui/features/test.feature#Same scenario") + ] + results = [] + allow(lifecycle).to receive(:start_test_case) { |result| results << result } + + %w[api ui].each do |directory| + Thread.new { CucumberHelper.new("#{test_tmp_dir}/#{directory}").execute(source) }.join + end + + expect(results.map(&:test_case_id)).to contain_exactly(*expected) + end + context "with allure environment" do let(:allure_environment) { "test" } @@ -211,6 +233,8 @@ expect(lifecycle).to(have_received(:start_test_case).twice) { |arg| examples.push(arg) } expect(examples[0].name).to include("Add a to b, Examples (#1)") expect(examples[1].name).to include("Add a to b, Examples (#2)") + expected_id = Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Add a to b") + expect(examples.map(&:test_case_id)).to all(eq(expected_id)) expect(examples[0].parameters).to contain_exactly( Allure::Parameter.new("num_a", "5"), Allure::Parameter.new("num_b", "10"), diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index e6f1f39a..80cdda74 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -123,6 +123,7 @@ def test_result(example) description_html: "Location - #{strip_relative(parser.location)}", history_id: example.id, full_name: example.full_description, + test_case_id: Allure::ResultUtils.test_case_id(example.metadata[:rerun_file_path], example.full_description), title_path: parser.title_path, labels: parser.labels, links: parser.links, diff --git a/allure-rspec/spec/rspec_runner_helper.rb b/allure-rspec/spec/rspec_runner_helper.rb index 5545ba43..f6a58cae 100644 --- a/allure-rspec/spec/rspec_runner_helper.rb +++ b/allure-rspec/spec/rspec_runner_helper.rb @@ -1,17 +1,24 @@ # frozen_string_literal: true class RspecRunner + SPEC_FILE = "spec/test_spec.rb" + def initialize(tmp_dir) @stdout = StringIO.new @stderr = StringIO.new @tmp_dir = tmp_dir end + # @param [String, Hash] spec source or relative paths mapped to source + # Hash inputs run all supplied files in insertion order. def run(spec) - setup(spec) + files = spec.is_a?(Hash) ? spec : { SPEC_FILE => spec } + spec_files = files.keys.select { |f| f.end_with?("_spec.rb") } + + setup(files) Dir.chdir(tmp_dir) do - RSpec::Core::Runner.run([spec_file, *args], @stdout, @stderr) + RSpec::Core::Runner.run([*spec_files, *args], @stdout, @stderr) end ensure write_file("#{tmp_dir}/rspec_output.txt", all_output) @@ -25,10 +32,10 @@ def all_output [@stdout.string, @stderr.string].reject(&:empty?).join("\n") end - def setup(spec) + def setup(files) FileUtils.rm_rf(tmp_dir) - write_file("#{tmp_dir}/#{spec_file}", spec) + files.each { |path, source| write_file("#{tmp_dir}/#{path}", source) } end def write_file(path, content) @@ -36,10 +43,6 @@ def write_file(path, content) File.write(path, content) end - def spec_file - "spec/test_spec.rb" - end - def args %w[ --no-color diff --git a/allure-rspec/spec/spec_helper.rb b/allure-rspec/spec/spec_helper.rb index d7342844..f938dd28 100644 --- a/allure-rspec/spec/spec_helper.rb +++ b/allure-rspec/spec/spec_helper.rb @@ -53,15 +53,13 @@ RSpec.shared_context("rspec runner") do let!(:test_tmp_dir) { |e| "tmp/#{e.full_description.tr(' ', '_')}" } - before do + def run_rspec(spec) configuration = RSpec::Core::Configuration.new world = RSpec::Core::World.new(configuration) allow(RSpec).to receive(:configuration).and_return(configuration) allow(RSpec).to receive(:world).and_return(world) - end - def run_rspec(spec) Thread.new { RspecRunner.new(test_tmp_dir).run(spec) }.join end end diff --git a/allure-rspec/spec/unit/formatter_example_started_spec.rb b/allure-rspec/spec/unit/formatter_example_started_spec.rb index c993b798..9582738e 100644 --- a/allure-rspec/spec/unit/formatter_example_started_spec.rb +++ b/allure-rspec/spec/unit/formatter_example_started_spec.rb @@ -44,6 +44,7 @@ expect(arg.name).to eq(spec) expect(arg.description).to eq("Location - #{test_tmp_dir}/spec/test_spec.rb:10") expect(arg.full_name).to eq("#{suite} #{spec}") + expect(arg.test_case_id).to eq(Digest::MD5.hexdigest("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}")) expect(arg.title_path).to eq(["#{test_tmp_dir}/spec/test_spec.rb", suite]) expect(arg.links).to be_empty expect(arg.parameters).to be_empty diff --git a/allure-rspec/spec/unit/test_case_id_spec.rb b/allure-rspec/spec/unit/test_case_id_spec.rb new file mode 100644 index 00000000..3d8176e7 --- /dev/null +++ b/allure-rspec/spec/unit/test_case_id_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +describe "RSpec test case identity" do + include_context "allure mock" + include_context "rspec runner" + + let(:results) { [] } + let(:test_tmp_dir) { "tmp/test_case_identity" } + let(:spec_source) do + <<~RUBY + RSpec.describe "Shared suite" do + it("same description") {} + end + RUBY + end + + before do + allow(lifecycle).to receive(:start_test_case) { |result| results << result } + end + + it "distinguishes identical descriptions in different files" do + files = { "spec/a_spec.rb" => spec_source, "spec/b_spec.rb" => spec_source } + expected = [ + Digest::MD5.hexdigest("tmp/test_case_identity/spec/a_spec.rb#Shared suite same description"), + Digest::MD5.hexdigest("tmp/test_case_identity/spec/b_spec.rb#Shared suite same description") + ] + + run_rspec(files) + + expect(results.map(&:test_case_id)).to contain_exactly(*expected) + end + + it "keeps identity when source lines and example placement change" do + run_rspec("spec/test_spec.rb" => spec_source) + original = results.first + moved_source = <<~RUBY + + + RSpec.describe "Shared suite" do + it("another example") {} + it("same description") {} + end + RUBY + + run_rspec("spec/test_spec.rb" => moved_source) + moved = results.last + + expect(moved.test_case_id).to eq(original.test_case_id).and( + eq(Digest::MD5.hexdigest("tmp/test_case_identity/spec/test_spec.rb#Shared suite same description")) + ) + end + + it "uses the owning file for shared examples" do + shared_source = <<~RUBY + RSpec.shared_examples "shared" do + it("same description") {} + end + RUBY + consuming_source = <<~RUBY + require_relative "shared.rb" + + RSpec.describe "Shared suite" do + it_behaves_like "shared" + end + RUBY + files = { + "spec/shared.rb" => shared_source, + "spec/a_spec.rb" => consuming_source, + "spec/b_spec.rb" => consuming_source + } + expected = [ + Digest::MD5.hexdigest( + "tmp/test_case_identity/spec/a_spec.rb#Shared suite behaves like shared same description" + ), + Digest::MD5.hexdigest( + "tmp/test_case_identity/spec/b_spec.rb#Shared suite behaves like shared same description" + ) + ] + + expect { run_rspec(files) }.not_to output.to_stderr + + expect(results.map(&:test_case_id)).to contain_exactly(*expected) + end +end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/model/test_result.rb b/allure-ruby-commons/lib/allure_ruby_commons/model/test_result.rb index 10647368..56b1e372 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/model/test_result.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/model/test_result.rb @@ -9,6 +9,7 @@ class TestResult < ExecutableItem # @param [Hash] options # @option options [String] :name # @option options [String] :full_name + # @option options [String] :test_case_id # @option options [Array] :title_path # @option options [String] :description # @option options [String] :description_html @@ -31,6 +32,7 @@ def initialize(uuid: SecureRandom.uuid, history_id: SecureRandom.uuid, environme @uuid = uuid @history_id = history_id @full_name = options[:full_name] || "Unnamed" + @test_case_id = options[:test_case_id] @title_path = options[:title_path] @precondition = options[:precondition] @precondition_html = options[:precondition_html] @@ -43,6 +45,7 @@ def initialize(uuid: SecureRandom.uuid, history_id: SecureRandom.uuid, environme attr_accessor :uuid, :full_name, + :test_case_id, :title_path, :precondition, :precondition_html, diff --git a/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb b/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb index e1e72c7f..31e04aea 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "pathname" require "socket" require "uri" @@ -29,6 +30,15 @@ module ResultUtils LANGUAGE_LABEL_NAME = "language" class << self + # Stable test case identity independent of source line and parameter values + # @param [String] file_path + # @param [String] full_name + # @return [String] + def test_case_id(file_path, full_name) + relative_path = Pathname.new(File.expand_path(file_path)).relative_path_from(Pathname.pwd).to_s + Digest::MD5.hexdigest("#{relative_path}##{full_name}") + end + # @param [Time] time # @return [Number] def timestamp(time = nil) diff --git a/allure-ruby-commons/spec/unit/result_utils_spec.rb b/allure-ruby-commons/spec/unit/result_utils_spec.rb index 7c2c14dd..a08432e8 100644 --- a/allure-ruby-commons/spec/unit/result_utils_spec.rb +++ b/allure-ruby-commons/spec/unit/result_utils_spec.rb @@ -64,6 +64,34 @@ def raise_multi_error expect(utils.issue_link("issue", "http://jira.com/123", "{}")).to eq(Allure::Link.new("issue", "issue", "http://jira.com/123")) end + describe ".test_case_id" do + it "hashes the normalized relative file and full name" do + paths = [ + "spec/test_spec.rb", + "./spec/test_spec.rb", + "spec/nested/../test_spec.rb", + File.expand_path("spec/test_spec.rb") + ] + expected = Digest::MD5.hexdigest("spec/test_spec.rb#Suite example") + + ids = paths.map { |path| utils.test_case_id(path, "Suite example") } + + expect(ids).to all(eq(expected)) + end + + it "distinguishes directories and full names" do + inputs = [ + ["spec/api/test_spec.rb", "Suite example"], + ["spec/ui/test_spec.rb", "Suite example"], + ["spec/api/test_spec.rb", "Suite another example"] + ] + + ids = inputs.map { |file, name| utils.test_case_id(file, name) } + + expect(ids.uniq.size).to eq(inputs.size) + end + end + describe ".prepare_global_error" do it "builds a broken global error from an exception" do error.set_backtrace(["first trace line", "second trace line"]) diff --git a/allure-ruby-commons/spec/unit/test_result_spec.rb b/allure-ruby-commons/spec/unit/test_result_spec.rb index f4b7ad16..10f75852 100644 --- a/allure-ruby-commons/spec/unit/test_result_spec.rb +++ b/allure-ruby-commons/spec/unit/test_result_spec.rb @@ -132,6 +132,32 @@ end end + context "test case id" do + it "serializes an ID supplied at initialization" do + result = Allure::TestResult.new(test_case_id: "stable-test-id") + + serialized = result.to_hash + + expect(serialized).to include("testCaseId" => "stable-test-id") + end + + it "serializes an ID updated through the accessor" do + result = Allure::TestResult.new + + result.test_case_id = "updated-test-id" + + expect(result.to_hash).to include("testCaseId" => "updated-test-id") + end + + it "omits an unspecified ID" do + result = Allure::TestResult.new + + serialized = result.to_hash + + expect(serialized).not_to have_key("testCaseId") + end + end + context "title path" do let(:environment) { nil } let(:title_path) { ["spec/test_spec.rb", "Suite", "Nested Suite"] }