Keep tests with identical names in different files separate

allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb

@@ -26,12 +26,10 @@ def test_result(test_case)
         description_html: scenario.description,
         history_id: scenario.id,
         full_name: scenario.name,
+        test_case_id: scenario.test_case_id,
         title_path: scenario.title_path,
-        labels: parser.labels,
-        links: parser.links,
-        parameters: parser.parameters,
-        status_details: parser.status_details,
-        environment: config.environment
+        environment: config.environment,
+        **test_result_metadata(parser)
       )
     end
 
@@ -70,6 +68,15 @@ def failure_details(result)
 
     attr_reader :ast_lookup, :config
 
+    def test_result_metadata(parser)
+      {
+        labels: parser.labels,
+        links: parser.links,
+        parameters: parser.parameters,
+        status_details: parser.status_details
+      }
+    end
+
     # @param [Step] step
     # @return [Array<Allure::Attachment>]
     def step_attachments(step)

allure-cucumber/lib/allure_cucumber/models/scenario.rb

@@ -19,6 +19,12 @@ def id
       @id ||= Digest::MD5.hexdigest(test_case.inspect)
     end
 
+    # Stable scenario identity shared by all rows of an outline
+    # @return [String]
+    def test_case_id
+      @test_case_id ||= Allure::ResultUtils.test_case_id(location_file, scenario.name)
+    end
+
     # Feature name scenario belongs to
     # @return [String]
     def feature_name

allure-cucumber/spec/unit/formatter_test_case_started_spec.rb

@@ -56,6 +56,7 @@
           expect(arg.name).to eq(scenario)
           expect(arg.description).to eq("Simple scenario description")
           expect(arg.full_name).to eq(scenario)
+          expect(arg.test_case_id).to eq(Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature##{scenario}"))
           expect(arg.title_path).to eq(["#{test_tmp_dir}/features/test.feature", feature])
           expect(arg.links).to be_empty
           expect(arg.parameters).to be_empty
@@ -65,6 +66,27 @@
     end
   end
 
+  it "distinguishes identical scenario names in different feature files" do
+    source = <<~FEATURE
+      Feature: Shared feature
+
+      Scenario: Same scenario
+        Given a is 5
+    FEATURE
+    expected = [
+      Digest::MD5.hexdigest("#{test_tmp_dir}/api/features/test.feature#Same scenario"),
+      Digest::MD5.hexdigest("#{test_tmp_dir}/ui/features/test.feature#Same scenario")
+    ]
+    results = []
+    allow(lifecycle).to receive(:start_test_case) { |result| results << result }
+
+    %w[api ui].each do |directory|
+      Thread.new { CucumberHelper.new("#{test_tmp_dir}/#{directory}").execute(source) }.join
+    end
+
+    expect(results.map(&:test_case_id)).to contain_exactly(*expected)
+  end
+
   context "with allure environment" do
     let(:allure_environment) { "test" }
 
@@ -211,6 +233,8 @@
         expect(lifecycle).to(have_received(:start_test_case).twice) { |arg| examples.push(arg) }
         expect(examples[0].name).to include("Add a to b, Examples (#1)")
         expect(examples[1].name).to include("Add a to b, Examples (#2)")
+        expected_id = Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Add a to b")
+        expect(examples.map(&:test_case_id)).to all(eq(expected_id))
         expect(examples[0].parameters).to contain_exactly(
           Allure::Parameter.new("num_a", "5"),
           Allure::Parameter.new("num_b", "10"),

allure-rspec/lib/allure_rspec/formatter.rb

@@ -123,6 +123,7 @@ def test_result(example)
         description_html: "Location - #{strip_relative(parser.location)}",
         history_id: example.id,
         full_name: example.full_description,
+        test_case_id: Allure::ResultUtils.test_case_id(example.metadata[:rerun_file_path], example.full_description),
         title_path: parser.title_path,
         labels: parser.labels,
         links: parser.links,

allure-rspec/spec/rspec_runner_helper.rb

@@ -1,17 +1,24 @@
 # frozen_string_literal: true
 
 class RspecRunner
+  SPEC_FILE = "spec/test_spec.rb"
+
   def initialize(tmp_dir)
     @stdout = StringIO.new
     @stderr = StringIO.new
     @tmp_dir = tmp_dir
   end
 
+  # @param [String, Hash<String, String>] spec source or relative paths mapped to source
+  # Hash inputs run all supplied files in insertion order.
   def run(spec)
-    setup(spec)
+    files = spec.is_a?(Hash) ? spec : { SPEC_FILE => spec }
+    spec_files = files.keys.select { |f| f.end_with?("_spec.rb") }
+
+    setup(files)
 
     Dir.chdir(tmp_dir) do
-      RSpec::Core::Runner.run([spec_file, *args], @stdout, @stderr)
+      RSpec::Core::Runner.run([*spec_files, *args], @stdout, @stderr)
     end
   ensure
     write_file("#{tmp_dir}/rspec_output.txt", all_output)
@@ -25,21 +32,17 @@ def all_output
     [@stdout.string, @stderr.string].reject(&:empty?).join("\n")
   end
 
-  def setup(spec)
+  def setup(files)
     FileUtils.rm_rf(tmp_dir)
 
-    write_file("#{tmp_dir}/#{spec_file}", spec)
+    files.each { |path, source| write_file("#{tmp_dir}/#{path}", source) }
   end
 
   def write_file(path, content)
     FileUtils.mkdir_p(File.dirname(path))
     File.write(path, content)
   end
 
-  def spec_file
-    "spec/test_spec.rb"
-  end
-
   def args
     %w[
       --no-color

allure-rspec/spec/spec_helper.rb

@@ -53,15 +53,13 @@
 RSpec.shared_context("rspec runner") do
   let!(:test_tmp_dir) { |e| "tmp/#{e.full_description.tr(' ', '_')}" }
 
-  before do
+  def run_rspec(spec)
     configuration = RSpec::Core::Configuration.new
     world = RSpec::Core::World.new(configuration)
 
     allow(RSpec).to receive(:configuration).and_return(configuration)
     allow(RSpec).to receive(:world).and_return(world)
-  end
 
-  def run_rspec(spec)
     Thread.new { RspecRunner.new(test_tmp_dir).run(spec) }.join
   end
 end

allure-rspec/spec/unit/formatter_example_started_spec.rb

@@ -44,6 +44,7 @@
           expect(arg.name).to eq(spec)
           expect(arg.description).to eq("Location - #{test_tmp_dir}/spec/test_spec.rb:10")
           expect(arg.full_name).to eq("#{suite} #{spec}")
+          expect(arg.test_case_id).to eq(Digest::MD5.hexdigest("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}"))
           expect(arg.title_path).to eq(["#{test_tmp_dir}/spec/test_spec.rb", suite])
           expect(arg.links).to be_empty
           expect(arg.parameters).to be_empty

allure-rspec/spec/unit/test_case_id_spec.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+describe "RSpec test case identity" do
+  include_context "allure mock"
+  include_context "rspec runner"
+
+  let(:results) { [] }
+  let(:test_tmp_dir) { "tmp/test_case_identity" }
+  let(:spec_source) do
+    <<~RUBY
+      RSpec.describe "Shared suite" do
+        it("same description") {}
+      end
+    RUBY
+  end
+
+  before do
+    allow(lifecycle).to receive(:start_test_case) { |result| results << result }
+  end
+
+  it "distinguishes identical descriptions in different files" do
+    files = { "spec/a_spec.rb" => spec_source, "spec/b_spec.rb" => spec_source }
+    expected = [
+      Digest::MD5.hexdigest("tmp/test_case_identity/spec/a_spec.rb#Shared suite same description"),
+      Digest::MD5.hexdigest("tmp/test_case_identity/spec/b_spec.rb#Shared suite same description")
+    ]
+
+    run_rspec(files)
+
+    expect(results.map(&:test_case_id)).to contain_exactly(*expected)
+  end
+
+  it "keeps identity when source lines and example placement change" do
+    run_rspec("spec/test_spec.rb" => spec_source)
+    original = results.first
+    moved_source = <<~RUBY
+
+
+      RSpec.describe "Shared suite" do
+        it("another example") {}
+        it("same description") {}
+      end
+    RUBY
+
+    run_rspec("spec/test_spec.rb" => moved_source)
+    moved = results.last
+
+    expect(moved.test_case_id).to eq(original.test_case_id).and(
+      eq(Digest::MD5.hexdigest("tmp/test_case_identity/spec/test_spec.rb#Shared suite same description"))
+    )
+  end
+
+  it "uses the owning file for shared examples" do
+    shared_source = <<~RUBY
+      RSpec.shared_examples "shared" do
+        it("same description") {}
+      end
+    RUBY
+    consuming_source = <<~RUBY
+      require_relative "shared.rb"
+
+      RSpec.describe "Shared suite" do
+        it_behaves_like "shared"
+      end
+    RUBY
+    files = {
+      "spec/shared.rb" => shared_source,
+      "spec/a_spec.rb" => consuming_source,
+      "spec/b_spec.rb" => consuming_source
+    }
+    expected = [
+      Digest::MD5.hexdigest(
+        "tmp/test_case_identity/spec/a_spec.rb#Shared suite behaves like shared same description"
+      ),
+      Digest::MD5.hexdigest(
+        "tmp/test_case_identity/spec/b_spec.rb#Shared suite behaves like shared same description"
+      )
+    ]
+
+    expect { run_rspec(files) }.not_to output.to_stderr
+
+    expect(results.map(&:test_case_id)).to contain_exactly(*expected)
+  end
+end

allure-ruby-commons/lib/allure_ruby_commons/model/test_result.rb

@@ -9,6 +9,7 @@ class TestResult < ExecutableItem
     # @param [Hash] options
     # @option options [String] :name
     # @option options [String] :full_name
+    # @option options [String] :test_case_id
     # @option options [Array<String>] :title_path
     # @option options [String] :description
     # @option options [String] :description_html
@@ -31,6 +32,7 @@ def initialize(uuid: SecureRandom.uuid, history_id: SecureRandom.uuid, environme
       @uuid = uuid
       @history_id = history_id
       @full_name = options[:full_name] || "Unnamed"
+      @test_case_id = options[:test_case_id]
       @title_path = options[:title_path]
       @precondition = options[:precondition]
       @precondition_html = options[:precondition_html]
@@ -43,6 +45,7 @@ def initialize(uuid: SecureRandom.uuid, history_id: SecureRandom.uuid, environme
 
     attr_accessor :uuid,
                   :full_name,
+                  :test_case_id,
                   :title_path,
                   :precondition,
                   :precondition_html,

allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "pathname"
 require "socket"
 require "uri"
 
@@ -29,6 +30,15 @@ module ResultUtils
     LANGUAGE_LABEL_NAME = "language"
 
     class << self
+      # Stable test case identity independent of source line and parameter values
+      # @param [String] file_path
+      # @param [String] full_name
+      # @return [String]
+      def test_case_id(file_path, full_name)
+        relative_path = Pathname.new(File.expand_path(file_path)).relative_path_from(Pathname.pwd).to_s
+        Digest::MD5.hexdigest("#{relative_path}##{full_name}")
+      end
+
       # @param [Time] time
       # @return [Number]
       def timestamp(time = nil)

allure-ruby-commons/spec/unit/result_utils_spec.rb

@@ -64,6 +64,34 @@ def raise_multi_error
     expect(utils.issue_link("issue", "http://jira.com/123", "{}")).to eq(Allure::Link.new("issue", "issue", "http://jira.com/123"))
   end
 
+  describe ".test_case_id" do
+    it "hashes the normalized relative file and full name" do
+      paths = [
+        "spec/test_spec.rb",
+        "./spec/test_spec.rb",
+        "spec/nested/../test_spec.rb",
+        File.expand_path("spec/test_spec.rb")
+      ]
+      expected = Digest::MD5.hexdigest("spec/test_spec.rb#Suite example")
+
+      ids = paths.map { |path| utils.test_case_id(path, "Suite example") }
+
+      expect(ids).to all(eq(expected))
+    end
+
+    it "distinguishes directories and full names" do
+      inputs = [
+        ["spec/api/test_spec.rb", "Suite example"],
+        ["spec/ui/test_spec.rb", "Suite example"],
+        ["spec/api/test_spec.rb", "Suite another example"]
+      ]
+
+      ids = inputs.map { |file, name| utils.test_case_id(file, name) }
+
+      expect(ids.uniq.size).to eq(inputs.size)
+    end
+  end
+
   describe ".prepare_global_error" do
     it "builds a broken global error from an exception" do
       error.set_backtrace(["first trace line", "second trace line"])

allure-ruby-commons/spec/unit/test_result_spec.rb

@@ -132,6 +132,32 @@
     end
   end
 
+  context "test case id" do
+    it "serializes an ID supplied at initialization" do
+      result = Allure::TestResult.new(test_case_id: "stable-test-id")
+
+      serialized = result.to_hash
+
+      expect(serialized).to include("testCaseId" => "stable-test-id")
+    end
+
+    it "serializes an ID updated through the accessor" do
+      result = Allure::TestResult.new
+
+      result.test_case_id = "updated-test-id"
+
+      expect(result.to_hash).to include("testCaseId" => "updated-test-id")
+    end
+
+    it "omits an unspecified ID" do
+      result = Allure::TestResult.new
+
+      serialized = result.to_hash
+
+      expect(serialized).not_to have_key("testCaseId")
+    end
+  end
+
   context "title path" do
     let(:environment) { nil }
     let(:title_path) { ["spec/test_spec.rb", "Suite", "Nested Suite"] }