From d44a4646666bf166669a39955497e5d370011569 Mon Sep 17 00:00:00 2001 From: "M. Scott Ford" Date: Wed, 5 Apr 2023 21:02:29 -0400 Subject: [PATCH 1/7] Restore coverage collection for cucumber features Note: `simplecov_setup` has been moved to it's own directory because of the way that it needs to be specified when setting the `RUBYOPTS` environment variable. Just specifying the path to the full file path via `-r` does not work. When doing that an error is generated that the file cannot be found. Instead, the directory containing the file is added to the load path via `-I`, and then the file is required with a simple `-rsimplecov_setup`. Because this modifies the load path, having a file named `aruba.rb` in the same directory as `simplecov_setup.rb` caused problems for files that attempted to `require 'aruba'`. --- features/support/env.rb | 15 +++++++++------ .../support/{ => simplecov}/simplecov_setup.rb | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) rename features/support/{ => simplecov}/simplecov_setup.rb (89%) diff --git a/features/support/env.rb b/features/support/env.rb index 4a4e7fb06..8bc6d4749 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -1,29 +1,32 @@ $LOAD_PATH.unshift File.expand_path("../../lib", __dir__) # Has to be the first file required so that all other files show coverage information -require "simplecov" unless RUBY_PLATFORM.include?("java") +require_relative "simplecov/simplecov_setup" unless RUBY_PLATFORM.include?("java") # Standard Library require "fileutils" require "pathname" - # Gems require "aruba/cucumber" require "aruba/config/jruby" require "rspec/expectations" -Before do |test_case| +Around do |test_case, block| command_name = "#{test_case.location.file}:#{test_case.location.line} # #{test_case.name}" # Used in simplecov_setup so that each scenario has a different name and # their coverage results are merged instead of overwriting each other as # 'Cucumber Features' - ENV["SIMPLECOV_COMMAND_NAME"] = command_name.to_s + set_environment_variable 'SIMPLECOV_COMMAND_NAME', command_name.to_s simplecov_setup_pathname = - Pathname.new(__FILE__).expand_path.parent.join("simplecov_setup") + Pathname.new(__FILE__).expand_path.parent.join('simplecov').to_s # set environment variable so child processes will merge their coverage data # with parent process's coverage data. - ENV["RUBYOPT"] = "-r#{simplecov_setup_pathname} #{ENV['RUBYOPT']}" + prepend_environment_variable 'RUBYOPT', "-I#{simplecov_setup_pathname} -rsimplecov_setup " + + with_environment do + block.call + end end diff --git a/features/support/simplecov_setup.rb b/features/support/simplecov/simplecov_setup.rb similarity index 89% rename from features/support/simplecov_setup.rb rename to features/support/simplecov/simplecov_setup.rb index 598dc8673..286a6868a 100644 --- a/features/support/simplecov_setup.rb +++ b/features/support/simplecov/simplecov_setup.rb @@ -2,7 +2,7 @@ # child processes and @in-process unless RUBY_PLATFORM.include?("java") require "simplecov" - root = File.expand_path("../..", __dir__) + root = File.expand_path("../../..", __dir__) command_name = ENV["SIMPLECOV_COMMAND_NAME"] || "Cucumber Features" SimpleCov.command_name(command_name) SimpleCov.root(root) From b9f98f8e5fb938d394e091539db9ae708975fbce Mon Sep 17 00:00:00 2001 From: "M. Scott Ford" Date: Wed, 5 Apr 2023 21:09:33 -0400 Subject: [PATCH 2/7] Suppress console output generated by Simplecov HTML formatter The Simplecov HTML formatter, which is the default, writes a message to the console after it has stored coverage information the `coverage` directory. There is no built in way to suppress this message. See simplecov-ruby/simplecov-html#116 and https://github.com/simplecov-ruby/simplecov-html/blob/v0.12.3/lib/simplecov-html.rb#L23..L32. The console output generated by this formatter causes failures for tests that are expecting exact command output. The approach employed here is to monkey patch the method so that it still generates the same message, but instead of writing to the console, it writes to a file in the `coverage` directory named `summary.txt`. --- features/support/simplecov/simplecov_setup.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/features/support/simplecov/simplecov_setup.rb b/features/support/simplecov/simplecov_setup.rb index 286a6868a..553ea863d 100644 --- a/features/support/simplecov/simplecov_setup.rb +++ b/features/support/simplecov/simplecov_setup.rb @@ -9,4 +9,23 @@ # Run simplecov by default SimpleCov.start unless ENV.key? "ARUBA_NO_COVERAGE" + + module SimpleCov + module Formatter + class HTMLFormatter + def format(result) + Dir[File.join(File.dirname(__FILE__), "../public/*")].each do |path| + FileUtils.cp_r(path, asset_output_path) + end + + File.open(File.join(output_path, "index.html"), "wb") do |file| + file.puts template("layout").result(binding) + end + File.open(File.join(output_path, "summary.txt"), "w") do |file| + file.puts output_message(result) + end + end + end + end + end end From 7c712517e57e95772fbdce6214368b0cf402cc61 Mon Sep 17 00:00:00 2001 From: "M. Scott Ford" Date: Wed, 5 Apr 2023 21:34:56 -0400 Subject: [PATCH 3/7] Adjusts timeouts based on the extra work that's being performed There are a few different reasons that we need to wait a little bit longer than we were doing so before. * On process start, there are more files that are being required, so we need to wait a little bit more. * During process execution, coverage data is being collected. * On process stop, there are files that are being written to the file system. --- .../steps/command/stop_command.feature | 20 +++++++++---------- .../command/stop_single_command.feature | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature b/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature index 1e7995b74..bf8ea270b 100644 --- a/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature +++ b/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature @@ -45,8 +45,8 @@ Feature: Stop commands """ Feature: Run it Scenario: Run command - Given the default aruba exit timeout is 0.2 seconds - And I wait 0.2 seconds for a command to start up + Given the default aruba exit timeout is 0.3 seconds + And I wait 0.3 seconds for a command to start up When I run `aruba-test-cli1` in background And I run `aruba-test-cli2` in background And I terminate the command started last @@ -97,8 +97,8 @@ Feature: Stop commands Background: Scenario: Run command - Given the default aruba exit timeout is 0.2 seconds - And I wait 0.2 seconds for a command to start up + Given the default aruba exit timeout is 0.3 seconds + And I wait 0.3 seconds for a command to start up When I run `aruba-test-cli1` in background And I run `aruba-test-cli2` in background And I stop the command started last @@ -145,10 +145,10 @@ Feature: Stop commands """ Feature: Run it Background: - Given the default aruba exit timeout is 0.2 seconds + Given the default aruba exit timeout is 0.3 seconds Scenario: Run command - Given I wait 0.2 seconds for a command to start up + Given I wait 0.3 seconds for a command to start up When I run `aruba-test-cli1` in background When I run `aruba-test-cli2` in background And I terminate the command "aruba-test-cli1" @@ -197,10 +197,10 @@ Feature: Stop commands """ Feature: Run it Background: - Given the default aruba exit timeout is 0.2 seconds + Given the default aruba exit timeout is 0.3 seconds Scenario: Run command - Given I wait 0.2 seconds for a command to start up + Given I wait 0.3 seconds for a command to start up When I run `aruba-test-cli1` in background And I run `aruba-test-cli2` in background When I stop the command "aruba-test-cli1" @@ -239,7 +239,7 @@ Feature: Stop commands Feature: Run it Scenario: Run command Given the default aruba stop signal is "HUP" - And the default aruba exit timeout is 0.2 seconds + And the default aruba exit timeout is 0.3 seconds When I run `aruba-test-cli` Then the exit status should be 155 """ @@ -272,7 +272,7 @@ Feature: Stop commands Feature: Run it Scenario: Run command Given the default aruba exit timeout is 0.3 seconds - And I wait 0.2 seconds for a command to start up + And I wait 0.3 seconds for a command to start up When I run `aruba-test-cli1` in background And I terminate the command started last Then the exit status should be 100 diff --git a/features/04_aruba_api/command/stop_single_command.feature b/features/04_aruba_api/command/stop_single_command.feature index 120f19221..18dfd15aa 100644 --- a/features/04_aruba_api/command/stop_single_command.feature +++ b/features/04_aruba_api/command/stop_single_command.feature @@ -27,7 +27,7 @@ Feature: Stop command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.2 do + RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.5 do before { run_command('aruba-test-cli') } before { last_command_started.stop } it { expect(last_command_started).to be_successfully_executed } @@ -52,7 +52,7 @@ Feature: Stop command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, startup_wait_time: 0.1, exit_timeout: 0.3 do + RSpec.describe 'Run command', type: :aruba, startup_wait_time: 0.1, exit_timeout: 0.5 do before { run_command('aruba-test-cli') } before { find_command('aruba-test-cli').stop } it { expect(last_command_started).to be_successfully_executed } @@ -85,7 +85,7 @@ Feature: Stop command Aruba.configure do |config| config.stop_signal = 'HUP' - config.exit_timeout = 0.2 + config.exit_timeout = 0.5 end RSpec.describe 'Run command', type: :aruba do @@ -120,7 +120,7 @@ Feature: Stop command Aruba.configure do |config| config.stop_signal = 'HUP' - config.exit_timeout = 0.2 + config.exit_timeout = 0.5 end RSpec.describe 'Run command', type: :aruba do From f8ed82ebcd18b55a390511864d9a9f37fef0fb93 Mon Sep 17 00:00:00 2001 From: "M. Scott Ford" Date: Thu, 6 Apr 2023 22:11:16 -0400 Subject: [PATCH 4/7] Adjusts timings to account for extra work being done by SImplecov --- features/02_configure_aruba/basics.feature | 12 ++++++------ .../steps/command/check_for_exit_statuses.feature | 4 ++-- .../cucumber/steps/command/stop_command.feature | 14 +++++++------- features/04_aruba_api/command/run_command.feature | 14 +++++++------- .../04_aruba_api/filesystem/use_fixtures.feature | 10 +++++----- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/features/02_configure_aruba/basics.feature b/features/02_configure_aruba/basics.feature index a95bca709..492e29555 100644 --- a/features/02_configure_aruba/basics.feature +++ b/features/02_configure_aruba/basics.feature @@ -55,7 +55,7 @@ Feature: Usage of configuration Given a file named "spec/support/aruba_config.rb" with: """ruby Aruba.configure do |config| - config.exit_timeout = 0.5 + config.exit_timeout = 0.7 end """ And a file named "spec/support/hooks.rb" with: @@ -96,7 +96,7 @@ Feature: Usage of configuration Given a file named "features/support/aruba_config.rb" with: """ruby Aruba.configure do |config| - config.exit_timeout = 0.5 + config.exit_timeout = 0.7 end """ And a file named "features/run.feature" with: @@ -107,7 +107,7 @@ Feature: Usage of configuration Then the exit status should be 0 Scenario: Slow command - When I run `aruba-test-cli 1.0` + When I run `aruba-test-cli 1.5` Then the exit status should be 128 """ When I run `cucumber` @@ -122,7 +122,7 @@ Feature: Usage of configuration Given a file named "features/support/aruba_config.rb" with: """ruby Aruba.configure do |config| - config.exit_timeout = 0.2 + config.exit_timeout = 0.7 end """ And a file named "features/support/hooks.rb" with: @@ -140,11 +140,11 @@ Feature: Usage of configuration @slow-command Scenario: Slow command known by the developer - When I run `aruba-test-cli 0.5` + When I run `aruba-test-cli 1` Then the exit status should be 0 Scenario: Slow command which might be a failure - When I run `aruba-test-cli 0.5` + When I run `aruba-test-cli 1` Then the exit status should be 128 """ When I run `cucumber` diff --git a/features/03_testing_frameworks/cucumber/steps/command/check_for_exit_statuses.feature b/features/03_testing_frameworks/cucumber/steps/command/check_for_exit_statuses.feature index d3405e762..9127ccfd1 100644 --- a/features/03_testing_frameworks/cucumber/steps/command/check_for_exit_statuses.feature +++ b/features/03_testing_frameworks/cucumber/steps/command/check_for_exit_statuses.feature @@ -95,7 +95,7 @@ Feature: Check exit status of commands """ Feature: Failing program Scenario: Run command - Given the default aruba exit timeout is 0.4 seconds + Given the default aruba exit timeout is 0.7 seconds When I successfully run `aruba-test-cli` """ When I run `cucumber` @@ -112,7 +112,7 @@ Feature: Check exit status of commands Feature: Failing program Scenario: Run command Given the default aruba exit timeout is 0 seconds - When I successfully run `aruba-test-cli` for up to 0.4 seconds + When I successfully run `aruba-test-cli` for up to 0.7 seconds """ When I run `cucumber` Then the features should all pass diff --git a/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature b/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature index bf8ea270b..6ac4f9a15 100644 --- a/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature +++ b/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature @@ -45,8 +45,8 @@ Feature: Stop commands """ Feature: Run it Scenario: Run command - Given the default aruba exit timeout is 0.3 seconds - And I wait 0.3 seconds for a command to start up + Given the default aruba exit timeout is 1.5 seconds + And I wait 0.7 seconds for a command to start up When I run `aruba-test-cli1` in background And I run `aruba-test-cli2` in background And I terminate the command started last @@ -97,8 +97,8 @@ Feature: Stop commands Background: Scenario: Run command - Given the default aruba exit timeout is 0.3 seconds - And I wait 0.3 seconds for a command to start up + Given the default aruba exit timeout is 0.7 seconds + And I wait 0.5 seconds for a command to start up When I run `aruba-test-cli1` in background And I run `aruba-test-cli2` in background And I stop the command started last @@ -239,7 +239,7 @@ Feature: Stop commands Feature: Run it Scenario: Run command Given the default aruba stop signal is "HUP" - And the default aruba exit timeout is 0.3 seconds + And the default aruba exit timeout is 0.7 seconds When I run `aruba-test-cli` Then the exit status should be 155 """ @@ -271,8 +271,8 @@ Feature: Stop commands """ Feature: Run it Scenario: Run command - Given the default aruba exit timeout is 0.3 seconds - And I wait 0.3 seconds for a command to start up + Given the default aruba exit timeout is 0.7 seconds + And I wait 0.7 seconds for a command to start up When I run `aruba-test-cli1` in background And I terminate the command started last Then the exit status should be 100 diff --git a/features/04_aruba_api/command/run_command.feature b/features/04_aruba_api/command/run_command.feature index c7c655890..133a84a8f 100644 --- a/features/04_aruba_api/command/run_command.feature +++ b/features/04_aruba_api/command/run_command.feature @@ -116,7 +116,7 @@ Feature: Run command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.1, startup_wait_time: 0.4 do + RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.5, startup_wait_time: 1.5 do before do run_command('aruba-test-cli') last_command_started.send_signal 'HUP' @@ -154,7 +154,7 @@ Feature: Run command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.4 do + RSpec.describe 'Run command', type: :aruba, exit_timeout: 1.0 do before { run_command('aruba-test-cli') } it 'runs the command with the expected results' do @@ -233,10 +233,10 @@ Feature: Run command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.1 do + RSpec.describe 'Run command', type: :aruba, exit_timeout: 1.0 do before do - run_command 'aruba-test-cli1', startup_wait_time: 0.5 - run_command 'aruba-test-cli2', startup_wait_time: 0.2 + run_command 'aruba-test-cli1', startup_wait_time: 1.0 + run_command 'aruba-test-cli2', startup_wait_time: 0.7 last_command_started.send_signal 'HUP' end @@ -289,8 +289,8 @@ Feature: Run command RSpec.describe 'Run command', type: :aruba do before do - run_command 'aruba-test-cli1', exit_timeout: 0.4 - run_command 'aruba-test-cli2', exit_timeout: 0.1 + run_command 'aruba-test-cli1', exit_timeout: 1.5 + run_command 'aruba-test-cli2', exit_timeout: 0.7 end it 'runs both commands with the expected results' do diff --git a/features/04_aruba_api/filesystem/use_fixtures.feature b/features/04_aruba_api/filesystem/use_fixtures.feature index 99075efed..29e62454d 100644 --- a/features/04_aruba_api/filesystem/use_fixtures.feature +++ b/features/04_aruba_api/filesystem/use_fixtures.feature @@ -10,7 +10,7 @@ Feature: Use fixtures in your tests Given I use a fixture named "cli-app" Scenario: Use a fixture for your tests - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_a_spec.rb" with: """ require 'spec_helper' @@ -30,7 +30,7 @@ Feature: Use fixtures in your tests Then the specs should all pass Scenario: Pass file from fixtures to your command - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_b_spec.rb" with: """ require 'spec_helper' @@ -56,7 +56,7 @@ Feature: Use fixtures in your tests Then the specs should all pass Scenario: Use a fixture for your tests in test/ - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_c_spec.rb" with: """ require 'spec_helper' @@ -76,7 +76,7 @@ Feature: Use fixtures in your tests Then the specs should all pass Scenario: Use a fixture for your tests in spec/ - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_d_spec.rb" with: """ require 'spec_helper' @@ -96,7 +96,7 @@ Feature: Use fixtures in your tests Then the specs should all pass Scenario: Use a fixture for your tests in features/ - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_e_spec.rb" with: """ require 'spec_helper' From edd59928cb33a02686af250ef78c30c623ce507a Mon Sep 17 00:00:00 2001 From: "M. Scott Ford" Date: Thu, 6 Apr 2023 22:11:54 -0400 Subject: [PATCH 5/7] Removes coverage directory before running full test suite Coverage results get skewed when a `coverage` directory is already present --- Rakefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 81d742e99..83e85470a 100644 --- a/Rakefile +++ b/Rakefile @@ -8,6 +8,7 @@ Bundler.setup require "cucumber/rake/task" require "rspec/core/rake_task" require "rubocop/rake_task" +require "rake/clean" Cucumber::Rake::Task.new do |t| t.cucumber_opts = %w(--format progress) @@ -21,8 +22,10 @@ end RSpec::Core::RakeTask.new +CLOBBER << "coverage/**/*" + desc "Run the whole test suite." -task test: [:spec, :cucumber] +task test: [:clobber, :spec, :cucumber] RuboCop::RakeTask.new From 0cbb23b8b8948544fe55a875d7c7e3456e491d46 Mon Sep 17 00:00:00 2001 From: "M. Scott Ford" Date: Fri, 7 Jul 2023 12:54:49 -0400 Subject: [PATCH 6/7] Switches to double quotes This addresses a linter error --- features/support/env.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/features/support/env.rb b/features/support/env.rb index 8bc6d4749..43bc1b0ad 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -17,14 +17,14 @@ # Used in simplecov_setup so that each scenario has a different name and # their coverage results are merged instead of overwriting each other as # 'Cucumber Features' - set_environment_variable 'SIMPLECOV_COMMAND_NAME', command_name.to_s + set_environment_variable "SIMPLECOV_COMMAND_NAME", command_name.to_s simplecov_setup_pathname = - Pathname.new(__FILE__).expand_path.parent.join('simplecov').to_s + Pathname.new(__FILE__).expand_path.parent.join("simplecov").to_s # set environment variable so child processes will merge their coverage data # with parent process's coverage data. - prepend_environment_variable 'RUBYOPT', "-I#{simplecov_setup_pathname} -rsimplecov_setup " + prepend_environment_variable "RUBYOPT", "-I#{simplecov_setup_pathname} -rsimplecov_setup " with_environment do block.call From fb37cfee132e72375138b1b2d3899c8ecafaddd4 Mon Sep 17 00:00:00 2001 From: "M. Scott Ford" Date: Fri, 7 Jul 2023 13:02:15 -0400 Subject: [PATCH 7/7] Adds class level documentation This is based on a copy of the commit message from b9f98f8e5fb938d394e091539db9ae708975fbce. --- features/support/simplecov/simplecov_setup.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/features/support/simplecov/simplecov_setup.rb b/features/support/simplecov/simplecov_setup.rb index 553ea863d..3c9c1e52f 100644 --- a/features/support/simplecov/simplecov_setup.rb +++ b/features/support/simplecov/simplecov_setup.rb @@ -12,6 +12,21 @@ module SimpleCov module Formatter + # @note this is a monkey patch to suppress console output generated by + # the Simplecov HTML formatter. + # + # The Simplecov HTML formatter, which is the default, writes a message to + # the console after it has stored coverage information the `coverage` + # directory. There is no built in way to suppress this message. See + # simplecov-ruby/simplecov-html#116 and + # https://github.com/simplecov-ruby/simplecov-html/blob/v0.12.3/lib/simplecov-html.rb#L23..L32. + # + # The console output generated by the default formatter causes failures + # for Aruba tests that are expecting exact command output. + # + # The approach employed here is to monkey patch the method so that it + # still generates the same message, but instead of writing to the console, + # it writes to a file in the `coverage` directory named `summary.txt`. class HTMLFormatter def format(result) Dir[File.join(File.dirname(__FILE__), "../public/*")].each do |path| @@ -21,6 +36,7 @@ def format(result) File.open(File.join(output_path, "index.html"), "wb") do |file| file.puts template("layout").result(binding) end + File.open(File.join(output_path, "summary.txt"), "w") do |file| file.puts output_message(result) end