Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ruby.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
bundler-args: --without development
- name: Quickdraw tests
run: bundle exec qt -t 1
run: bundle exec qt -p 1
rubocop:
runs-on: ubuntu-latest
strategy:
Expand Down
4 changes: 4 additions & 0 deletions lib/quickdraw/basic_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ def run
})
end

def seed
@runner.seed
end

def match
yield
success!
Expand Down
8 changes: 7 additions & 1 deletion lib/quickdraw/cli.rb
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ def seed=(value)
@seed = value
end

def seed_from_env
return unless ENV["SEED"]

Integer(ENV["SEED"], exception: false) || fail("SEED must be an integer, got #{ENV['SEED'].inspect}")
end

def efficiency!
if Quickdraw::Platform.m_series?
fail "You cannot use the efficiency flag with the processes flag." if @processes
Expand Down Expand Up @@ -124,7 +130,7 @@ def run_once
backtrace: @backtrace,
processes: @processes || Quickdraw::Config.processes,
files: Dir.glob(@files),
seed: @seed || Random.new_seed,
seed: @seed || seed_from_env || (Random.new_seed % (1 << 32)),
)

$quickdraw_runner.call
Expand Down
7 changes: 6 additions & 1 deletion lib/quickdraw/runner.rb
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,12 @@ def initialize(backtrace: false, processes:, files:, seed:)
@successes = Quickdraw::ConcurrentInteger.new
end

attr_reader :seed

def call
puts "Seed: #{@seed}"

Kernel.srand(@seed)
load_tests

if @processes > 1
Expand Down Expand Up @@ -75,7 +80,7 @@ def call

puts

puts "Passed: #{@successes.value} | Failed: #{@failures.size} | Errors: #{@errors.size}"
puts "Passed: #{@successes.value} | Failed: #{@failures.size} | Errors: #{@errors.size} | Seed: #{@seed}"

exit(1) if @failures.any? || @errors.any?
end
Expand Down
13 changes: 13 additions & 0 deletions test/fixtures/seed/recorder.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# frozen_string_literal: true

class SeedRecorderTest < Quickdraw::Test
20.times do |i|
test "test #{i}" do
File.open(ENV.fetch("SEED_LOG"), "a") do |log|
log.puts [Process.pid, i, seed, rand(1_000_000)].join(" ")
end

assert true
end
end
end
67 changes: 67 additions & 0 deletions test/seed.test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# frozen_string_literal: true

require "open3"
require "tmpdir"

class SeedTest < Quickdraw::Test
QT = File.expand_path("../exe/qt", __dir__)
RECORDER = File.expand_path("fixtures/seed/recorder.rb", __dir__)

def run_recorder(*args, env: {})
Dir.mktmpdir do |dir|
log = File.join(dir, "seed.log")
output, status = Open3.capture2e(env.merge("SEED_LOG" => log), RbConfig.ruby, QT, *args, RECORDER, chdir: dir)
assert(status.success?) { output }

lines = File.readlines(log).map { |it| it.split.map(&:to_i) }
{ output:, workers: lines.group_by(&:first).values.map { |it| it.map { |_, *rest| rest } } }
end
end

test "seed is readable from inside a test" do
assert_equal seed, $quickdraw_runner.seed
end

test "seed is printed at the start and in the summary" do
output = run_recorder("-p", "1", "--seed", "1234")[:output]

assert output.start_with?("Seed: 1234\n")
assert_includes output, "| Seed: 1234"
end

test "--seed sets the seed seen by tests" do
workers = run_recorder("-p", "1", "--seed", "42")[:workers]

assert_equal workers.flatten(1).map { |it| it[1] }.uniq, [42]
end

test "SEED is used when --seed is not given" do
assert_includes run_recorder("-p", "1", env: { "SEED" => "77" })[:output], "Seed: 77"
end

test "--seed takes precedence over SEED" do
assert_includes run_recorder("-p", "1", "-s", "5", env: { "SEED" => "77" })[:output], "Seed: 5"
end

test "a seed is generated when none is given" do
output = run_recorder("-p", "1", env: { "SEED" => nil })[:output]

match { output => /\ASeed: \d+$/ }
end

test "a SEED that is not an integer is refused" do
output, status = Open3.capture2e({ "SEED" => "abc" }, RbConfig.ruby, QT, RECORDER, chdir: Dir.tmpdir)

refute status.success?
assert_includes output, "SEED must be an integer"
end

test "same seed gives the same order and the same rand values" do
first = run_recorder("-p", "1", "--seed", "1234")[:workers]
second = run_recorder("-p", "1", "--seed", "1234")[:workers]
other = run_recorder("-p", "1", "--seed", "4321")[:workers]

assert_equal first, second
refute_equal first, other # with 20 tests a collision is vanishingly unlikely
end
end