debian-mirror-gitlab/derailed-benchmarks/test/derailed_benchmarks/stats_from_dir_test.rb

106 lines
2.9 KiB
Ruby
Raw Normal View History

2019-10-03 14:02:42 +05:30
# frozen_string_literal: true
require 'test_helper'
class StatsFromDirTest < ActiveSupport::TestCase
test "that it works" do
dir = fixtures_dir("stats/significant")
branch_info = {}
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
newest = stats.newest
oldest = stats.oldest
assert newest.average < oldest.average
assert_equal "winner", newest.name
assert_equal "loser", oldest.name
2019-10-07 21:24:36 +05:30
assert_in_delta 0.26, stats.d_max, 0.01
assert_in_delta 0.1730818382602285, stats.d_critical, 0.00001
2019-10-03 14:02:42 +05:30
assert_equal true, stats.significant?
assert_equal "1.0062", stats.x_faster
assert_equal "0.6131", stats.percent_faster
end
test "banner faster" do
dir = fixtures_dir("stats/significant")
branch_info = {}
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
newest = stats.newest
oldest = stats.oldest
# Test fixture for banner
2019-10-07 21:24:36 +05:30
def stats.d_max
"0.037"
end
def stats.d_critical
"0.001"
2019-10-03 14:02:42 +05:30
end
def newest.average
10.5
end
def oldest.average
11.0
end
expected = <<-EOM
[winner] "I am the new commit" - (10.5 seconds)
FASTER by:
1.0476x [older/newer]
4.5455% [(older - newer) / older * 100]
[loser] "Old commit" - (11.0 seconds)
EOM
actual = StringIO.new
stats.banner(actual)
assert_match expected, actual.string
end
test "banner slower" do
dir = fixtures_dir("stats/significant")
branch_info = {}
branch_info["loser"] = { desc: "I am the new commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "Old commit", time: Time.now - 10, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
newest = stats.newest
oldest = stats.oldest
def oldest.average
10.5
end
def newest.average
11.0
end
expected = <<-EOM
[loser] "I am the new commit" - (11.0 seconds)
SLOWER by:
0.9545x [older/newer]
-4.7619% [(older - newer) / older * 100]
[winner] "Old commit" - (10.5 seconds)
EOM
actual = StringIO.new
stats.banner(actual)
assert_match expected, actual.string
end
test "stats from samples with slightly different sizes" do
stats = DerailedBenchmarks::StatsFromDir.new({})
2019-10-07 21:24:36 +05:30
out = stats.statistical_test([100,101,102, 100, 101, 99], [1,3, 3, 2])
2019-10-03 14:02:42 +05:30
assert out[:alternative]
end
end