require_relative 'load_tasks' namespace :perf do desc "runs the performance test against two most recent commits of the current app" task :app do ENV["DERAILED_PATH_TO_LIBRARY"] = '.' Rake::Task["perf:library"].invoke end desc "runs the same test against two different branches for statistical comparison" task :library do begin DERAILED_SCRIPT_COUNT = (ENV["DERAILED_SCRIPT_COUNT"] ||= "200").to_i ENV["TEST_COUNT"] ||= "200" raise "test count must be at least 2, is set to #{DERAILED_SCRIPT_COUNT}" if DERAILED_SCRIPT_COUNT < 2 script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test" if ENV["DERAILED_PATH_TO_LIBRARY"] library_dir = ENV["DERAILED_PATH_TO_LIBRARY"] else library_dir = DerailedBenchmarks.rails_path_on_disk end raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git")) # Use either the explicit SHAs when present or grab last two SHAs from commit history # if only one SHA is given, then use it and the last SHA from commit history branch_names = [] branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"] if branch_names.length < 2 Dir.chdir(library_dir) do run!("git checkout '#{branch_names.first}'") unless branch_names.empty? branches = run!('git log --format="%H" -n 2').chomp.split($/) if branch_names.empty? branch_names = branches else branches.shift branch_names << branches.shift end end end current_library_branch = "" Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp } out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}") out_dir.mkpath branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" } branch_info = {} branch_to_sha = {} branches_to_test.each do |branch, file| Dir.chdir(library_dir) do run!("git checkout '#{branch}'") description = run!("git log --oneline --format=%B -n 1 HEAD | head -n 1").strip time_stamp = run!("git log -n 1 --pretty=format:%ci").strip # https://stackoverflow.com/a/25921837/147390 short_sha = run!("git rev-parse --short HEAD").strip branch_to_sha[branch] = short_sha branch_info[short_sha] = { desc: description, time: DateTime.parse(time_stamp), file: file } end run!("#{script}") end puts puts branches_to_test.each.with_index do |(branch, _), i| short_sha = branch_to_sha[branch] desc = branch_info[short_sha][:desc] puts "Testing #{i + 1}: #{short_sha}: #{desc}" end puts puts raise "SHAs to test must be different" if branch_info.length == 1 stats = DerailedBenchmarks::StatsFromDir.new(branch_info) puts "Env var no longer has any affect DERAILED_STOP_VALID_COUNT" if ENV["DERAILED_STOP_VALID_COUNT"] DERAILED_SCRIPT_COUNT.times do |i| puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}" branches_to_test.each do |branch, file| Dir.chdir(library_dir) { run!("git checkout '#{branch}'") } run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'") end if (i % 50).zero? puts "Intermediate result" stats.call.banner puts "Continuing execution" end end ensure if library_dir && current_library_branch puts "Resetting git dir of '#{library_dir.to_s}' to #{current_library_branch.inspect}" Dir.chdir(library_dir) do run!("git checkout '#{current_library_branch}'") end end if stats stats.call.banner result_file = out_dir + "results.txt" File.open(result_file, "w") do |f| stats.banner(f) end puts "Output: #{result_file.to_s}" end end end desc "hits the url TEST_COUNT times" task :test => [:setup] do require 'benchmark' Benchmark.bm { |x| x.report("#{TEST_COUNT} derailed requests") { TEST_COUNT.times { call_app } } } end desc "stackprof" task :stackprof => [:setup] do # [:wall, :cpu, :object] begin require 'stackprof' rescue LoadError raise "Add stackprof to your gemfile to continue `gem 'stackprof', group: :development`" end TEST_COUNT = (ENV["TEST_COUNT"] ||= "100").to_i file = "tmp/#{Time.now.iso8601}-stackprof-cpu-myapp.dump" StackProf.run(mode: :cpu, out: file) do Rake::Task["perf:test"].invoke end cmd = "stackprof #{file}" puts "Running `#{cmd}`. Execute `stackprof --help` for more info" puts `#{cmd}` end task :kernel_require_patch do require 'derailed_benchmarks/core_ext/kernel_require.rb' end desc "show memory usage caused by invoking require per gem" task :mem => [:kernel_require_patch, :setup] do puts "## Impact of `require ` on RAM" puts puts "Showing all `require ` calls that consume #{ENV['CUT_OFF']} MiB or more of RSS" puts "Configure with `CUT_OFF=0` for all entries or `CUT_OFF=5` for few entries" puts "Note: Files only count against RAM on their first load." puts " If multiple libraries require the same file, then" puts " the 'cost' only shows up under the first library" puts call_app TOP_REQUIRE.print_sorted_children end desc "outputs memory usage over time" task :mem_over_time => [:setup] do require 'get_process_mem' puts "PID: #{Process.pid}" ram = GetProcessMem.new @keep_going = true begin unless ENV["SKIP_FILE_WRITE"] ruby = `ruby -v`.chomp FileUtils.mkdir_p("tmp") file = File.open("tmp/#{Time.now.iso8601}-#{ruby}-memory-#{TEST_COUNT}-times.txt", 'w') file.sync = true end ram_thread = Thread.new do while @keep_going mb = ram.mb STDOUT.puts mb file.puts mb unless ENV["SKIP_FILE_WRITE"] sleep 5 end end TEST_COUNT.times { call_app } ensure @keep_going = false ram_thread.join file.close unless ENV["SKIP_FILE_WRITE"] end end task :ram_over_time do raise "Use mem_over_time" end desc "iterations per second" task :ips => [:setup] do require 'benchmark/ips' Benchmark.ips do |x| x.warmup = Float(ENV["IPS_WARMUP"] || 2) x.time = Float(ENV["IPS_TIME"] || 5) x.suite = ENV["IPS_SUITE"] if ENV["IPS_SUITE"] x.iterations = Integer(ENV["IPS_ITERATIONS"] || 1) x.report("ips") { call_app } end end desc "outputs GC::Profiler.report data while app is called TEST_COUNT times" task :gc => [:setup] do GC::Profiler.enable TEST_COUNT.times { call_app } GC::Profiler.report GC::Profiler.disable end desc "outputs allocated object diff after app is called TEST_COUNT times" task :allocated_objects => [:setup] do call_app GC.start GC.disable start = ObjectSpace.count_objects TEST_COUNT.times { call_app } finish = ObjectSpace.count_objects GC.enable finish.each do |k,v| puts k => (v - start[k]) / TEST_COUNT.to_f end end desc "profiles ruby allocation" task :objects => [:setup] do require 'memory_profiler' call_app GC.start num = Integer(ENV["TEST_COUNT"] || 1) opts = {} opts[:ignore_files] = /#{ENV['IGNORE_FILES_REGEXP']}/ if ENV['IGNORE_FILES_REGEXP'] opts[:allow_files] = "#{ENV['ALLOW_FILES']}" if ENV['ALLOW_FILES'] puts "Running #{num} times" report = MemoryProfiler.report(opts) do num.times { call_app } end report.pretty_print end desc "heap analyzer" task :heap => [:setup] do require 'objspace' file_name = "tmp/#{Time.now.iso8601}-heap.dump" FileUtils.mkdir_p("tmp") ObjectSpace.trace_object_allocations_start puts "Running #{ TEST_COUNT } times" TEST_COUNT.times { call_app } GC.start puts "Heap file generated: #{ file_name.inspect }" ObjectSpace.dump_all(output: File.open(file_name, 'w')) require 'heapy' Heapy::Analyzer.new(file_name).analyze puts "" puts "Run `$ heapy --help` for more options" puts "" puts "Also try uploading #{file_name.inspect} to http://tenderlove.github.io/heap-analyzer/" end def run!(cmd) out = `#{cmd}` raise "Error while running #{cmd.inspect}: #{out}" unless $?.success? out end end