#!/usr/bin/env ruby ################################ # Requires ################################ require 'fileutils' require 'open3' require 'optparse' require 'erb' ################################ # Options ################################ @options = { branch: 'HEAD', iterations: 5, skip_clean: false, verbose: false } OptionParser.new do |opts| opts.on('--branch BRANCH', "compares the performance of BRANCH against 'master'") do |branch| @options[:branch] = branch end opts.on('--iterations N', Integer, 'iterates lint N times on each repositories') do |iterations| @options[:iterations] = iterations end opts.on('--skip-clean', 'skip cleaning on completion') do |skip_clean| @options[:skip_clean] = skip_clean end opts.on('-v', '--[no-]verbose', 'Run verbosely') do |v| @options[:verbose] = v end end.parse! ################################ # Classes ################################ class Repo attr_accessor :name attr_accessor :github_location attr_accessor :commit_hash attr_accessor :branch_exit_value attr_accessor :branch_duration attr_accessor :master_exit_value attr_accessor :master_duration def initialize(name, github_location) @name = name @github_location = github_location end def git_url "https://github.com/#{github_location}" end def to_s @name end def duration_report percent_change = 100 * (@master_duration - @branch_duration) / @master_duration faster_slower = nil if @branch_duration < @master_duration faster_slower = 'faster' else faster_slower = 'slower' percent_change *= -1 end "Linting #{self} with this PR took #{@branch_duration}s " \ "vs #{@master_duration}s on master (#{percent_change.to_i}\% #{faster_slower})" end end ################################ # Methods ################################ def message(str) $stderr.puts('Message: ' + str) end def warn(str) $stderr.puts('Warning: ' + str) end def fail(str) $stderr.puts('Error: ' + str) exit end def perform(*args) commands = args if @options[:verbose] commands.each do |x| puts(x) system(x) end else commands.each { |x| `#{x}` } end end def validate_state_to_run if `git symbolic-ref HEAD --short`.strip == 'master' && @options[:branch] == 'HEAD' fail "can't run osscheck without '--branch' option from 'master' as the script compares " \ "the performance of this branch against 'master'" end end def make_directory_structure ['branch_reports', 'master_reports'].each do |dir| FileUtils.mkdir_p("#{@working_dir}/#{dir}") end end def convert_to_link(repo, string) string.sub!("#{Dir.pwd}/#{@working_dir}/#{repo.name}", '') string.sub!('.swift:', '.swift#L') string = string.partition(': warning:').first.partition(': error:').first "#{repo.git_url}/blob/#{repo.commit_hash}#{string}" end def non_empty_lines(path) File.read(path).split(/\n+/).reject(&:empty?) end def setup_repos @repos.each do |repo| dir = "#{@working_dir}/#{repo.name}" puts "Cloning #{repo}" perform("git clone #{repo.git_url} --depth 1 #{dir} 2> /dev/null") if repo.name == 'Swift' File.open("#{dir}/.swiftlint.yml", 'w') do |file| file << 'included: stdlib' end end Dir.chdir(dir) do repo.commit_hash = `git rev-parse HEAD`.strip end end end def generate_reports(branch) @repos.each do |repo| Dir.chdir("#{@working_dir}/#{repo.name}") do iterations = @options[:iterations] print "Linting #{iterations} iterations of #{repo} with #{branch}: 1" durations = [] start = Time.now command = '../builds/.build/release/swiftlint lint --no-cache --enable-all-rules --reporter xcode' File.open("../#{branch}_reports/#{repo}.txt", 'w') do |file| puts "\n#{command}" if @options[:verbose] Open3.popen3(command) do |_, stdout, _, wait_thr| while line = stdout.gets file.puts line end if branch == 'branch' repo.branch_exit_value = wait_thr.value else repo.master_exit_value = wait_thr.value end end end durations << Time.now - start for i in 2..iterations print "..#{i}" start = Time.now puts command if @options[:verbose] Open3.popen3(command) { |_, stdout, _, _| stdout.read } durations << Time.now - start end puts '' average_duration = (durations.reduce(:+) / iterations).round(2) if branch == 'branch' repo.branch_duration = average_duration else repo.master_duration = average_duration end end end end def build(branch) puts "Building #{branch}" dir = "#{@working_dir}/builds" target = branch == 'master' ? @effective_master_commitish : @options[:branch] if File.directory?(dir) perform("cd #{dir}; git checkout #{target}") else perform("git fetch && git worktree add --detach #{dir} #{target}") end build_command = "cd #{dir}; swift build -c release" perform(build_command) return if $?.success? # Couldn't build, start fresh Dir.chdir(dir) do FileUtils.rm_rf %w[Packages .build] end return_value = nil puts build_command if @options[:verbose] Open3.popen3(build_command) do |_, stdout, _, wait_thr| puts stdout.read.chomp return_value = wait_thr.value end fail "Could not build #{branch}" unless return_value.success? end def diff_and_report_changes_to_danger @repos.each { |repo| message repo.duration_report } @repos.each do |repo| if repo.master_exit_value != repo.branch_exit_value warn "This PR changed the exit value when running on #{repo.name}: " \ "(#{repo.master_exit_value} to #{repo.branch_exit_value})" # If the exit value changed, don't show the fixes or regressions for this # repo because it's likely due to a crash, and all violations would be noisy next end branch = non_empty_lines("#{@working_dir}/branch_reports/#{repo.name}.txt") master = non_empty_lines("#{@working_dir}/master_reports/#{repo.name}.txt") (master - branch).each do |fixed| escaped_message = ERB::Util.html_escape fixed message "This PR fixed a violation in #{repo.name}: [#{escaped_message}](#{convert_to_link(repo, fixed)})" end (branch - master).each do |violation| escaped_message = ERB::Util.html_escape violation warn "This PR introduced a violation in #{repo.name}: [#{escaped_message}](#{convert_to_link(repo, violation)})" end end end def fetch_origin perform('git fetch origin') end def clean_up FileUtils.rm_rf(@working_dir) perform('git worktree prune') end ################################ # Script ################################ # Constants @working_dir = 'osscheck' @repos = [ Repo.new('Aerial', 'JohnCoates/Aerial'), Repo.new('Alamofire', 'Alamofire/Alamofire'), Repo.new('Firefox', 'mozilla-mobile/firefox-ios'), Repo.new('Kickstarter', 'kickstarter/ios-oss'), Repo.new('Moya', 'Moya/Moya'), Repo.new('Nimble', 'Quick/Nimble'), Repo.new('Quick', 'Quick/Quick'), Repo.new('Realm', 'realm/realm-cocoa'), Repo.new('SourceKitten', 'jpsim/SourceKitten'), Repo.new('Sourcery', 'krzysztofzablocki/Sourcery'), Repo.new('Swift', 'apple/swift'), Repo.new('WordPress', 'wordpress-mobile/WordPress-iOS') ] # Prep $stdout.sync = true validate_state_to_run setup_repos make_directory_structure fetch_origin @effective_master_commitish = `git merge-base origin/master #{@options[:branch]}` # Build & generate reports for branch & master %w[branch master].each do |branch| build(branch) generate_reports(branch) end # Diff and report changes to Danger diff_and_report_changes_to_danger # Clean up clean_up unless @options[:skip_clean]