#!/usr/bin/env ruby ################################ # Requires ################################ require 'fileutils' require 'open3' require 'optparse' require 'erb' ################################ # Options ################################ @options = { branch: 'HEAD', iterations: 5, skip_clean: false, verbose: false } OptionParser.new do |opts| opts.on('--branch BRANCH', "compares the performance of BRANCH against 'master'") do |branch| @options[:branch] = branch end opts.on('--iterations N', Integer, 'iterates lint N times on each repositories') do |iterations| @options[:iterations] = iterations end opts.on('--skip-clean', 'skip cleaning on completion') do |skip_clean| @options[:skip_clean] = skip_clean end opts.on('-v', '--[no-]verbose', 'Run verbosely') do |v| @options[:verbose] = v end end.parse! ################################ # Classes ################################ class Repo attr_accessor :name attr_accessor :github_location attr_accessor :commit_hash attr_accessor :branch_exit_value attr_accessor :branch_duration attr_accessor :master_exit_value attr_accessor :master_duration def initialize(name, github_location) @name = name @github_location = github_location end def git_url "https://github.com/#{github_location}" end def to_s @name end def duration_report percent_change = 100 * (@master_duration - @branch_duration) / @master_duration faster_slower = nil if @branch_duration < @master_duration faster_slower = 'faster' else faster_slower = 'slower' percent_change *= -1 end "Linting #{self} with this PR took #{@branch_duration}s " \ "vs #{@master_duration}s on master (#{percent_change.to_i}\% #{faster_slower})" end end ################################ # Methods ################################ def message(str) $stderr.puts('Message: ' + str) end def warn(str) $stderr.puts('Warning: ' + str) end def fail(str) $stderr.puts('Error: ' + str) exit end def perform(*args) commands = args if @options[:verbose] commands.each do |x| puts(x) system(x) end else commands.each { |x| `#{x}` } end end def validate_state_to_run if `git symbolic-ref HEAD --short || true`.strip == 'master' && @options[:branch] == 'HEAD' fail "can't run osscheck without '--branch' option from 'master' as the script compares " \ "the performance of this branch against 'master'" end end def make_directory_structure ['branch_reports', 'master_reports'].each do |dir| FileUtils.mkdir_p("#{@working_dir}/#{dir}") end end def convert_to_link(repo, string) string = remove_base_path(repo, string) string.sub!('.swift:', '.swift#L') string = string.partition(': warning:').first.partition(': error:').first "#{repo.git_url}/blob/#{repo.commit_hash}#{string}" end def remove_base_path(repo, string) string.sub("#{Dir.pwd}/#{@working_dir}/#{repo.name}", '') end def non_empty_lines(path) File.read(path).split(/\n+/).reject(&:empty?) end def setup_repos @repos.each do |repo| dir = "#{@working_dir}/#{repo.name}" puts "Cloning #{repo}" perform("git clone #{repo.git_url} --depth 1 #{dir} 2> /dev/null") swiftlint_config = "#{dir}/.swiftlint.yml" FileUtils.rm_rf(swiftlint_config) if repo.name == 'Swift' File.open(swiftlint_config, 'w') do |file| file.puts('included: stdlib') end end if @only_rules_changed && @rules_changed File.open(swiftlint_config, 'w') do |file| file.puts('only_rules:') file.puts(@rules_changed.map { |rule| " - #{rule}" }) end end Dir.chdir(dir) do repo.commit_hash = `git rev-parse HEAD`.strip end end end def generate_reports(branch) @repos.each do |repo| Dir.chdir("#{@working_dir}/#{repo.name}") do perform("git checkout #{repo.commit_hash}") iterations = @options[:iterations] print "Linting #{iterations} iterations of #{repo} with #{branch}: 1" durations = [] start = Time.now command = "../builds/swiftlint-#{branch} lint --no-cache #{'--enable-all-rules' unless @only_rules_changed} --reporter xcode" File.open("../#{branch}_reports/#{repo}.txt", 'w') do |file| puts "\n#{command}" if @options[:verbose] Open3.popen2(command) do |_, stdout, wait_thr| while line = stdout.gets file.puts line end if branch == 'branch' repo.branch_exit_value = wait_thr.value else repo.master_exit_value = wait_thr.value end end end durations << Time.now - start for i in 2..iterations print "..#{i}" start = Time.now puts command if @options[:verbose] Open3.popen2(command) { |_, stdout, _| stdout.read } durations << Time.now - start end puts '' average_duration = (durations.reduce(:+) / iterations).round(2) if branch == 'branch' repo.branch_duration = average_duration else repo.master_duration = average_duration end end end end def build(branch) puts "Building #{branch}" dir = "#{@working_dir}/builds" target = branch == 'master' ? @effective_master_commitish : @options[:branch] if File.directory?(dir) perform("cd #{dir}; git checkout #{target}") else perform("git worktree add --detach #{dir} #{target}") end build_command = "cd #{dir}; bazel build -c opt @SwiftLint//:swiftlint && mv bazel-bin/swiftlint swiftlint-#{branch}" perform(build_command) return if $?.success? return_value = nil puts build_command if @options[:verbose] Open3.popen3(build_command) do |_, stdout, _, wait_thr| puts stdout.read.chomp return_value = wait_thr.value end fail "Could not build #{branch}" unless return_value.success? end def diff_and_report_changes_to_danger @repos.each { |repo| message repo.duration_report } @repos.each do |repo| if repo.master_exit_value != repo.branch_exit_value warn "This PR changed the exit value when running on #{repo.name}: " \ "(#{repo.master_exit_value} to #{repo.branch_exit_value})" # If the exit value changed, don't show the fixes or regressions for this # repo because it's likely due to a crash, and all violations would be noisy next end branch = non_empty_lines("#{@working_dir}/branch_reports/#{repo.name}.txt").sort master = non_empty_lines("#{@working_dir}/master_reports/#{repo.name}.txt").sort (master - branch).each do |fixed| escaped_message = ERB::Util.html_escape remove_base_path(repo, fixed) message "This PR fixed a violation in #{repo.name}: [#{escaped_message}](#{convert_to_link(repo, fixed)})" end (branch - master).each do |violation| escaped_message = ERB::Util.html_escape remove_base_path(repo, violation) warn "This PR introduced a violation in #{repo.name}: [#{escaped_message}](#{convert_to_link(repo, violation)})" end end end def fetch_origin perform('git fetch origin') end def clean_up FileUtils.rm_rf(@working_dir) perform('git worktree prune') end def set_globals @effective_master_commitish = `git merge-base origin/master #{@options[:branch]}`.chomp @changed_swift_files = `git diff --diff-filter=AMRCU #{@effective_master_commitish} --name-only | grep "\.swift$" || true`.split("\n") @changed_rule_files = @changed_swift_files.select do |file| file.start_with? 'Source/SwiftLintFramework/Rules/' end @rules_changed = @changed_rule_files.map do |path| if File.read(path) =~ /^\s+identifier: "(\w+)",$/ $1 else nil end end.compact.sort # True iff the only Swift files that were changed are SwiftLint rules, and that number is one or greater @only_rules_changed = !@rules_changed.empty? && @changed_swift_files.count == @rules_changed.count end def print_rules_changed if @only_rules_changed puts "Only #{@rules_changed.count} rules changed: #{@rules_changed.join(', ')}" end end ################################ # Script ################################ # Constants @working_dir = 'osscheck' @repos = [ Repo.new('Aerial', 'JohnCoates/Aerial'), Repo.new('Alamofire', 'Alamofire/Alamofire'), Repo.new('Firefox', 'mozilla-mobile/firefox-ios'), Repo.new('Kickstarter', 'kickstarter/ios-oss'), Repo.new('Moya', 'Moya/Moya'), Repo.new('Nimble', 'Quick/Nimble'), Repo.new('Quick', 'Quick/Quick'), Repo.new('Realm', 'realm/realm-cocoa'), Repo.new('SourceKitten', 'jpsim/SourceKitten'), Repo.new('Sourcery', 'krzysztofzablocki/Sourcery'), Repo.new('Swift', 'apple/swift'), Repo.new('WordPress', 'wordpress-mobile/WordPress-iOS') ] # Clean up clean_up unless @options[:skip_clean] # Prep $stdout.sync = true validate_state_to_run fetch_origin set_globals print_rules_changed make_directory_structure # Build & generate reports for branch & master %w[branch master].each do |branch| build(branch) end full_version_branch = `#{@working_dir}/builds/swiftlint-branch version --verbose` full_version_master = `#{@working_dir}/builds/swiftlint-master version --verbose` if full_version_branch == full_version_master message "Skipping OSSCheck because SwiftLint hasn't changed compared to 'master'" # Clean up clean_up unless @options[:skip_clean] exit end setup_repos %w[branch master].each do |branch| generate_reports(branch) end # Diff and report changes to Danger diff_and_report_changes_to_danger # Clean up clean_up unless @options[:skip_clean]