debian-mirror-gitlab/lib/gitlab/ee_compat_check.rb

441 lines
14 KiB
Ruby
Raw Normal View History

2017-08-17 22:00:37 +05:30
# rubocop: disable Rails/Output
module Gitlab
# Checks if a set of migrations requires downtime or not.
class EeCompatCheck
2018-05-09 12:01:36 +05:30
CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze
CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
2017-08-17 22:00:37 +05:30
CHECK_DIR = Rails.root.join('ee_compat_check')
2018-10-15 14:42:47 +05:30
IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb|locale/gitlab\.pot}i.freeze
2017-08-17 22:00:37 +05:30
PLEASE_READ_THIS_BANNER = %Q{
============================================================
===================== PLEASE READ THIS =====================
============================================================
}.freeze
2018-05-09 12:01:36 +05:30
STAY_STRONG_LINK_TO_DOCS = %Q{
Stay 💪! For more information, see
https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
}.freeze
2017-08-17 22:00:37 +05:30
THANKS_FOR_READING_BANNER = %Q{
============================================================
==================== THANKS FOR READING ====================
============================================================\n
}.freeze
2018-05-09 12:01:36 +05:30
attr_reader :ee_repo_dir, :patches_dir
attr_reader :ce_project_url, :ee_repo_url
attr_reader :ce_branch, :ee_remote_with_branch, :ee_branch_found
2018-03-17 18:26:18 +05:30
attr_reader :job_id, :failed_files
2017-08-17 22:00:37 +05:30
2018-05-09 12:01:36 +05:30
def initialize(branch:, ce_project_url: CANONICAL_CE_PROJECT_URL, job_id: nil)
2017-08-17 22:00:37 +05:30
@ee_repo_dir = CHECK_DIR.join('ee-repo')
@patches_dir = CHECK_DIR.join('patches')
@ce_branch = branch
2018-03-17 18:26:18 +05:30
@ce_project_url = ce_project_url
2018-05-09 12:01:36 +05:30
@ee_repo_url = ce_public_repo_url.sub('gitlab-ce', 'gitlab-ee')
2018-03-17 18:26:18 +05:30
@job_id = job_id
2017-08-17 22:00:37 +05:30
end
def check
ensure_patches_dir
2018-05-09 12:01:36 +05:30
# We're generating the patch against the canonical-ce remote since forks'
# master branch are not necessarily up-to-date.
add_remote('canonical-ce', "#{CANONICAL_CE_PROJECT_URL}.git")
generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, branch_remote: 'origin', master_remote: 'canonical-ce')
2017-08-17 22:00:37 +05:30
ensure_ee_repo
Dir.chdir(ee_repo_dir) do
step("In the #{ee_repo_dir} directory")
2018-05-09 12:01:36 +05:30
ee_remotes.each do |key, url|
add_remote(key, url)
end
fetch(branch: 'master', depth: 20, remote: 'canonical-ee')
2018-03-17 18:26:18 +05:30
2017-08-17 22:00:37 +05:30
status = catch(:halt_check) do
ce_branch_compat_check!
delete_ee_branches_locally!
ee_branch_presence_check!
2018-03-17 18:26:18 +05:30
2018-05-09 12:01:36 +05:30
step("Checking out #{ee_remote_with_branch}/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} #{ee_remote_with_branch}/#{ee_branch_found}])
generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, branch_remote: ee_remote_with_branch, master_remote: 'canonical-ee')
2017-08-17 22:00:37 +05:30
ee_branch_compat_check!
end
delete_ee_branches_locally!
2018-05-09 12:01:36 +05:30
status.nil?
2017-08-17 22:00:37 +05:30
end
end
private
2018-05-09 12:01:36 +05:30
def fork?
ce_project_url != CANONICAL_CE_PROJECT_URL
end
def ee_remotes
return @ee_remotes if defined?(@ee_remotes)
remotes =
{
'ee' => ee_repo_url,
'canonical-ee' => CANONICAL_EE_REPO_URL
}
remotes.delete('ee') unless fork?
@ee_remotes = remotes
end
2018-03-17 18:26:18 +05:30
def add_remote(name, url)
step(
"Adding the #{name} remote (#{url})",
%W[git remote add #{name} #{url}]
)
end
2017-08-17 22:00:37 +05:30
def ensure_ee_repo
2018-05-09 12:01:36 +05:30
unless clone_repo(ee_repo_url, ee_repo_dir)
# Fallback to using the canonical EE if there is no forked EE
clone_repo(CANONICAL_EE_REPO_URL, ee_repo_dir)
2017-08-17 22:00:37 +05:30
end
end
2018-05-09 12:01:36 +05:30
def clone_repo(url, dir)
_, status = step(
"Cloning #{url} into #{dir}",
%W[git clone --branch master --single-branch --depth=200 #{url} #{dir}]
)
status.zero?
end
2017-08-17 22:00:37 +05:30
def ensure_patches_dir
FileUtils.mkdir_p(patches_dir)
end
2018-05-09 12:01:36 +05:30
def generate_patch(branch:, patch_path:, branch_remote:, master_remote:)
2017-08-17 22:00:37 +05:30
FileUtils.rm(patch_path, force: true)
2018-05-09 12:01:36 +05:30
find_merge_base_with_master(branch: branch, branch_remote: branch_remote, master_remote: master_remote)
2017-08-17 22:00:37 +05:30
step(
2018-05-09 12:01:36 +05:30
"Generating the patch against #{master_remote}/master in #{patch_path}",
%W[git diff --binary #{master_remote}/master...#{branch_remote}/#{branch}]
2017-08-17 22:00:37 +05:30
) do |output, status|
throw(:halt_check, :ko) unless status.zero?
File.write(patch_path, output)
throw(:halt_check, :ko) unless File.exist?(patch_path)
end
end
def ce_branch_compat_check!
if check_patch(ce_patch_full_path).zero?
puts applies_cleanly_msg(ce_branch)
throw(:halt_check)
end
end
def ee_branch_presence_check!
2018-05-09 12:01:36 +05:30
ee_remotes.keys.each do |remote|
2018-11-18 11:00:15 +05:30
output, _ = step(
"Searching #{remote}",
%W[git ls-remote #{remote} *#{minimal_ee_branch_name}*])
2018-05-09 12:01:36 +05:30
2018-11-18 11:00:15 +05:30
branches =
output.scan(%r{(?<=refs/heads/|refs/tags/).+}).sort_by(&:size)
next if branches.empty?
branch = branches.first
step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}])
@ee_remote_with_branch = remote
@ee_branch_found = branch
return true
2017-08-17 22:00:37 +05:30
end
2018-05-09 12:01:36 +05:30
puts
puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
2017-08-17 22:00:37 +05:30
2018-05-09 12:01:36 +05:30
throw(:halt_check, :ko)
2017-08-17 22:00:37 +05:30
end
def ee_branch_compat_check!
unless check_patch(ee_patch_full_path).zero?
puts
puts ee_branch_doesnt_apply_cleanly_msg
throw(:halt_check, :ko)
end
puts
puts applies_cleanly_msg(ee_branch_found)
end
def check_patch(patch_path)
step("Checking out master", %w[git checkout master])
2018-03-17 18:26:18 +05:30
step("Resetting to latest master", %w[git reset --hard canonical-ee/master])
2017-08-17 22:00:37 +05:30
step(
"Checking if #{patch_path} applies cleanly to EE/master",
2017-09-10 17:25:29 +05:30
# Don't use --check here because it can result in a 0-exit status even
# though the patch doesn't apply cleanly, e.g.:
# > git apply --check --3way foo.patch
# error: patch failed: lib/gitlab/ee_compat_check.rb:74
# Falling back to three-way merge...
# Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
# > echo $?
# 0
%W[git apply --3way #{patch_path}]
2017-08-17 22:00:37 +05:30
) do |output, status|
2017-09-10 17:25:29 +05:30
puts output
2018-03-17 18:26:18 +05:30
2017-08-17 22:00:37 +05:30
unless status.zero?
@failed_files = output.lines.reduce([]) do |memo, line|
if line.start_with?('error: patch failed:')
file = line.sub(/\Aerror: patch failed: /, '')
memo << file unless file =~ IGNORED_FILES_REGEX
end
2018-03-17 18:26:18 +05:30
2017-08-17 22:00:37 +05:30
memo
end
status = 0 if failed_files.empty?
end
2017-09-10 17:25:29 +05:30
command(%w[git reset --hard])
2017-08-17 22:00:37 +05:30
status
end
end
def delete_ee_branches_locally!
command(%w[git checkout master])
command(%W[git branch --delete --force #{ee_branch_prefix}])
command(%W[git branch --delete --force #{ee_branch_suffix}])
end
2018-05-09 12:01:36 +05:30
def merge_base_found?(branch:, branch_remote:, master_remote:)
2017-08-17 22:00:37 +05:30
step(
2018-03-17 18:26:18 +05:30
"Finding merge base with #{master_remote}/master",
2018-05-09 12:01:36 +05:30
%W[git merge-base #{master_remote}/master #{branch_remote}/#{branch}]
2017-08-17 22:00:37 +05:30
) do |output, status|
if status.zero?
puts "Merge base was found: #{output}"
true
end
end
end
2018-05-09 12:01:36 +05:30
def find_merge_base_with_master(branch:, branch_remote:, master_remote:)
2017-08-17 22:00:37 +05:30
# Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403)
# In total we go (20 + 54 + 148 + 403 = 625) commits deeper
depth = 20
success =
(3..6).any? do |factor|
depth += Math.exp(factor).to_i
# Repository is initially cloned with a depth of 20 so we need to fetch
# deeper in the case the branch has more than 20 commits on top of master
2018-05-09 12:01:36 +05:30
fetch(branch: branch, depth: depth, remote: branch_remote)
2018-03-17 18:26:18 +05:30
fetch(branch: 'master', depth: depth, remote: master_remote)
2017-08-17 22:00:37 +05:30
2018-05-09 12:01:36 +05:30
merge_base_found?(branch: branch, branch_remote: branch_remote, master_remote: master_remote)
2017-08-17 22:00:37 +05:30
end
2018-03-17 18:26:18 +05:30
raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success
2017-08-17 22:00:37 +05:30
end
2018-03-17 18:26:18 +05:30
def fetch(branch:, depth:, remote: 'origin')
2017-08-17 22:00:37 +05:30
step(
"Fetching deeper...",
2018-03-17 18:26:18 +05:30
%W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/#{remote}/#{branch}]
2017-08-17 22:00:37 +05:30
) do |output, status|
raise "Fetch failed: #{output}" unless status.zero?
end
end
def ce_patch_name
@ce_patch_name ||= patch_name_from_branch(ce_branch)
end
def ce_patch_full_path
@ce_patch_full_path ||= patches_dir.join(ce_patch_name)
end
def ee_branch_suffix
@ee_branch_suffix ||= "#{ce_branch}-ee"
end
def ee_branch_prefix
@ee_branch_prefix ||= "ee-#{ce_branch}"
end
def ee_patch_name
@ee_patch_name ||= patch_name_from_branch(ee_branch_found)
end
def ee_patch_full_path
@ee_patch_full_path ||= patches_dir.join(ee_patch_name)
end
2018-11-18 11:00:15 +05:30
def minimal_ee_branch_name
@minimal_ee_branch_name ||= ce_branch.sub(/(\Ace\-|\-ce\z)/, '')
end
2017-08-17 22:00:37 +05:30
def patch_name_from_branch(branch_name)
branch_name.parameterize << '.patch'
end
2017-09-10 17:25:29 +05:30
def patch_url
2018-03-17 18:26:18 +05:30
"#{ce_project_url}/-/jobs/#{job_id}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}"
2017-09-10 17:25:29 +05:30
end
2017-08-17 22:00:37 +05:30
def step(desc, cmd = nil)
puts "\n=> #{desc}\n"
if cmd
start = Time.now
puts "\n$ #{cmd.join(' ')}"
output, status = command(cmd)
puts "\n==> Finished in #{Time.now - start} seconds"
if block_given?
yield(output, status)
else
[output, status]
end
end
end
def command(cmd)
Gitlab::Popen.popen(cmd)
end
2018-05-09 12:01:36 +05:30
# We're "re-creating" the repo URL because ENV['CI_REPOSITORY_URL'] contains
# redacted credentials (e.g. "***:****") which are useless in instructions
# the job gives.
def ce_public_repo_url
"#{ce_project_url}.git"
end
2017-08-17 22:00:37 +05:30
def applies_cleanly_msg(branch)
%Q{
#{PLEASE_READ_THIS_BANNER}
🎉 Congratulations!! 🎉
The `#{branch}` branch applies cleanly to EE/master!
Much ! For more information, see
2018-03-17 18:26:18 +05:30
https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
2017-08-17 22:00:37 +05:30
#{THANKS_FOR_READING_BANNER}
}
end
def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
2018-05-09 12:01:36 +05:30
ee_repos = ee_remotes.values.uniq
2017-08-17 22:00:37 +05:30
%Q{
#{PLEASE_READ_THIS_BANNER}
💥 Oh no! 💥
The `#{ce_branch}` branch does not apply cleanly to the current
EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
2018-05-09 12:01:36 +05:30
was found in #{ee_repos.join(' nor in ')}.
2017-08-17 22:00:37 +05:30
2018-03-17 18:26:18 +05:30
If you're a community contributor, don't worry, someone from
GitLab Inc. will take care of this, and you don't have to do anything.
If you're willing to help, and are ok to contribute to EE as well,
you're welcome to help. You could follow the instructions below.
2017-08-17 22:00:37 +05:30
#{conflicting_files_msg}
We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}`
branch that includes changes from `#{ce_branch}` but also specific changes
than can be applied cleanly to EE/master. In some cases, the conflicts
are trivial and you can ignore the warning from this job. As always,
2018-03-17 18:26:18 +05:30
use your best judgement!
2017-08-17 22:00:37 +05:30
There are different ways to create such branch:
1. Create a new branch from master and cherry-pick your CE commits
# In the EE repo
2018-05-09 12:01:36 +05:30
$ git fetch #{CANONICAL_EE_REPO_URL} master
2018-03-17 18:26:18 +05:30
$ git checkout -b #{ee_branch_prefix} FETCH_HEAD
2018-05-09 12:01:36 +05:30
$ git fetch #{ce_public_repo_url} #{ce_branch}
2017-08-17 22:00:37 +05:30
$ git cherry-pick SHA # Repeat for all the commits you want to pick
2018-05-09 12:01:36 +05:30
Note: You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
2017-08-17 22:00:37 +05:30
2. Apply your branch's patch to EE
# In the EE repo
2018-05-09 12:01:36 +05:30
$ git fetch #{CANONICAL_EE_REPO_URL} master
2018-03-17 18:26:18 +05:30
$ git checkout -b #{ee_branch_prefix} FETCH_HEAD
$ wget #{patch_url} && git apply --3way #{ce_patch_name}
2017-08-17 22:00:37 +05:30
At this point you might have conflicts such as:
error: patch failed: lib/gitlab/ee_compat_check.rb:5
Falling back to three-way merge...
Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
U lib/gitlab/ee_compat_check.rb
Resolve them, stage the changes and commit them.
2017-09-10 17:25:29 +05:30
If the patch couldn't be applied cleanly, use the following command:
# In the EE repo
$ git apply --reject #{ce_patch_name}
This option makes git apply the parts of the patch that are applicable,
and leave the rejected hunks in corresponding `.rej` files.
You can then resolve the conflicts highlighted in `.rej` by
manually applying the correct diff from the `.rej` file to the file with conflicts.
When finished, you can delete the `.rej` files and commit your changes.
2017-08-17 22:00:37 +05:30
Don't forget to push your branch to gitlab-ee:
# In the EE repo
$ git push origin #{ee_branch_prefix}
2017-09-10 17:25:29 +05:30
Also, don't forget to create a new merge request on gitlab-ee and
2017-08-17 22:00:37 +05:30
cross-link it with the CE merge request.
2018-05-09 12:01:36 +05:30
Once this is done, you can retry this failed job, and it should pass.
2017-08-17 22:00:37 +05:30
2018-05-09 12:01:36 +05:30
#{STAY_STRONG_LINK_TO_DOCS}
2017-08-17 22:00:37 +05:30
#{THANKS_FOR_READING_BANNER}
}
end
def ee_branch_doesnt_apply_cleanly_msg
%Q{
#{PLEASE_READ_THIS_BANNER}
💥 Oh no! 💥
The `#{ce_branch}` does not apply cleanly to the current EE/master, and
even though a `#{ee_branch_found}` branch
2018-05-09 12:01:36 +05:30
exists in #{ee_repo_url}, it does not apply cleanly either to
2017-08-17 22:00:37 +05:30
EE/master!
#{conflicting_files_msg}
Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and
2018-05-09 12:01:36 +05:30
retry this job.
2017-08-17 22:00:37 +05:30
2018-05-09 12:01:36 +05:30
#{STAY_STRONG_LINK_TO_DOCS}
2017-08-17 22:00:37 +05:30
#{THANKS_FOR_READING_BANNER}
}
end
def conflicting_files_msg
failed_files.reduce("The conflicts detected were as follows:\n") do |memo, file|
memo << "\n - #{file}"
end
end
end
end