debian-mirror-gitlab/scripts/review_apps/review-apps.sh

341 lines
12 KiB
Bash
Raw Normal View History

2018-12-05 23:21:45 +05:30
[[ "$TRACE" ]] && set -x
2019-09-30 21:07:59 +05:30
function deploy_exists() {
2019-07-07 11:18:12 +05:30
local namespace="${1}"
2020-01-01 13:55:28 +05:30
local release="${2}"
local deploy_exists
2019-07-07 11:18:12 +05:30
2020-01-01 13:55:28 +05:30
echoinfo "Checking if ${release} exists in the ${namespace} namespace..." true
2019-07-07 11:18:12 +05:30
2020-04-22 19:07:51 +05:30
helm status --namespace "${namespace}" "${release}" >/dev/null 2>&1
2020-01-01 13:55:28 +05:30
deploy_exists=$?
echoinfo "Deployment status for ${release} is ${deploy_exists}"
2019-07-07 11:18:12 +05:30
return $deploy_exists
}
2019-09-30 21:07:59 +05:30
function previous_deploy_failed() {
2019-12-26 22:10:19 +05:30
local namespace="${1}"
2020-01-01 13:55:28 +05:30
local release="${2}"
2019-12-26 22:10:19 +05:30
2020-01-01 13:55:28 +05:30
echoinfo "Checking for previous deployment of ${release}" true
2019-07-07 11:18:12 +05:30
2020-04-22 19:07:51 +05:30
helm status --namespace "${namespace}" "${release}" >/dev/null 2>&1
2019-07-07 11:18:12 +05:30
local status=$?
# if `status` is `0`, deployment exists, has a status
if [ $status -eq 0 ]; then
echoinfo "Previous deployment found, checking status..."
2020-04-22 19:07:51 +05:30
deployment_status=$(helm status --namespace "${namespace}" "${release}" | grep ^STATUS | cut -d' ' -f2)
2019-07-07 11:18:12 +05:30
echoinfo "Previous deployment state: ${deployment_status}"
2020-04-22 19:07:51 +05:30
if [[ "$deployment_status" == "failed" || "$deployment_status" == "pending-upgrade" || "$deployment_status" == "pending-install" ]]; then
2019-07-07 11:18:12 +05:30
status=0;
else
status=1;
fi
else
echoerr "Previous deployment NOT found."
fi
return $status
}
2019-12-04 20:38:33 +05:30
function delete_release() {
2021-09-04 01:27:46 +05:30
local namespace="${CI_ENVIRONMENT_SLUG}"
2020-01-01 13:55:28 +05:30
local release="${CI_ENVIRONMENT_SLUG}"
2019-12-26 22:10:19 +05:30
2020-01-01 13:55:28 +05:30
if [ -z "${release}" ]; then
2019-07-07 11:18:12 +05:30
echoerr "No release given, aborting the delete!"
return
fi
2021-09-30 23:02:18 +05:30
if deploy_exists "${namespace}" "${release}"; then
helm uninstall --namespace="${namespace}" "${release}"
fi
2019-07-07 11:18:12 +05:30
}
2019-12-04 20:38:33 +05:30
function delete_failed_release() {
2021-09-04 01:27:46 +05:30
local namespace="${CI_ENVIRONMENT_SLUG}"
2020-01-01 13:55:28 +05:30
local release="${CI_ENVIRONMENT_SLUG}"
if [ -z "${release}" ]; then
2019-12-04 20:38:33 +05:30
echoerr "No release given, aborting the delete!"
return
fi
2020-01-01 13:55:28 +05:30
if ! deploy_exists "${namespace}" "${release}"; then
echoinfo "No Review App with ${release} is currently deployed."
2019-12-04 20:38:33 +05:30
else
# Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
2020-01-01 13:55:28 +05:30
if previous_deploy_failed "${namespace}" "${release}" ; then
2021-09-04 01:27:46 +05:30
echoinfo "Review App deployment in bad state, cleaning up namespace ${release}"
2021-09-30 23:02:18 +05:30
delete_k8s_release_namespace
2019-12-04 20:38:33 +05:30
else
echoinfo "Review App deployment in good state"
fi
fi
}
2021-09-04 01:27:46 +05:30
function delete_k8s_release_namespace() {
local namespace="${CI_ENVIRONMENT_SLUG}"
kubectl delete namespace "${namespace}" --wait
}
2019-07-07 11:18:12 +05:30
function get_pod() {
2021-09-04 01:27:46 +05:30
local namespace="${CI_ENVIRONMENT_SLUG}"
2020-01-01 13:55:28 +05:30
local release="${CI_ENVIRONMENT_SLUG}"
2019-07-07 11:18:12 +05:30
local app_name="${1}"
local status="${2-Running}"
2020-01-01 13:55:28 +05:30
get_pod_cmd="kubectl get pods --namespace ${namespace} --field-selector=status.phase=${status} -lapp=${app_name},release=${release} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1"
2019-09-30 21:07:59 +05:30
echoinfo "Waiting till '${app_name}' pod is ready" true
echoinfo "Running '${get_pod_cmd}'"
2019-07-07 11:18:12 +05:30
2019-09-30 21:07:59 +05:30
local interval=5
local elapsed_seconds=0
local max_seconds=$((2 * 60))
2019-07-07 11:18:12 +05:30
while true; do
2019-07-31 22:56:46 +05:30
local pod_name
pod_name="$(eval "${get_pod_cmd}")"
2019-07-07 11:18:12 +05:30
[[ "${pod_name}" == "" ]] || break
2019-09-30 21:07:59 +05:30
if [[ "${elapsed_seconds}" -gt "${max_seconds}" ]]; then
echoerr "The pod name couldn't be found after ${elapsed_seconds} seconds, aborting."
break
fi
let "elapsed_seconds+=interval"
sleep ${interval}
2019-07-07 11:18:12 +05:30
done
echoinfo "The pod name is '${pod_name}'."
echo "${pod_name}"
}
2020-07-28 23:09:34 +05:30
function run_task() {
2021-09-04 01:27:46 +05:30
local namespace="${CI_ENVIRONMENT_SLUG}"
2020-07-28 23:09:34 +05:30
local ruby_cmd="${1}"
local task_runner_pod=$(get_pod "task-runner")
2021-01-03 14:25:43 +05:30
kubectl exec --namespace "${namespace}" "${task_runner_pod}" -- gitlab-rails runner "${ruby_cmd}"
2020-07-28 23:09:34 +05:30
}
function disable_sign_ups() {
if [ -z ${REVIEW_APPS_ROOT_TOKEN+x} ]; then
echoerr "In order to protect Review Apps, REVIEW_APPS_ROOT_TOKEN variable must be set"
false
else
true
fi
# Create the root token
2021-09-04 01:27:46 +05:30
local set_token_rb="token = User.find_by_username('root').personal_access_tokens.create(scopes: [:api], name: 'Token to disable sign-ups'); token.set_token('${REVIEW_APPS_ROOT_TOKEN}'); begin; token.save!; rescue(ActiveRecord::RecordNotUnique); end"
retry "run_task \"${set_token_rb}\""
2020-07-28 23:09:34 +05:30
# Disable sign-ups
2021-09-04 01:27:46 +05:30
local disable_signup_rb="Gitlab::CurrentSettings.current_application_settings.update!(signup_enabled: false)"
if (retry "run_task \"${disable_signup_rb}\""); then
2020-07-28 23:09:34 +05:30
echoinfo "Sign-ups have been disabled successfully."
else
2020-11-24 15:15:51 +05:30
echoerr "Sign-ups are still enabled!"
2020-07-28 23:09:34 +05:30
false
fi
}
2018-12-05 23:21:45 +05:30
function check_kube_domain() {
2019-07-07 11:18:12 +05:30
echoinfo "Checking that Kube domain exists..." true
2018-12-05 23:21:45 +05:30
if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then
echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set"
echo "You can do it in Auto DevOps project settings or defining a variable at group or project level"
echo "You can also manually add it in .gitlab-ci.yml"
false
else
true
fi
}
2019-05-30 16:15:17 +05:30
function ensure_namespace() {
2021-09-04 01:27:46 +05:30
local namespace="${1}"
2020-01-01 13:55:28 +05:30
echoinfo "Ensuring the ${namespace} namespace exists..." true
2019-07-07 11:18:12 +05:30
2020-01-01 13:55:28 +05:30
kubectl describe namespace "${namespace}" || kubectl create namespace "${namespace}"
2018-12-05 23:21:45 +05:30
}
2021-09-04 01:27:46 +05:30
function label_namespace() {
local namespace="${1}"
local label="${2}"
echoinfo "Labeling the ${namespace} namespace with ${label}" true
2021-09-30 23:02:18 +05:30
echoinfo "We should pass the --overwrite option!"
2021-09-04 01:27:46 +05:30
2021-09-30 23:02:18 +05:30
kubectl label --overwrite namespace "${namespace}" "${label}"
2020-07-28 23:09:34 +05:30
}
2019-09-30 21:07:59 +05:30
function create_application_secret() {
2021-09-04 01:27:46 +05:30
local namespace="${CI_ENVIRONMENT_SLUG}"
2020-01-01 13:55:28 +05:30
local release="${CI_ENVIRONMENT_SLUG}"
local initial_root_password_shared_secret
local gitlab_license_shared_secret
initial_root_password_shared_secret=$(kubectl get secret --namespace ${namespace} --no-headers -o=custom-columns=NAME:.metadata.name shared-gitlab-initial-root-password | tail -n 1)
if [[ "${initial_root_password_shared_secret}" == "" ]]; then
echoinfo "Creating the 'shared-gitlab-initial-root-password' secret in the ${namespace} namespace..." true
kubectl create secret generic --namespace "${namespace}" \
"shared-gitlab-initial-root-password" \
--from-literal="password=${REVIEW_APPS_ROOT_PASSWORD}" \
--dry-run -o json | kubectl apply -f -
else
echoinfo "The 'shared-gitlab-initial-root-password' secret already exists in the ${namespace} namespace."
fi
2019-12-26 22:10:19 +05:30
2021-06-08 01:23:25 +05:30
if [ -z "${REVIEW_APPS_EE_LICENSE_FILE}" ]; then echo "License not found" && return; fi
2019-12-26 22:10:19 +05:30
2020-01-01 13:55:28 +05:30
gitlab_license_shared_secret=$(kubectl get secret --namespace ${namespace} --no-headers -o=custom-columns=NAME:.metadata.name shared-gitlab-license | tail -n 1)
if [[ "${gitlab_license_shared_secret}" == "" ]]; then
echoinfo "Creating the 'shared-gitlab-license' secret in the ${namespace} namespace..." true
kubectl create secret generic --namespace "${namespace}" \
"shared-gitlab-license" \
2021-06-08 01:23:25 +05:30
--from-file=license="${REVIEW_APPS_EE_LICENSE_FILE}" \
2020-01-01 13:55:28 +05:30
--dry-run -o json | kubectl apply -f -
else
echoinfo "The 'shared-gitlab-license' secret already exists in the ${namespace} namespace."
fi
2018-12-05 23:21:45 +05:30
}
2019-09-30 21:07:59 +05:30
function download_chart() {
2019-07-07 11:18:12 +05:30
echoinfo "Downloading the GitLab chart..." true
2019-05-18 00:54:41 +05:30
2019-12-04 20:38:33 +05:30
curl --location -o gitlab.tar.bz2 "https://gitlab.com/gitlab-org/charts/gitlab/-/archive/${GITLAB_HELM_CHART_REF}/gitlab-${GITLAB_HELM_CHART_REF}.tar.bz2"
2019-07-07 11:18:12 +05:30
tar -xjf gitlab.tar.bz2
echoinfo "Adding the gitlab repo to Helm..."
helm repo add gitlab https://charts.gitlab.io
echoinfo "Building the gitlab chart's dependencies..."
2021-06-08 01:23:25 +05:30
helm dependency build "gitlab-${GITLAB_HELM_CHART_REF}"
2018-12-05 23:21:45 +05:30
}
2019-12-26 22:10:19 +05:30
function base_config_changed() {
if [ -z "${CI_MERGE_REQUEST_IID}" ]; then return; fi
curl "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/changes" | jq '.changes | any(.old_path == "scripts/review_apps/base-config.yaml")'
}
2020-07-28 23:09:34 +05:30
function parse_gitaly_image_tag() {
local gitaly_version="${GITALY_VERSION}"
# prepend semver version with `v`
if [[ $gitaly_version =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?(-ee)?$ ]]; then
echo "v${gitaly_version}"
else
echo "${gitaly_version}"
fi
}
2018-12-05 23:21:45 +05:30
function deploy() {
2021-09-04 01:27:46 +05:30
local namespace="${CI_ENVIRONMENT_SLUG}"
2020-01-01 13:55:28 +05:30
local release="${CI_ENVIRONMENT_SLUG}"
2021-06-08 01:23:25 +05:30
local base_config_file_ref="${CI_DEFAULT_BRANCH}"
2020-01-01 13:55:28 +05:30
if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="${CI_COMMIT_SHA}"; fi
2019-12-26 22:10:19 +05:30
local base_config_file="https://gitlab.com/gitlab-org/gitlab/raw/${base_config_file_ref}/scripts/review_apps/base-config.yaml"
2021-04-17 20:07:23 +05:30
echoinfo "Deploying ${release} to ${CI_ENVIRONMENT_URL} ..." true
2018-12-05 23:21:45 +05:30
2019-02-15 15:39:39 +05:30
IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror"
2020-05-24 23:13:21 +05:30
gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-ee"
gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-ee"
2020-06-23 00:09:42 +05:30
gitlab_webservice_image_repository="${IMAGE_REPOSITORY}/gitlab-webservice-ee"
2020-05-24 23:13:21 +05:30
gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-ee"
2019-02-15 15:39:39 +05:30
gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly"
2020-07-28 23:09:34 +05:30
gitaly_image_tag=$(parse_gitaly_image_tag)
2019-02-15 15:39:39 +05:30
gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
2020-05-24 23:13:21 +05:30
gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-ee"
2018-12-05 23:21:45 +05:30
2021-09-04 01:27:46 +05:30
ensure_namespace "${namespace}"
label_namespace "${namespace}" "tls=review-apps-tls" # label namespace for kubed to sync tls
2019-09-30 21:07:59 +05:30
create_application_secret
2018-12-05 23:21:45 +05:30
HELM_CMD=$(cat << EOF
2020-01-01 13:55:28 +05:30
helm upgrade \
--namespace="${namespace}" \
2021-09-04 01:27:46 +05:30
--create-namespace \
2020-01-01 13:55:28 +05:30
--install \
2019-12-04 20:38:33 +05:30
--wait \
2021-06-08 01:23:25 +05:30
--timeout "${HELM_INSTALL_TIMEOUT:-20m}" \
2020-01-01 13:55:28 +05:30
--set ci.branch="${CI_COMMIT_REF_NAME}" \
--set ci.commit.sha="${CI_COMMIT_SHORT_SHA}" \
--set ci.job.url="${CI_JOB_URL}" \
--set ci.pipeline.url="${CI_PIPELINE_URL}" \
--set releaseOverride="${release}" \
--set global.hosts.hostSuffix="${HOST_SUFFIX}" \
--set global.hosts.domain="${REVIEW_APPS_DOMAIN}" \
--set gitlab.migrations.image.repository="${gitlab_migrations_image_repository}" \
--set gitlab.migrations.image.tag="${CI_COMMIT_REF_SLUG}" \
--set gitlab.gitaly.image.repository="${gitlab_gitaly_image_repository}" \
2020-07-28 23:09:34 +05:30
--set gitlab.gitaly.image.tag="${gitaly_image_tag}" \
2020-01-01 13:55:28 +05:30
--set gitlab.gitlab-shell.image.repository="${gitlab_shell_image_repository}" \
--set gitlab.gitlab-shell.image.tag="v${GITLAB_SHELL_VERSION}" \
2020-05-24 23:13:21 +05:30
--set gitlab.sidekiq.annotations.commit="${CI_COMMIT_SHORT_SHA}" \
2020-01-01 13:55:28 +05:30
--set gitlab.sidekiq.image.repository="${gitlab_sidekiq_image_repository}" \
--set gitlab.sidekiq.image.tag="${CI_COMMIT_REF_SLUG}" \
2020-06-23 00:09:42 +05:30
--set gitlab.webservice.annotations.commit="${CI_COMMIT_SHORT_SHA}" \
--set gitlab.webservice.image.repository="${gitlab_webservice_image_repository}" \
--set gitlab.webservice.image.tag="${CI_COMMIT_REF_SLUG}" \
--set gitlab.webservice.workhorse.image="${gitlab_workhorse_image_repository}" \
--set gitlab.webservice.workhorse.tag="${CI_COMMIT_REF_SLUG}" \
2020-01-01 13:55:28 +05:30
--set gitlab.task-runner.image.repository="${gitlab_task_runner_image_repository}" \
--set gitlab.task-runner.image.tag="${CI_COMMIT_REF_SLUG}"
2019-09-30 21:07:59 +05:30
EOF
)
2021-06-08 01:23:25 +05:30
if [ -n "${REVIEW_APPS_EE_LICENSE_FILE}" ]; then
2019-12-26 22:10:19 +05:30
HELM_CMD=$(cat << EOF
${HELM_CMD} \
2020-01-01 13:55:28 +05:30
--set global.gitlab.license.secret="shared-gitlab-license"
2019-12-26 22:10:19 +05:30
EOF
)
fi
2019-09-30 21:07:59 +05:30
HELM_CMD=$(cat << EOF
2019-12-26 22:10:19 +05:30
${HELM_CMD} \
--version="${CI_PIPELINE_ID}-${CI_JOB_ID}" \
-f "${base_config_file}" \
2021-09-04 01:27:46 +05:30
-v "${HELM_LOG_VERBOSITY:-1}" \
2021-06-08 01:23:25 +05:30
"${release}" "gitlab-${GITLAB_HELM_CHART_REF}"
2018-12-05 23:21:45 +05:30
EOF
)
2019-07-07 11:18:12 +05:30
echoinfo "Deploying with:"
echoinfo "${HELM_CMD}"
2018-12-05 23:21:45 +05:30
2019-09-30 21:07:59 +05:30
eval "${HELM_CMD}"
}
2021-09-04 01:27:46 +05:30
function verify_deploy() {
echoinfo "Verifying deployment at ${CI_ENVIRONMENT_URL}"
if retry "test_url \"${CI_ENVIRONMENT_URL}\" curl_output.txt"; then
echoinfo "Review app is deployed to ${CI_ENVIRONMENT_URL}"
return 0
else
echoerr "Review app is not available at ${CI_ENVIRONMENT_URL}. See curl_output.txt artifact for detail."
return 1
fi
}
2019-09-30 21:07:59 +05:30
function display_deployment_debug() {
2021-09-04 01:27:46 +05:30
local namespace="${CI_ENVIRONMENT_SLUG}"
2020-01-01 13:55:28 +05:30
local release="${CI_ENVIRONMENT_SLUG}"
2019-12-21 20:55:43 +05:30
# Get all pods for this release
2020-01-01 13:55:28 +05:30
echoinfo "Pods for release ${release}"
kubectl get pods --namespace "${namespace}" -lrelease=${release}
2019-12-04 20:38:33 +05:30
2019-12-21 20:55:43 +05:30
# Get all non-completed jobs
2020-01-01 13:55:28 +05:30
echoinfo "Unsuccessful Jobs for release ${release}"
kubectl get jobs --namespace "${namespace}" -lrelease=${release} --field-selector=status.successful!=1
2019-07-31 22:56:46 +05:30
}