debian-mirror-gitlab/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml

968 lines
27 KiB
YAML
Raw Normal View History

2018-03-17 18:26:18 +05:30
# Auto DevOps
# This CI/CD configuration provides a standard pipeline for
# * building a Docker image (using a buildpack if necessary),
# * storing the image in the container registry,
# * running tests from a buildpack,
# * running code quality analysis,
# * creating a review app for each topic branch,
# * and continuous deployment to production
#
2018-11-08 19:23:39 +05:30
# Test jobs may be disabled by setting environment variables:
# * test: TEST_DISABLED
# * code_quality: CODE_QUALITY_DISABLED
# * license_management: LICENSE_MANAGEMENT_DISABLED
# * performance: PERFORMANCE_DISABLED
# * sast: SAST_DISABLED
# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED
# * container_scanning: CONTAINER_SCANNING_DISABLED
# * dast: DAST_DISABLED
# * review: REVIEW_DISABLED
# * stop_review: REVIEW_DISABLED
#
2018-03-17 18:26:18 +05:30
# In order to deploy, you must have a Kubernetes cluster configured either
# via a project integration, or via group/project variables.
2019-03-02 22:35:43 +05:30
# KUBE_INGRESS_BASE_DOMAIN must also be set on the cluster settings,
# as a variable at the group or project level, or manually added below.
2018-03-17 18:26:18 +05:30
#
2018-10-15 14:42:47 +05:30
# Continuous deployment to production is enabled by default.
2018-12-05 23:21:45 +05:30
# If you want to deploy to staging first, set STAGING_ENABLED environment variable.
# If you want to enable incremental rollout, either manual or time based,
2019-02-15 15:39:39 +05:30
# set INCREMENTAL_ROLLOUT_MODE environment variable to "manual" or "timed".
2018-11-08 19:23:39 +05:30
# If you want to use canary deployments, set CANARY_ENABLED environment variable.
2018-03-17 18:26:18 +05:30
#
# If Auto DevOps fails to detect the proper buildpack, or if you want to
# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the
# repository URL of the buildpack.
# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142
# If you need multiple buildpacks, add a file to your project called
# `.buildpacks` that contains the URLs, one on each line, in order.
# Note: Auto CI does not work with multiple buildpacks yet
image: alpine:latest
variables:
2019-03-02 22:35:43 +05:30
# KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level.
# KUBE_INGRESS_BASE_DOMAIN: domain.example.com
2018-03-17 18:26:18 +05:30
POSTGRES_USER: user
POSTGRES_PASSWORD: testing-password
POSTGRES_ENABLED: "true"
POSTGRES_DB: $CI_ENVIRONMENT_SLUG
2019-03-02 22:35:43 +05:30
KUBERNETES_VERSION: 1.11.6
HELM_VERSION: 2.12.2
2018-03-17 18:26:18 +05:30
2018-11-20 20:47:30 +05:30
DOCKER_DRIVER: overlay2
2018-03-17 18:26:18 +05:30
stages:
- build
- test
- review
- dast
- staging
- canary
- production
2018-12-05 23:21:45 +05:30
- incremental rollout 10%
- incremental rollout 25%
- incremental rollout 50%
- incremental rollout 100%
2018-03-17 18:26:18 +05:30
- performance
- cleanup
build:
stage: build
2018-05-09 12:01:36 +05:30
image: docker:stable-git
2018-03-17 18:26:18 +05:30
services:
2018-05-09 12:01:36 +05:30
- docker:stable-dind
2018-03-17 18:26:18 +05:30
script:
- setup_docker
- build
only:
- branches
test:
services:
- postgres:latest
variables:
POSTGRES_DB: test
stage: test
image: gliderlabs/herokuish:latest
script:
- setup_test_db
- cp -R . /tmp/app
- /bin/herokuish buildpack test
only:
- branches
2018-11-08 19:23:39 +05:30
except:
variables:
- $TEST_DISABLED
code_quality:
stage: test
image: docker:stable
allow_failure: true
services:
- docker:stable-dind
script:
- setup_docker
- code_quality
artifacts:
paths: [gl-code-quality-report.json]
only:
- branches
except:
variables:
- $CODE_QUALITY_DISABLED
2018-03-17 18:26:18 +05:30
2018-11-08 19:23:39 +05:30
license_management:
stage: test
2019-03-02 22:35:43 +05:30
image:
2018-12-13 13:39:08 +05:30
name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable"
entrypoint: [""]
2018-03-17 18:26:18 +05:30
allow_failure: true
script:
2018-11-08 19:23:39 +05:30
- license_management
2018-03-17 18:26:18 +05:30
artifacts:
2018-11-08 19:23:39 +05:30
paths: [gl-license-management-report.json]
only:
2018-12-13 13:39:08 +05:30
refs:
- branches
2018-12-05 23:21:45 +05:30
variables:
- $GITLAB_FEATURES =~ /\blicense_management\b/
2018-11-08 19:23:39 +05:30
except:
variables:
- $LICENSE_MANAGEMENT_DISABLED
2018-03-17 18:26:18 +05:30
performance:
stage: performance
2018-05-09 12:01:36 +05:30
image: docker:stable
2018-03-17 18:26:18 +05:30
allow_failure: true
services:
2018-05-09 12:01:36 +05:30
- docker:stable-dind
2018-03-17 18:26:18 +05:30
script:
- setup_docker
- performance
artifacts:
paths:
- performance.json
- sitespeed-results/
only:
refs:
- branches
kubernetes: active
2018-11-08 19:23:39 +05:30
except:
variables:
- $PERFORMANCE_DISABLED
2018-03-17 18:26:18 +05:30
sast:
2018-11-08 19:23:39 +05:30
stage: test
2018-05-09 12:01:36 +05:30
image: docker:stable
2018-03-17 18:26:18 +05:30
allow_failure: true
services:
2018-05-09 12:01:36 +05:30
- docker:stable-dind
2018-03-17 18:26:18 +05:30
script:
- setup_docker
- sast
artifacts:
2019-02-15 15:39:39 +05:30
reports:
sast: gl-sast-report.json
2018-11-08 19:23:39 +05:30
only:
2018-12-05 23:21:45 +05:30
refs:
- branches
variables:
- $GITLAB_FEATURES =~ /\bsast\b/
2018-11-08 19:23:39 +05:30
except:
variables:
- $SAST_DISABLED
2018-03-17 18:26:18 +05:30
2018-05-09 12:01:36 +05:30
dependency_scanning:
2018-11-08 19:23:39 +05:30
stage: test
2018-05-09 12:01:36 +05:30
image: docker:stable
allow_failure: true
services:
- docker:stable-dind
script:
- setup_docker
- dependency_scanning
artifacts:
2019-02-15 15:39:39 +05:30
reports:
dependency_scanning: gl-dependency-scanning-report.json
2018-11-08 19:23:39 +05:30
only:
2018-12-05 23:21:45 +05:30
refs:
- branches
variables:
- $GITLAB_FEATURES =~ /\bdependency_scanning\b/
2018-11-08 19:23:39 +05:30
except:
variables:
- $DEPENDENCY_SCANNING_DISABLED
2018-10-15 14:42:47 +05:30
2018-11-08 19:23:39 +05:30
container_scanning:
stage: test
2018-05-09 12:01:36 +05:30
image: docker:stable
2018-03-17 18:26:18 +05:30
allow_failure: true
services:
2018-05-09 12:01:36 +05:30
- docker:stable-dind
2018-03-17 18:26:18 +05:30
script:
- setup_docker
2018-11-08 19:23:39 +05:30
- container_scanning
2018-03-17 18:26:18 +05:30
artifacts:
2018-11-08 19:23:39 +05:30
paths: [gl-container-scanning-report.json]
only:
2018-12-05 23:21:45 +05:30
refs:
- branches
variables:
2018-12-13 13:39:08 +05:30
- $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
2018-11-08 19:23:39 +05:30
except:
variables:
- $CONTAINER_SCANNING_DISABLED
2018-03-17 18:26:18 +05:30
dast:
stage: dast
allow_failure: true
2018-03-27 19:54:05 +05:30
image: registry.gitlab.com/gitlab-org/security-products/zaproxy
2018-03-17 18:26:18 +05:30
variables:
POSTGRES_DB: "false"
script:
- dast
artifacts:
paths: [gl-dast-report.json]
only:
refs:
- branches
kubernetes: active
2018-12-05 23:21:45 +05:30
variables:
- $GITLAB_FEATURES =~ /\bdast\b/
2018-03-17 18:26:18 +05:30
except:
2018-11-08 19:23:39 +05:30
refs:
- master
variables:
- $DAST_DISABLED
2018-03-17 18:26:18 +05:30
review:
stage: review
script:
- check_kube_domain
- install_dependencies
- download_chart
- ensure_namespace
2018-12-05 23:21:45 +05:30
- initialize_tiller
2018-03-17 18:26:18 +05:30
- create_secret
- deploy
- persist_environment_url
environment:
name: review/$CI_COMMIT_REF_NAME
2019-03-02 22:35:43 +05:30
url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
2018-03-17 18:26:18 +05:30
on_stop: stop_review
artifacts:
paths: [environment_url.txt]
only:
refs:
- branches
kubernetes: active
except:
2018-11-08 19:23:39 +05:30
refs:
- master
variables:
- $REVIEW_DISABLED
2018-03-17 18:26:18 +05:30
stop_review:
stage: cleanup
variables:
GIT_STRATEGY: none
script:
- install_dependencies
2018-12-05 23:21:45 +05:30
- initialize_tiller
2018-03-17 18:26:18 +05:30
- delete
environment:
name: review/$CI_COMMIT_REF_NAME
action: stop
when: manual
allow_failure: true
only:
refs:
- branches
kubernetes: active
except:
2018-11-08 19:23:39 +05:30
refs:
- master
variables:
- $REVIEW_DISABLED
2018-03-17 18:26:18 +05:30
# Staging deploys are disabled by default since
# continuous deployment to production is enabled by default
# If you prefer to automatically deploy to staging and
2018-10-15 14:42:47 +05:30
# only manually promote to production, enable this job by setting
# STAGING_ENABLED.
2018-03-17 18:26:18 +05:30
2018-10-15 14:42:47 +05:30
staging:
2018-03-17 18:26:18 +05:30
stage: staging
script:
- check_kube_domain
- install_dependencies
- download_chart
- ensure_namespace
2018-12-05 23:21:45 +05:30
- initialize_tiller
2018-03-17 18:26:18 +05:30
- create_secret
- deploy
environment:
name: staging
2019-03-02 22:35:43 +05:30
url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN
2018-03-17 18:26:18 +05:30
only:
refs:
- master
kubernetes: active
2018-10-15 14:42:47 +05:30
variables:
- $STAGING_ENABLED
2018-03-17 18:26:18 +05:30
2018-11-08 19:23:39 +05:30
# Canaries are also disabled by default, but if you want them,
# and know what the downsides are, you can enable this by setting
# CANARY_ENABLED.
2018-03-17 18:26:18 +05:30
2018-11-08 19:23:39 +05:30
canary:
2018-03-17 18:26:18 +05:30
stage: canary
script:
- check_kube_domain
- install_dependencies
- download_chart
- ensure_namespace
2018-12-05 23:21:45 +05:30
- initialize_tiller
2018-03-17 18:26:18 +05:30
- create_secret
- deploy canary
environment:
name: production
2019-03-02 22:35:43 +05:30
url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
2018-03-17 18:26:18 +05:30
when: manual
only:
refs:
- master
kubernetes: active
2018-11-08 19:23:39 +05:30
variables:
- $CANARY_ENABLED
2018-03-17 18:26:18 +05:30
2018-10-15 14:42:47 +05:30
.production: &production_template
stage: production
script:
- check_kube_domain
- install_dependencies
- download_chart
- ensure_namespace
2018-12-05 23:21:45 +05:30
- initialize_tiller
2018-10-15 14:42:47 +05:30
- create_secret
- deploy
- delete canary
- delete rollout
- persist_environment_url
environment:
name: production
2019-03-02 22:35:43 +05:30
url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
2018-10-15 14:42:47 +05:30
artifacts:
paths: [environment_url.txt]
2018-03-17 18:26:18 +05:30
production:
2018-10-15 14:42:47 +05:30
<<: *production_template
only:
refs:
- master
kubernetes: active
except:
variables:
- $STAGING_ENABLED
2018-11-08 19:23:39 +05:30
- $CANARY_ENABLED
2018-10-15 14:42:47 +05:30
- $INCREMENTAL_ROLLOUT_ENABLED
2018-12-05 23:21:45 +05:30
- $INCREMENTAL_ROLLOUT_MODE
2018-10-15 14:42:47 +05:30
production_manual:
<<: *production_template
when: manual
allow_failure: false
only:
refs:
- master
kubernetes: active
variables:
- $STAGING_ENABLED
2018-11-08 19:23:39 +05:30
- $CANARY_ENABLED
2018-10-15 14:42:47 +05:30
except:
variables:
- $INCREMENTAL_ROLLOUT_ENABLED
2018-12-05 23:21:45 +05:30
- $INCREMENTAL_ROLLOUT_MODE
2018-10-15 14:42:47 +05:30
# This job implements incremental rollout on for every push to `master`.
.rollout: &rollout_template
2018-03-17 18:26:18 +05:30
script:
- check_kube_domain
- install_dependencies
- download_chart
- ensure_namespace
2018-12-05 23:21:45 +05:30
- initialize_tiller
2018-03-17 18:26:18 +05:30
- create_secret
2018-10-15 14:42:47 +05:30
- deploy rollout $ROLLOUT_PERCENTAGE
- scale stable $((100-ROLLOUT_PERCENTAGE))
2018-03-17 18:26:18 +05:30
- delete canary
- persist_environment_url
environment:
name: production
2019-03-02 22:35:43 +05:30
url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
2018-03-17 18:26:18 +05:30
artifacts:
paths: [environment_url.txt]
2018-10-15 14:42:47 +05:30
2018-12-05 23:21:45 +05:30
.manual_rollout_template: &manual_rollout_template
2018-10-15 14:42:47 +05:30
<<: *rollout_template
2018-12-05 23:21:45 +05:30
stage: production
2018-10-15 14:42:47 +05:30
when: manual
2018-12-05 23:21:45 +05:30
# This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4)
2018-10-15 14:42:47 +05:30
only:
refs:
- master
kubernetes: active
variables:
2018-12-05 23:21:45 +05:30
- $INCREMENTAL_ROLLOUT_MODE == "manual"
2018-10-15 14:42:47 +05:30
- $INCREMENTAL_ROLLOUT_ENABLED
2018-12-05 23:21:45 +05:30
except:
variables:
- $INCREMENTAL_ROLLOUT_MODE == "timed"
2018-10-15 14:42:47 +05:30
2018-12-05 23:21:45 +05:30
.timed_rollout_template: &timed_rollout_template
2018-10-15 14:42:47 +05:30
<<: *rollout_template
2018-12-05 23:21:45 +05:30
when: delayed
start_in: 5 minutes
2018-10-15 14:42:47 +05:30
only:
refs:
- master
kubernetes: active
variables:
2018-12-05 23:21:45 +05:30
- $INCREMENTAL_ROLLOUT_MODE == "timed"
timed rollout 10%:
<<: *timed_rollout_template
stage: incremental rollout 10%
variables:
ROLLOUT_PERCENTAGE: 10
timed rollout 25%:
<<: *timed_rollout_template
stage: incremental rollout 25%
variables:
ROLLOUT_PERCENTAGE: 25
timed rollout 50%:
<<: *timed_rollout_template
stage: incremental rollout 50%
variables:
ROLLOUT_PERCENTAGE: 50
timed rollout 100%:
<<: *timed_rollout_template
<<: *production_template
stage: incremental rollout 100%
variables:
ROLLOUT_PERCENTAGE: 100
rollout 10%:
<<: *manual_rollout_template
variables:
ROLLOUT_PERCENTAGE: 10
rollout 25%:
<<: *manual_rollout_template
variables:
ROLLOUT_PERCENTAGE: 25
2018-10-15 14:42:47 +05:30
rollout 50%:
2018-12-05 23:21:45 +05:30
<<: *manual_rollout_template
2018-10-15 14:42:47 +05:30
variables:
ROLLOUT_PERCENTAGE: 50
rollout 100%:
2018-12-05 23:21:45 +05:30
<<: *manual_rollout_template
2018-10-15 14:42:47 +05:30
<<: *production_template
allow_failure: false
2018-03-17 18:26:18 +05:30
# ---------------------------------------------------------------------------
.auto_devops: &auto_devops |
# Auto DevOps variables and functions
[[ "$TRACE" ]] && set -x
auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}
export DATABASE_URL=${DATABASE_URL-$auto_database_url}
export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
export CI_APPLICATION_TAG=$CI_COMMIT_SHA
export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID}
export TILLER_NAMESPACE=$KUBE_NAMESPACE
2018-03-27 19:54:05 +05:30
# Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products
export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
2018-03-17 18:26:18 +05:30
2018-12-05 23:21:45 +05:30
function registry_login() {
2018-03-17 18:26:18 +05:30
if [[ -n "$CI_REGISTRY_USER" ]]; then
echo "Logging to GitLab Container Registry with CI credentials..."
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
echo ""
fi
2018-12-05 23:21:45 +05:30
}
function container_scanning() {
registry_login
2018-03-17 18:26:18 +05:30
docker run -d --name db arminc/clair-db:latest
2018-10-15 14:42:47 +05:30
docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.1
2018-03-17 18:26:18 +05:30
apk add -U wget ca-certificates
docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}
wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
mv clair-scanner_linux_amd64 clair-scanner
chmod +x clair-scanner
touch clair-whitelist.yml
2018-05-09 12:01:36 +05:30
retries=0
echo "Waiting for clair daemon to start"
while( ! wget -T 10 -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
2018-11-08 19:23:39 +05:30
./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true
2018-03-17 18:26:18 +05:30
}
2018-11-08 19:23:39 +05:30
function code_quality() {
2018-03-27 19:54:05 +05:30
docker run --env SOURCE_CODE="$PWD" \
--volume "$PWD":/code \
--volume /var/run/docker.sock:/var/run/docker.sock \
"registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
2018-03-17 18:26:18 +05:30
}
2018-11-08 19:23:39 +05:30
function license_management() {
2018-12-13 13:39:08 +05:30
/run.sh analyze .
2018-11-08 19:23:39 +05:30
}
2018-03-17 18:26:18 +05:30
function sast() {
case "$CI_SERVER_VERSION" in
*-ee)
# Deprecation notice for CONFIDENCE_LEVEL variable
if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then
SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL"
echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL"
fi
docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \
--volume "$PWD:/code" \
--volume /var/run/docker.sock:/var/run/docker.sock \
2018-03-27 19:54:05 +05:30
"registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code
2018-03-17 18:26:18 +05:30
;;
*)
echo "GitLab EE is required"
;;
esac
}
2018-05-09 12:01:36 +05:30
function dependency_scanning() {
case "$CI_SERVER_VERSION" in
*-ee)
docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \
--volume "$PWD:/code" \
--volume /var/run/docker.sock:/var/run/docker.sock \
"registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
;;
*)
echo "GitLab EE is required"
;;
esac
}
2018-10-15 14:42:47 +05:30
function get_replicas() {
track="${1:-stable}"
percentage="${2:-100}"
2018-03-17 18:26:18 +05:30
env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' )
env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' )
2018-10-15 14:42:47 +05:30
if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then
2018-03-17 18:26:18 +05:30
# for stable track get number of replicas from `PRODUCTION_REPLICAS`
eval new_replicas=\$${env_slug}_REPLICAS
2018-05-09 12:01:36 +05:30
if [[ -z "$new_replicas" ]]; then
new_replicas=$REPLICAS
fi
2018-03-17 18:26:18 +05:30
else
# for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
eval new_replicas=\$${env_track}_${env_slug}_REPLICAS
2018-05-09 12:01:36 +05:30
if [[ -z "$new_replicas" ]]; then
eval new_replicas=\${env_track}_REPLICAS
fi
2018-03-17 18:26:18 +05:30
fi
2018-10-15 14:42:47 +05:30
replicas="${new_replicas:-1}"
replicas="$(($replicas * $percentage / 100))"
# always return at least one replicas
if [[ $replicas -gt 0 ]]; then
echo "$replicas"
else
echo 1
fi
}
2019-02-15 15:39:39 +05:30
# Extracts variables prefixed with K8S_SECRET_
# and creates a Kubernetes secret.
#
# e.g. If we have the following environment variables:
# K8S_SECRET_A=value1
# K8S_SECRET_B=multi\ word\ value
#
# Then we will create a secret with the following key-value pairs:
# data:
# A: dmFsdWUxCg==
# B: bXVsdGkgd29yZCB2YWx1ZQo=
function create_application_secret() {
track="${1-stable}"
export APPLICATION_SECRET_NAME=$(application_secret_name "$track")
2019-03-02 22:35:43 +05:30
env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables
2019-02-15 15:39:39 +05:30
2019-03-02 22:35:43 +05:30
kubectl create secret \
-n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \
--from-env-file k8s_prefixed_variables -o yaml --dry-run |
kubectl replace -n "$KUBE_NAMESPACE" --force -f -
export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1)
rm k8s_prefixed_variables
2019-02-15 15:39:39 +05:30
}
function deploy_name() {
name="$CI_ENVIRONMENT_SLUG"
track="${1-stable}"
if [[ "$track" != "stable" ]]; then
name="$name-$track"
fi
echo $name
}
function application_secret_name() {
track="${1-stable}"
name=$(deploy_name "$track")
echo "${name}-secret"
}
2018-10-15 14:42:47 +05:30
function deploy() {
track="${1-stable}"
percentage="${2:-100}"
2019-02-15 15:39:39 +05:30
name=$(deploy_name "$track")
2018-10-15 14:42:47 +05:30
replicas="1"
service_enabled="true"
postgres_enabled="$POSTGRES_ENABLED"
# if track is different than stable,
# re-use all attached resources
if [[ "$track" != "stable" ]]; then
service_enabled="false"
postgres_enabled="false"
2018-03-17 18:26:18 +05:30
fi
2018-10-15 14:42:47 +05:30
replicas=$(get_replicas "$track" "$percentage")
2018-03-17 18:26:18 +05:30
if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then
secret_name='gitlab-registry'
else
secret_name=''
fi
2019-02-15 15:39:39 +05:30
create_application_secret "$track"
2019-03-02 22:35:43 +05:30
env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]')
eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS
if [ -n "$env_ADDITIONAL_HOSTS" ]; then
additional_hosts="{$env_ADDITIONAL_HOSTS}"
elif [ -n "$ADDITIONAL_HOSTS" ]; then
additional_hosts="{$ADDITIONAL_HOSTS}"
fi
2018-12-05 23:21:45 +05:30
if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then
2019-02-15 15:39:39 +05:30
echo "Deploying first release with database initialization..."
2018-12-05 23:21:45 +05:30
helm upgrade --install \
--wait \
--set service.enabled="$service_enabled" \
--set releaseOverride="$CI_ENVIRONMENT_SLUG" \
--set image.repository="$CI_APPLICATION_REPOSITORY" \
--set image.tag="$CI_APPLICATION_TAG" \
--set image.pullPolicy=IfNotPresent \
--set image.secrets[0].name="$secret_name" \
--set application.track="$track" \
--set application.database_url="$DATABASE_URL" \
2019-02-15 15:39:39 +05:30
--set application.secretName="$APPLICATION_SECRET_NAME" \
2019-03-02 22:35:43 +05:30
--set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \
--set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \
2018-12-05 23:21:45 +05:30
--set service.url="$CI_ENVIRONMENT_URL" \
2019-03-02 22:35:43 +05:30
--set service.additionalHosts="$additional_hosts" \
2018-12-05 23:21:45 +05:30
--set replicaCount="$replicas" \
--set postgresql.enabled="$postgres_enabled" \
--set postgresql.nameOverride="postgres" \
--set postgresql.postgresUser="$POSTGRES_USER" \
--set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
--set postgresql.postgresDatabase="$POSTGRES_DB" \
--set application.initializeCommand="$DB_INITIALIZE" \
--namespace="$KUBE_NAMESPACE" \
"$name" \
chart/
2019-02-15 15:39:39 +05:30
echo "Deploying second release..."
2018-12-05 23:21:45 +05:30
helm upgrade --reuse-values \
--wait \
--set application.initializeCommand="" \
--set application.migrateCommand="$DB_MIGRATE" \
--namespace="$KUBE_NAMESPACE" \
"$name" \
chart/
else
2019-02-15 15:39:39 +05:30
echo "Deploying new release..."
2018-12-05 23:21:45 +05:30
helm upgrade --install \
--wait \
--set service.enabled="$service_enabled" \
--set releaseOverride="$CI_ENVIRONMENT_SLUG" \
--set image.repository="$CI_APPLICATION_REPOSITORY" \
--set image.tag="$CI_APPLICATION_TAG" \
--set image.pullPolicy=IfNotPresent \
--set image.secrets[0].name="$secret_name" \
--set application.track="$track" \
--set application.database_url="$DATABASE_URL" \
2019-02-15 15:39:39 +05:30
--set application.secretName="$APPLICATION_SECRET_NAME" \
2019-03-02 22:35:43 +05:30
--set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \
--set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \
2018-12-05 23:21:45 +05:30
--set service.url="$CI_ENVIRONMENT_URL" \
2019-03-02 22:35:43 +05:30
--set service.additionalHosts="$additional_hosts" \
2018-12-05 23:21:45 +05:30
--set replicaCount="$replicas" \
--set postgresql.enabled="$postgres_enabled" \
--set postgresql.nameOverride="postgres" \
--set postgresql.postgresUser="$POSTGRES_USER" \
--set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
--set postgresql.postgresDatabase="$POSTGRES_DB" \
--set application.migrateCommand="$DB_MIGRATE" \
--namespace="$KUBE_NAMESPACE" \
"$name" \
chart/
fi
kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name"
2018-03-17 18:26:18 +05:30
}
2018-10-15 14:42:47 +05:30
function scale() {
track="${1-stable}"
percentage="${2-100}"
2019-02-15 15:39:39 +05:30
name=$(deploy_name "$track")
2018-10-15 14:42:47 +05:30
replicas=$(get_replicas "$track" "$percentage")
if [[ -n "$(helm ls -q "^$name$")" ]]; then
helm upgrade --reuse-values \
--wait \
--set replicaCount="$replicas" \
--namespace="$KUBE_NAMESPACE" \
"$name" \
chart/
fi
}
2018-03-17 18:26:18 +05:30
function install_dependencies() {
apk add -U openssl curl tar gzip bash ca-certificates git
2018-12-05 23:21:45 +05:30
curl -L -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk
2018-11-20 20:47:30 +05:30
apk add glibc-2.28-r0.apk
rm glibc-2.28-r0.apk
2018-03-17 18:26:18 +05:30
curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx
mv linux-amd64/helm /usr/bin/
2018-12-05 23:21:45 +05:30
mv linux-amd64/tiller /usr/bin/
2018-03-17 18:26:18 +05:30
helm version --client
2018-12-05 23:21:45 +05:30
tiller -version
2018-03-17 18:26:18 +05:30
curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
chmod +x /usr/bin/kubectl
kubectl version --client
}
function setup_docker() {
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
export DOCKER_HOST='tcp://localhost:2375'
fi
fi
}
function setup_test_db() {
if [ -z ${KUBERNETES_PORT+x} ]; then
DB_HOST=postgres
else
DB_HOST=localhost
fi
export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}"
}
function download_chart() {
if [[ ! -d chart ]]; then
auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app}
auto_chart_name=$(basename $auto_chart)
auto_chart_name=${auto_chart_name%.tgz}
2018-11-18 11:00:15 +05:30
auto_chart_name=${auto_chart_name%.tar.gz}
2018-03-17 18:26:18 +05:30
else
auto_chart="chart"
auto_chart_name="chart"
fi
helm init --client-only
2019-03-02 22:35:43 +05:30
helm repo add gitlab ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io}
2018-03-17 18:26:18 +05:30
if [[ ! -d "$auto_chart" ]]; then
helm fetch ${auto_chart} --untar
fi
if [ "$auto_chart_name" != "chart" ]; then
mv ${auto_chart_name} chart
fi
helm dependency update chart/
helm dependency build chart/
}
function ensure_namespace() {
kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
}
2019-03-02 22:35:43 +05:30
# Function to ensure backwards compatibility with AUTO_DEVOPS_DOMAIN
function ensure_kube_ingress_base_domain() {
if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ] && [ -n "$AUTO_DEVOPS_DOMAIN" ] ; then
export KUBE_INGRESS_BASE_DOMAIN=$AUTO_DEVOPS_DOMAIN
fi
}
2018-03-17 18:26:18 +05:30
function check_kube_domain() {
2019-03-02 22:35:43 +05:30
ensure_kube_ingress_base_domain
if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ]; then
echo "In order to deploy or use Review Apps,"
echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set"
echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings"
echo "or by defining a variable at group or project level."
2018-03-17 18:26:18 +05:30
echo "You can also manually add it in .gitlab-ci.yml"
2019-03-02 22:35:43 +05:30
echo "AUTO_DEVOPS_DOMAIN support will be dropped on 12.0"
2018-03-17 18:26:18 +05:30
false
else
true
fi
}
function build() {
2018-12-05 23:21:45 +05:30
registry_login
2018-03-17 18:26:18 +05:30
if [[ -f Dockerfile ]]; then
echo "Building Dockerfile-based application..."
2018-11-20 20:47:30 +05:30
docker build \
--build-arg HTTP_PROXY="$HTTP_PROXY" \
--build-arg http_proxy="$http_proxy" \
--build-arg HTTPS_PROXY="$HTTPS_PROXY" \
--build-arg https_proxy="$https_proxy" \
--build-arg FTP_PROXY="$FTP_PROXY" \
--build-arg ftp_proxy="$ftp_proxy" \
--build-arg NO_PROXY="$NO_PROXY" \
--build-arg no_proxy="$no_proxy" \
-t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" .
2018-03-17 18:26:18 +05:30
else
echo "Building Heroku-based application using gliderlabs/herokuish docker image..."
2018-11-20 20:47:30 +05:30
docker run -i \
-e BUILDPACK_URL \
-e HTTP_PROXY \
-e http_proxy \
-e HTTPS_PROXY \
-e https_proxy \
-e FTP_PROXY \
-e ftp_proxy \
-e NO_PROXY \
-e no_proxy \
--name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" gliderlabs/herokuish /bin/herokuish buildpack build
2018-03-17 18:26:18 +05:30
docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
docker rm "$CI_CONTAINER_NAME" >/dev/null
echo ""
echo "Configuring $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG docker image..."
docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" /bin/herokuish procfile start web
docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
docker rm "$CI_CONTAINER_NAME" >/dev/null
echo ""
fi
echo "Pushing to GitLab Container Registry..."
docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
echo ""
}
2018-12-05 23:21:45 +05:30
function initialize_tiller() {
2018-03-17 18:26:18 +05:30
echo "Checking Tiller..."
2018-12-05 23:21:45 +05:30
2018-12-13 13:39:08 +05:30
export HELM_HOST="localhost:44134"
2018-12-05 23:21:45 +05:30
tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 &
echo "Tiller is listening on ${HELM_HOST}"
2018-03-17 18:26:18 +05:30
if ! helm version --debug; then
echo "Failed to init Tiller."
return 1
fi
echo ""
}
function create_secret() {
echo "Create secret..."
if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then
return
fi
kubectl create secret -n "$KUBE_NAMESPACE" \
docker-registry gitlab-registry \
--docker-server="$CI_REGISTRY" \
2018-10-15 14:42:47 +05:30
--docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \
--docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \
2018-03-17 18:26:18 +05:30
--docker-email="$GITLAB_USER_EMAIL" \
-o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -
}
function dast() {
export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
mkdir /zap/wrk/
/zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true
cp /zap/wrk/gl-dast-report.json .
}
function performance() {
export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
mkdir gitlab-exporter
wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js
mkdir sitespeed-results
if [ -f .gitlab-urls.txt ]
then
sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt
docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt
else
docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
fi
mv sitespeed-results/data/performance.json performance.json
}
function persist_environment_url() {
echo $CI_ENVIRONMENT_URL > environment_url.txt
}
function delete() {
track="${1-stable}"
2019-02-15 15:39:39 +05:30
name=$(deploy_name "$track")
2018-03-17 18:26:18 +05:30
if [[ -n "$(helm ls -q "^$name$")" ]]; then
2018-11-20 20:47:30 +05:30
helm delete --purge "$name"
2018-03-17 18:26:18 +05:30
fi
2019-02-15 15:39:39 +05:30
secret_name=$(application_secret_name "$track")
kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name"
2018-03-17 18:26:18 +05:30
}
before_script:
- *auto_devops