debian-mirror-gitlab/.gitlab/ci/cache-repo.gitlab-ci.yml

64 lines
3.5 KiB
YAML
Raw Normal View History

2021-04-29 21:17:54 +05:30
# Builds a cached .tar.gz of the $CI_DEFAULT_BRANCH branch with full history and
2020-01-01 13:55:28 +05:30
# uploads it to Google Cloud Storage. This archive is downloaded by a
# script defined by a CI/CD variable named CI_PRE_CLONE_SCRIPT. This has
# two benefits:
#
# 1. It speeds up builds. A 800 MB download only takes seconds.
# 2. It significantly reduces load on the file server. Smaller deltas
# means less time spent in git pack-objects.
#
# Since the destination directory of the archive depends on the project
# ID, this is only run on GitLab.com.
#
# CI_REPO_CACHE_CREDENTIALS contains the Google Cloud service account
# JSON for uploading to the gitlab-ci-git-repo-cache bucket. These
# credentials are stored in the Production vault.
#
# Note that this bucket should be located in the same continent as the
# runner, or network egress charges will apply:
# https://cloud.google.com/storage/pricing
cache-repo:
2020-03-13 15:44:24 +05:30
extends: .cache-repo:rules
2020-01-01 13:55:28 +05:30
image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
stage: sync
variables:
2020-03-13 15:44:24 +05:30
GIT_STRATEGY: none
2021-01-29 00:20:46 +05:30
SHALLOW_CLONE_TAR_FILENAME: gitlab-master-shallow.tar
FULL_CLONE_TAR_FILENAME: gitlab-master.tar
before_script:
- '[ -z "$CI_REPO_CACHE_CREDENTIALS" ] || gcloud auth activate-service-account --key-file=$CI_REPO_CACHE_CREDENTIALS'
2020-01-01 13:55:28 +05:30
script:
2021-06-08 01:23:25 +05:30
# Enable shallow repo caching unless the $DISABLE_SHALLOW_REPO_CACHING variable exists (in the case the shallow clone caching isn't working well)
2021-04-17 20:07:23 +05:30
# The `git repack` call works around a Git bug with shallow clones: https://gitlab.com/gitlab-org/git/-/issues/86
2021-06-08 01:23:25 +05:30
- if [ -z "$DISABLE_SHALLOW_REPO_CACHING" ]; then
2021-01-29 00:20:46 +05:30
cd .. && rm -rf $CI_PROJECT_NAME;
today=$(date +%Y-%m-%d);
year=$(date +%Y);
last_year=`expr $year - 1`;
one_year_ago=$(echo $today | sed "s/$year/$last_year/");
echo "Cloning $CI_REPOSITORY_URL into $CI_PROJECT_NAME with commits from $one_year_ago.";
time git clone --progress --no-checkout --shallow-since=$one_year_ago $CI_REPOSITORY_URL $CI_PROJECT_NAME;
cd $CI_PROJECT_NAME;
2021-04-17 20:07:23 +05:30
time git repack -d;
2021-01-29 00:20:46 +05:30
echo "Archiving $CI_PROJECT_NAME into /tmp/$SHALLOW_CLONE_TAR_FILENAME.";
2021-04-29 21:17:54 +05:30
time git remote rm origin;
2021-01-29 00:20:46 +05:30
time tar cf /tmp/$SHALLOW_CLONE_TAR_FILENAME .;
echo "GZipping /tmp/$SHALLOW_CLONE_TAR_FILENAME.";
time gzip /tmp/$SHALLOW_CLONE_TAR_FILENAME;
[ -z "$CI_REPO_CACHE_CREDENTIALS" ] || (echo "Uploading /tmp/$SHALLOW_CLONE_TAR_FILENAME.gz to GCloud." && time gsutil cp /tmp/$SHALLOW_CLONE_TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/$SHALLOW_CLONE_TAR_FILENAME.gz);
fi
2021-06-08 01:23:25 +05:30
# Disable the full repo caching unless the $DISABLE_SHALLOW_REPO_CACHING variable exists (in the case the shallow clone caching isn't working well)
- if [ -n "$DISABLE_SHALLOW_REPO_CACHING" ]; then
2021-01-29 00:20:46 +05:30
cd .. && rm -rf $CI_PROJECT_NAME;
echo "Cloning $CI_REPOSITORY_URL into $CI_PROJECT_NAME.";
time git clone --progress $CI_REPOSITORY_URL $CI_PROJECT_NAME;
cd $CI_PROJECT_NAME;
2021-04-29 21:17:54 +05:30
time git repack -d;
2021-01-29 00:20:46 +05:30
echo "Archiving $CI_PROJECT_NAME into /tmp/$FULL_CLONE_TAR_FILENAME.";
2021-04-29 21:17:54 +05:30
time git remote rm origin;
2021-01-29 00:20:46 +05:30
time tar cf /tmp/$FULL_CLONE_TAR_FILENAME .;
echo "GZipping /tmp/$FULL_CLONE_TAR_FILENAME.";
time gzip /tmp/$FULL_CLONE_TAR_FILENAME;
[ -z "$CI_REPO_CACHE_CREDENTIALS" ] || (echo "Uploading /tmp/$FULL_CLONE_TAR_FILENAME.gz to GCloud." && time gsutil cp /tmp/$FULL_CLONE_TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/$FULL_CLONE_TAR_FILENAME.gz);
fi