debian-mirror-gitlab/app/services/projects/lfs_pointers/lfs_download_service.rb

160 lines
4.4 KiB
Ruby
Raw Normal View History

2018-11-18 11:00:15 +05:30
# frozen_string_literal: true
2018-11-08 19:23:39 +05:30
# This service downloads and links lfs objects from a remote URL
module Projects
module LfsPointers
class LfsDownloadService < BaseService
2019-02-02 18:00:53 +05:30
SizeError = Class.new(StandardError)
OidError = Class.new(StandardError)
2020-11-24 15:15:51 +05:30
ResponseError = Class.new(StandardError)
LARGE_FILE_SIZE = 1.megabytes
2018-12-15 14:41:45 +05:30
2019-02-02 18:00:53 +05:30
attr_reader :lfs_download_object
delegate :oid, :size, :credentials, :sanitized_url, to: :lfs_download_object, prefix: :lfs
2018-11-08 19:23:39 +05:30
2019-02-02 18:00:53 +05:30
def initialize(project, lfs_download_object)
super(project)
2018-11-08 19:23:39 +05:30
2019-02-02 18:00:53 +05:30
@lfs_download_object = lfs_download_object
end
2018-11-08 19:23:39 +05:30
2019-02-02 18:00:53 +05:30
def execute
return unless project&.lfs_enabled? && lfs_download_object
return error("LFS file with oid #{lfs_oid} has invalid attributes") unless lfs_download_object.valid?
2021-01-29 00:20:46 +05:30
return link_existing_lfs_object! if lfs_size > LARGE_FILE_SIZE && lfs_object
2018-11-08 19:23:39 +05:30
2019-02-02 18:00:53 +05:30
wrap_download_errors do
download_lfs_file!
2018-11-08 19:23:39 +05:30
end
end
private
2019-02-02 18:00:53 +05:30
def wrap_download_errors(&block)
yield
2020-11-24 15:15:51 +05:30
rescue SizeError, OidError, ResponseError, StandardError => e
2019-02-02 18:00:53 +05:30
error("LFS file with oid #{lfs_oid} could't be downloaded from #{lfs_sanitized_url}: #{e.message}")
end
def download_lfs_file!
with_tmp_file do |tmp_file|
download_and_save_file!(tmp_file)
2020-03-07 23:17:34 +05:30
project.lfs_objects << find_or_create_lfs_object(tmp_file)
2019-02-02 18:00:53 +05:30
success
2019-01-03 12:48:30 +05:30
end
end
2020-03-07 23:17:34 +05:30
def find_or_create_lfs_object(tmp_file)
lfs_obj = LfsObject.safe_find_or_create_by!(
oid: lfs_oid,
size: lfs_size
)
lfs_obj.update!(file: tmp_file) unless lfs_obj.file.file
lfs_obj
end
2019-02-02 18:00:53 +05:30
def download_and_save_file!(file)
digester = Digest::SHA256.new
2020-11-24 15:15:51 +05:30
fetch_file do |fragment|
2019-02-02 18:00:53 +05:30
digester << fragment
2019-01-03 12:48:30 +05:30
file.write(fragment)
2019-02-02 18:00:53 +05:30
raise_size_error! if file.size > lfs_size
2019-01-03 12:48:30 +05:30
end
2019-02-02 18:00:53 +05:30
raise_size_error! if file.size != lfs_size
raise_oid_error! if digester.hexdigest != lfs_oid
end
2018-11-08 19:23:39 +05:30
2019-02-02 18:00:53 +05:30
def download_headers
{ stream_body: true }.tap do |headers|
if lfs_credentials[:user].present? || lfs_credentials[:password].present?
2018-11-08 19:23:39 +05:30
# Using authentication headers in the request
2019-02-02 18:00:53 +05:30
headers[:basic_auth] = { username: lfs_credentials[:user], password: lfs_credentials[:password] }
2018-11-08 19:23:39 +05:30
end
end
end
2020-11-24 15:15:51 +05:30
def fetch_file(&block)
response = Gitlab::HTTP.get(lfs_sanitized_url, download_headers, &block)
raise ResponseError, "Received error code #{response.code}" unless response.success?
end
2019-02-02 18:00:53 +05:30
def with_tmp_file
2018-11-08 19:23:39 +05:30
create_tmp_storage_dir
2019-02-02 18:00:53 +05:30
File.open(tmp_filename, 'wb') do |file|
2019-07-07 11:18:12 +05:30
yield file
rescue StandardError => e
# If the lfs file is successfully downloaded it will be removed
# when it is added to the project's lfs files.
# Nevertheless if any excetion raises the file would remain
# in the file system. Here we ensure to remove it
File.unlink(file) if File.exist?(file)
raise e
2019-02-02 18:00:53 +05:30
end
end
def tmp_filename
File.join(tmp_storage_dir, lfs_oid)
2018-11-08 19:23:39 +05:30
end
def create_tmp_storage_dir
FileUtils.makedirs(tmp_storage_dir) unless Dir.exist?(tmp_storage_dir)
end
def tmp_storage_dir
@tmp_storage_dir ||= File.join(storage_dir, 'tmp', 'download')
end
def storage_dir
@storage_dir ||= Gitlab.config.lfs.storage_path
end
2019-02-02 18:00:53 +05:30
def raise_size_error!
raise SizeError, 'Size mistmatch'
end
def raise_oid_error!
raise OidError, 'Oid mismatch'
end
def error(message, http_status = nil)
log_error(message)
super
end
2020-11-24 15:15:51 +05:30
def lfs_object
@lfs_object ||= LfsObject.find_by_oid(lfs_oid)
end
def link_existing_lfs_object!
existing_file = lfs_object.file.open
buffer_size = 0
result = fetch_file do |fragment|
unless fragment == existing_file.read(fragment.size)
break error("LFS file with oid #{lfs_oid} cannot be linked with an existing LFS object")
end
buffer_size += fragment.size
break success if buffer_size > LARGE_FILE_SIZE
end
project.lfs_objects << lfs_object
result
ensure
existing_file&.close
end
2018-11-08 19:23:39 +05:30
end
end
end