debian-mirror-gitlab/app/services/packages/debian/generate_distribution_service.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

272 lines
9.4 KiB
Ruby
Raw Normal View History

2021-06-08 01:23:25 +05:30
# frozen_string_literal: true
module Packages
module Debian
class GenerateDistributionService
include Gitlab::Utils::StrongMemoize
2022-08-13 15:12:31 +05:30
include ::Packages::FIPS
2021-06-08 01:23:25 +05:30
include ExclusiveLeaseGuard
2021-09-04 01:27:46 +05:30
ONE_HOUR = 1.hour.freeze
2021-06-08 01:23:25 +05:30
# used by ExclusiveLeaseGuard
DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze
# From https://salsa.debian.org/ftp-team/dak/-/blob/991aaa27a7f7aa773bb9c0cf2d516e383d9cffa0/setup/core-init.d/080_metadatakeys#L9
2021-10-27 15:23:28 +05:30
METADATA_KEYS = %w(
2021-06-08 01:23:25 +05:30
Package
Source
Binary
Version
Essential
Installed-Size
Maintainer
Uploaders
Original-Maintainer
Build-Depends
Build-Depends-Indep
Build-Conflicts
Build-Conflicts-Indep
Architecture
Standards-Version
Format
Files
Dm-Upload-Allowed
Vcs-Browse
Vcs-Hg
Vcs-Darcs
Vcs-Svn
Vcs-Git
Vcs-Browser
Vcs-Arch
Vcs-Bzr
Vcs-Mtn
Vcs-Cvs
Checksums-Sha256
Checksums-Sha1
Replaces
Provides
Depends
Pre-Depends
Recommends
Suggests
Enhances
Conflicts
Breaks
Description
Origin
Bugs
Multi-Arch
Homepage
Tag
Package-Type
Installer-Menu-Item
).freeze
def initialize(distribution)
@distribution = distribution
2021-09-04 01:27:46 +05:30
@oldest_kept_generated_at = nil
2021-06-08 01:23:25 +05:30
@md5sum = []
@sha256 = []
end
def execute
2022-08-13 15:12:31 +05:30
raise DisabledError, 'Debian registry is not FIPS compliant' if Gitlab::FIPS.enabled?
2021-06-08 01:23:25 +05:30
try_obtain_lease do
@distribution.transaction do
2021-09-04 01:27:46 +05:30
# We consider `apt-get update` can take at most one hour
# We keep all generations younger than one hour
# and the previous generation
@oldest_kept_generated_at = @distribution.component_files.updated_before(release_date - ONE_HOUR).maximum(:updated_at)
2021-06-08 01:23:25 +05:30
generate_component_files
generate_release
destroy_old_component_files
end
end
end
private
def generate_component_files
@distribution.components.ordered_by_name.each do |component|
@distribution.architectures.ordered_by_name.each do |architecture|
generate_component_file(component, :packages, architecture, :deb)
2021-10-27 15:23:28 +05:30
generate_component_file(component, :di_packages, architecture, :udeb)
2021-06-08 01:23:25 +05:30
end
2022-01-26 12:08:38 +05:30
generate_component_file(component, :sources, nil, :dsc)
2021-06-08 01:23:25 +05:30
end
end
def generate_component_file(component, component_file_type, architecture, package_file_type)
paragraphs = @distribution.package_files
2021-10-27 15:23:28 +05:30
.preload_package
2021-06-08 01:23:25 +05:30
.preload_debian_file_metadata
.with_debian_component_name(component.name)
2021-10-27 15:23:28 +05:30
.with_debian_architecture_name(architecture&.name)
2021-06-08 01:23:25 +05:30
.with_debian_file_type(package_file_type)
.find_each
.map(&method(:package_stanza_from_fields))
2021-09-04 01:27:46 +05:30
reuse_or_create_component_file(component, component_file_type, architecture, paragraphs.join("\n"))
2021-06-08 01:23:25 +05:30
end
def package_stanza_from_fields(package_file)
[
2021-10-27 15:23:28 +05:30
METADATA_KEYS.map do |metadata_key|
metadata_name = metadata_key
metadata_value = package_file.debian_fields[metadata_key]
if package_file.debian_dsc?
metadata_name = 'Package' if metadata_key == 'Source'
checksum = case metadata_key
when 'Files' then package_file.file_md5
when 'Checksums-Sha256' then package_file.file_sha256
when 'Checksums-Sha1' then package_file.file_sha1
end
if checksum
metadata_value = "\n#{checksum} #{package_file.size} #{package_file.file_name}#{metadata_value}"
end
end
rfc822_field(metadata_name, metadata_value)
2021-06-08 01:23:25 +05:30
end,
rfc822_field('Section', package_file.debian_fields['Section'] || 'misc'),
rfc822_field('Priority', package_file.debian_fields['Priority'] || 'extra'),
2021-10-27 15:23:28 +05:30
package_file_extra_fields(package_file)
2021-06-08 01:23:25 +05:30
].flatten.compact.join('')
end
2021-10-27 15:23:28 +05:30
def package_file_extra_fields(package_file)
if package_file.debian_dsc?
[
rfc822_field('Directory', package_dirname(package_file))
]
else
[
rfc822_field('Filename', "#{package_dirname(package_file)}/#{package_file.file_name}"),
rfc822_field('Size', package_file.size),
rfc822_field('MD5sum', package_file.file_md5),
rfc822_field('SHA256', package_file.file_sha256)
]
end
end
def package_dirname(package_file)
2021-06-08 01:23:25 +05:30
letter = package_file.package.name.start_with?('lib') ? package_file.package.name[0..3] : package_file.package.name[0]
2021-10-27 15:23:28 +05:30
"#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.package.version}"
2021-06-08 01:23:25 +05:30
end
def pool_prefix(package_file)
case @distribution
when ::Packages::Debian::GroupDistribution
"pool/#{@distribution.codename}/#{package_file.package.project_id}"
else
2021-10-27 15:23:28 +05:30
"pool/#{@distribution.codename}"
2021-06-08 01:23:25 +05:30
end
end
2021-09-04 01:27:46 +05:30
def reuse_or_create_component_file(component, component_file_type, architecture, content)
file_md5 = Digest::MD5.hexdigest(content)
file_sha256 = Digest::SHA256.hexdigest(content)
component_file = component.files
.with_file_type(component_file_type)
.with_architecture(architecture)
.with_compression_type(nil)
.with_file_sha256(file_sha256)
.last
if component_file
component_file.touch(time: release_date)
else
component_file = component.files.create!(
updated_at: release_date,
file_type: component_file_type,
architecture: architecture,
compression_type: nil,
file: CarrierWaveStringFile.new(content),
file_md5: file_md5,
file_sha256: file_sha256
)
end
@md5sum.append(" #{file_md5} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
@sha256.append(" #{file_sha256} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
2021-06-08 01:23:25 +05:30
end
def generate_release
2021-10-27 15:23:28 +05:30
@distribution.key || @distribution.create_key(GenerateDistributionKeyService.new.execute)
@distribution.file = CarrierWaveStringFile.new(release_content)
@distribution.file_signature = SignDistributionService.new(@distribution, release_content, detach: true).execute
@distribution.signed_file = CarrierWaveStringFile.new(
SignDistributionService.new(@distribution, release_content).execute
)
2021-06-08 01:23:25 +05:30
@distribution.updated_at = release_date
@distribution.save!
end
2021-10-27 15:23:28 +05:30
def release_content
strong_memoize(:release_content) do
release_header + release_sums
2021-06-08 01:23:25 +05:30
end
end
2021-10-27 15:23:28 +05:30
def release_header
[
%w[origin label suite version codename].map do |attribute|
rfc822_field(attribute.capitalize, @distribution.attributes[attribute])
end,
rfc822_field('Date', release_date.to_formatted_s(:rfc822)),
valid_until_field,
rfc822_field('NotAutomatic', !@distribution.automatic, !@distribution.automatic),
rfc822_field('ButAutomaticUpgrades', @distribution.automatic_upgrades, !@distribution.automatic && @distribution.automatic_upgrades),
2022-10-11 01:57:18 +05:30
rfc822_field('Acquire-By-Hash', 'yes'),
2021-10-27 15:23:28 +05:30
rfc822_field('Architectures', @distribution.architectures.map { |architecture| architecture.name }.sort.join(' ')),
rfc822_field('Components', @distribution.components.map { |component| component.name }.sort.join(' ')),
rfc822_field('Description', @distribution.description)
].flatten.compact.join('')
end
2021-06-08 01:23:25 +05:30
def release_date
strong_memoize(:release_date) do
Time.now.utc
end
end
def release_sums
["MD5Sum:", @md5sum, "SHA256:", @sha256].flatten.compact.join("\n") + "\n"
end
def rfc822_field(name, value, condition = true)
return unless condition
return if value.blank?
2021-10-27 15:23:28 +05:30
value = " #{value}" unless value[0] == "\n"
"#{name}:#{value.to_s.gsub("\n\n", "\n.\n").gsub("\n", "\n ")}\n"
2021-06-08 01:23:25 +05:30
end
def valid_until_field
return unless @distribution.valid_time_duration_seconds
rfc822_field('Valid-Until', release_date.since(@distribution.valid_time_duration_seconds).to_formatted_s(:rfc822))
end
def destroy_old_component_files
2021-09-04 01:27:46 +05:30
return if @oldest_kept_generated_at.nil?
2021-06-08 01:23:25 +05:30
2021-09-04 01:27:46 +05:30
@distribution.component_files.updated_before(@oldest_kept_generated_at).destroy_all # rubocop:disable Cop/DestroyAll
2021-06-08 01:23:25 +05:30
end
# used by ExclusiveLeaseGuard
def lease_key
"packages:debian:generate_distribution_service:distribution:#{@distribution.id}"
end
# used by ExclusiveLeaseGuard
def lease_timeout
DEFAULT_LEASE_TIMEOUT
end
end
end
end