debian-mirror-gitlab/app/services/packages/debian/generate_distribution_service.rb

# frozen_string_literal: true

module Packages
  module Debian
    class GenerateDistributionService
      include Gitlab::Utils::StrongMemoize
      include ExclusiveLeaseGuard

      ONE_HOUR = 1.hour.freeze

      # used by ExclusiveLeaseGuard
      DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze

      # From https://salsa.debian.org/ftp-team/dak/-/blob/991aaa27a7f7aa773bb9c0cf2d516e383d9cffa0/setup/core-init.d/080_metadatakeys#L9
      METADATA_KEYS = %w(
        Package
        Source
        Binary
        Version
        Essential
        Installed-Size
        Maintainer
        Uploaders
        Original-Maintainer
        Build-Depends
        Build-Depends-Indep
        Build-Conflicts
        Build-Conflicts-Indep
        Architecture
        Standards-Version
        Format
        Files
        Dm-Upload-Allowed
        Vcs-Browse
        Vcs-Hg
        Vcs-Darcs
        Vcs-Svn
        Vcs-Git
        Vcs-Browser
        Vcs-Arch
        Vcs-Bzr
        Vcs-Mtn
        Vcs-Cvs
        Checksums-Sha256
        Checksums-Sha1
        Replaces
        Provides
        Depends
        Pre-Depends
        Recommends
        Suggests
        Enhances
        Conflicts
        Breaks
        Description
        Origin
        Bugs
        Multi-Arch
        Homepage
        Tag
        Package-Type
        Installer-Menu-Item
      ).freeze

      def initialize(distribution)
        @distribution = distribution
        @oldest_kept_generated_at = nil
        @sha256 = []
      end

      def execute
        try_obtain_lease do
          @distribution.transaction do
            # We consider `apt-get update` can take at most one hour
            # We keep all generations younger than one hour
            # and the previous generation
            @oldest_kept_generated_at = @distribution.component_files.updated_before(release_date - ONE_HOUR).maximum(:updated_at)
            generate_component_files
            generate_release
            destroy_old_component_files
          end
        end
      end

      private

      def generate_component_files
        @distribution.components.ordered_by_name.each do |component|
          @distribution.architectures.ordered_by_name.each do |architecture|
            generate_component_file(component, :packages, architecture, :deb)
            generate_component_file(component, :di_packages, architecture, :udeb)
          end
          generate_component_file(component, :sources, nil, :dsc)
        end
      end

      def generate_component_file(component, component_file_type, architecture, package_file_type)
        paragraphs = @distribution.package_files
                                  .preload_package
                                  .preload_debian_file_metadata
                                  .with_debian_component_name(component.name)
                                  .with_debian_architecture_name(architecture&.name)
                                  .with_debian_file_type(package_file_type)
                                  .find_each
                                  .map { |package_file| package_stanza_from_fields(package_file) }
        reuse_or_create_component_file(component, component_file_type, architecture, paragraphs.join("\n"))
      end

      def package_stanza_from_fields(package_file)
        [
          METADATA_KEYS.map do |metadata_key|
            metadata_name = metadata_key
            metadata_value = package_file.debian_fields[metadata_key]

            if package_file.debian_dsc?
              metadata_name = 'Package' if metadata_key == 'Source'
              checksum = case metadata_key
                         when 'Files' then package_file.file_md5
                         when 'Checksums-Sha256' then package_file.file_sha256
                         when 'Checksums-Sha1' then package_file.file_sha1
                         end

              if checksum
                metadata_value = "\n#{checksum} #{package_file.size} #{package_file.file_name}#{metadata_value}"
              end
            end

            rfc822_field(metadata_name, metadata_value)
          end,
          rfc822_field('Section', package_file.debian_fields['Section'] || 'misc'),
          rfc822_field('Priority', package_file.debian_fields['Priority'] || 'extra'),
          package_file_extra_fields(package_file)
        ].flatten.compact.join('')
      end

      def package_file_extra_fields(package_file)
        if package_file.debian_dsc?
          [
            rfc822_field('Directory', package_dirname(package_file))
          ]
        else
          # NB: MD5sum was removed for FIPS compliance
          [
            rfc822_field('Filename', "#{package_dirname(package_file)}/#{package_file.file_name}"),
            rfc822_field('Size', package_file.size),
            rfc822_field('SHA256', package_file.file_sha256)
          ]
        end
      end

      def package_dirname(package_file)
        letter = package_file.package.name.start_with?('lib') ? package_file.package.name[0..3] : package_file.package.name[0]
        "#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.package.version}"
      end

      def pool_prefix(package_file)
        case @distribution
        when ::Packages::Debian::GroupDistribution
          "pool/#{@distribution.codename}/#{package_file.package.project_id}"
        else
          "pool/#{@distribution.codename}"
        end
      end

      def reuse_or_create_component_file(component, component_file_type, architecture, content)
        file_md5 = Digest::MD5.hexdigest(content)
        file_sha256 = Digest::SHA256.hexdigest(content)
        component_file = component.files
                                  .with_file_type(component_file_type)
                                  .with_architecture(architecture)
                                  .with_compression_type(nil)
                                  .with_file_sha256(file_sha256)
                                  .last

        if component_file
          component_file.touch(time: release_date)
        else
          component_file = component.files.create!(
            updated_at: release_date,
            file_type: component_file_type,
            architecture: architecture,
            compression_type: nil,
            file: CarrierWaveStringFile.new(content),
            file_md5: file_md5,
            file_sha256: file_sha256
          )
        end

        @sha256.append(" #{file_sha256} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
      end

      def generate_release
        @distribution.key || @distribution.create_key(GenerateDistributionKeyService.new.execute)
        @distribution.file = CarrierWaveStringFile.new(release_content)
        @distribution.file_signature = SignDistributionService.new(@distribution, release_content, detach: true).execute
        @distribution.signed_file = CarrierWaveStringFile.new(
          SignDistributionService.new(@distribution, release_content).execute
        )
        @distribution.updated_at = release_date
        @distribution.save!
      end

      def release_content
        strong_memoize(:release_content) do
          release_header + release_sums
        end
      end

      def release_header
        [
          %w[origin label suite version codename].map do |attribute|
            rfc822_field(attribute.capitalize, @distribution.attributes[attribute])
          end,
          rfc822_field('Date', release_date.to_formatted_s(:rfc822)),
          valid_until_field,
          rfc822_field('NotAutomatic', !@distribution.automatic, !@distribution.automatic),
          rfc822_field('ButAutomaticUpgrades', @distribution.automatic_upgrades, !@distribution.automatic && @distribution.automatic_upgrades),
          rfc822_field('Acquire-By-Hash', 'yes'),
          rfc822_field('Architectures', @distribution.architectures.map { |architecture| architecture.name }.sort.join(' ')),
          rfc822_field('Components', @distribution.components.map { |component| component.name }.sort.join(' ')),
          rfc822_field('Description', @distribution.description)
        ].flatten.compact.join('')
      end

      def release_date
        strong_memoize(:release_date) do
          Time.now.utc
        end
      end

      def release_sums
        # NB: MD5Sum was removed for FIPS compliance
        ["SHA256:", @sha256].flatten.compact.join("\n") + "\n"
      end

      def rfc822_field(name, value, condition = true)
        return unless condition
        return if value.blank?

        value = " #{value}" unless value[0] == "\n"
        "#{name}:#{value.to_s.gsub("\n\n", "\n.\n").gsub("\n", "\n ")}\n"
      end

      def valid_until_field
        return unless @distribution.valid_time_duration_seconds

        rfc822_field('Valid-Until', release_date.since(@distribution.valid_time_duration_seconds).to_formatted_s(:rfc822))
      end

      def destroy_old_component_files
        return if @oldest_kept_generated_at.nil?

        @distribution.component_files.updated_before(@oldest_kept_generated_at).destroy_all # rubocop:disable Cop/DestroyAll
      end

      # used by ExclusiveLeaseGuard
      def lease_key
        "packages:debian:generate_distribution_service:distribution:#{@distribution.id}"
      end

      # used by ExclusiveLeaseGuard
      def lease_timeout
        DEFAULT_LEASE_TIMEOUT
      end
    end
  end
end