debian-mirror-gitlab/app/models/ci/job_artifact.rb

333 lines
10 KiB
Ruby
Raw Normal View History

2018-11-18 11:00:15 +05:30
# frozen_string_literal: true
2018-03-17 18:26:18 +05:30
module Ci
2019-07-07 11:18:12 +05:30
class JobArtifact < ApplicationRecord
2018-05-09 12:01:36 +05:30
include AfterCommitQueue
include ObjectStorage::BackgroundMove
2019-07-31 22:56:46 +05:30
include UpdateProjectStatistics
2020-06-23 00:09:42 +05:30
include UsageStatistics
2019-12-21 20:55:43 +05:30
include Sortable
2020-10-24 23:57:45 +05:30
include Artifactable
include FileStoreMounter
2021-03-08 18:12:59 +05:30
include EachBatch
2018-03-17 18:26:18 +05:30
extend Gitlab::Ci::Model
2018-11-18 11:00:15 +05:30
TEST_REPORT_FILE_TYPES = %w[junit].freeze
2020-04-08 14:13:33 +05:30
COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze
2021-02-22 17:27:13 +05:30
CODEQUALITY_REPORT_FILE_TYPES = %w[codequality].freeze
2020-05-24 23:13:21 +05:30
ACCESSIBILITY_REPORT_FILE_TYPES = %w[accessibility].freeze
2018-12-05 23:21:45 +05:30
NON_ERASABLE_FILE_TYPES = %w[trace].freeze
2020-05-24 23:13:21 +05:30
TERRAFORM_REPORT_FILE_TYPES = %w[terraform].freeze
UNSUPPORTED_FILE_TYPES = %i[license_management].freeze
2021-03-11 19:13:27 +05:30
SAST_REPORT_TYPES = %w[sast].freeze
SECRET_DETECTION_REPORT_TYPES = %w[secret_detection].freeze
2018-12-05 23:21:45 +05:30
DEFAULT_FILE_NAMES = {
archive: nil,
metadata: nil,
trace: nil,
2020-03-13 15:44:24 +05:30
metrics_referee: nil,
network_referee: nil,
2018-12-05 23:21:45 +05:30
junit: 'junit.xml',
2020-05-24 23:13:21 +05:30
accessibility: 'gl-accessibility.json',
2018-12-13 13:39:08 +05:30
codequality: 'gl-code-quality-report.json',
2018-12-05 23:21:45 +05:30
sast: 'gl-sast-report.json',
2020-06-23 00:09:42 +05:30
secret_detection: 'gl-secret-detection-report.json',
2018-12-05 23:21:45 +05:30
dependency_scanning: 'gl-dependency-scanning-report.json',
container_scanning: 'gl-container-scanning-report.json',
2018-12-13 13:39:08 +05:30
dast: 'gl-dast-report.json',
license_management: 'gl-license-management-report.json',
2020-03-13 15:44:24 +05:30
license_scanning: 'gl-license-scanning-report.json',
2019-07-07 11:18:12 +05:30
performance: 'performance.json',
2020-07-28 23:09:34 +05:30
browser_performance: 'browser-performance.json',
load_performance: 'load-performance.json',
2020-03-13 15:44:24 +05:30
metrics: 'metrics.txt',
2020-04-08 14:13:33 +05:30
lsif: 'lsif.json',
dotenv: '.env',
2020-04-22 19:07:51 +05:30
cobertura: 'cobertura-coverage.xml',
2020-05-24 23:13:21 +05:30
terraform: 'tfplan.json',
2020-06-23 00:09:42 +05:30
cluster_applications: 'gl-cluster-applications.json',
2020-07-28 23:09:34 +05:30
requirements: 'requirements.json',
2021-01-03 14:25:43 +05:30
coverage_fuzzing: 'gl-coverage-fuzzing.json',
api_fuzzing: 'gl-api-fuzzing-report.json'
2018-12-05 23:21:45 +05:30
}.freeze
2019-09-04 21:01:54 +05:30
INTERNAL_TYPES = {
2018-12-05 23:21:45 +05:30
archive: :zip,
metadata: :gzip,
2019-09-04 21:01:54 +05:30
trace: :raw
}.freeze
REPORT_TYPES = {
2018-12-05 23:21:45 +05:30
junit: :gzip,
2019-07-07 11:18:12 +05:30
metrics: :gzip,
2020-03-13 15:44:24 +05:30
metrics_referee: :gzip,
network_referee: :gzip,
2020-04-08 14:13:33 +05:30
dotenv: :gzip,
cobertura: :gzip,
2020-05-24 23:13:21 +05:30
cluster_applications: :gzip,
lsif: :zip,
2018-12-13 13:39:08 +05:30
2021-01-03 14:25:43 +05:30
# Security reports and license scanning reports are raw artifacts
# because they used to be fetched by the frontend, but this is not the case anymore.
2018-12-13 13:39:08 +05:30
sast: :raw,
2020-06-23 00:09:42 +05:30
secret_detection: :raw,
2018-12-13 13:39:08 +05:30
dependency_scanning: :raw,
container_scanning: :raw,
dast: :raw,
license_management: :raw,
2020-03-13 15:44:24 +05:30
license_scanning: :raw,
2021-01-03 14:25:43 +05:30
# All these file formats use `raw` as we need to store them uncompressed
# for Frontend to fetch the files and do analysis
# When they will be only used by backend, they can be `gzipped`.
accessibility: :raw,
codequality: :raw,
2020-04-22 19:07:51 +05:30
performance: :raw,
2020-07-28 23:09:34 +05:30
browser_performance: :raw,
load_performance: :raw,
2020-06-23 00:09:42 +05:30
terraform: :raw,
2020-07-28 23:09:34 +05:30
requirements: :raw,
2021-01-03 14:25:43 +05:30
coverage_fuzzing: :raw,
api_fuzzing: :raw
2018-12-05 23:21:45 +05:30
}.freeze
2018-11-18 11:00:15 +05:30
2020-05-24 23:13:21 +05:30
DOWNLOADABLE_TYPES = %w[
accessibility
2021-01-03 14:25:43 +05:30
api_fuzzing
2020-05-24 23:13:21 +05:30
archive
cobertura
codequality
container_scanning
dast
dependency_scanning
dotenv
junit
license_management
license_scanning
lsif
metrics
performance
2020-07-28 23:09:34 +05:30
browser_performance
load_performance
2020-05-24 23:13:21 +05:30
sast
2020-06-23 00:09:42 +05:30
secret_detection
requirements
2020-05-24 23:13:21 +05:30
].freeze
2019-09-04 21:01:54 +05:30
TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze
2020-07-28 23:09:34 +05:30
PLAN_LIMIT_PREFIX = 'ci_max_artifact_size_'
2020-05-24 23:13:21 +05:30
2018-03-17 18:26:18 +05:30
belongs_to :project
belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
2020-10-24 23:57:45 +05:30
mount_file_store_uploader JobArtifactUploader
2018-05-09 12:01:36 +05:30
2018-11-18 11:00:15 +05:30
validates :file_format, presence: true, unless: :trace?, on: :create
2020-05-24 23:13:21 +05:30
validate :validate_supported_file_format!, on: :create
validate :validate_file_format!, unless: :trace?, on: :create
2018-03-17 18:26:18 +05:30
before_save :set_size, if: :file_changed?
2019-09-04 21:01:54 +05:30
update_project_statistics project_statistics_name: :build_artifacts_size
2019-07-31 22:56:46 +05:30
2020-06-23 00:09:42 +05:30
scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) }
2020-07-28 23:09:34 +05:30
scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) }
2019-12-21 20:55:43 +05:30
scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
2020-04-08 14:13:33 +05:30
scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) }
2020-04-22 19:07:51 +05:30
scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) }
2018-03-17 18:26:18 +05:30
2021-04-17 20:07:23 +05:30
scope :with_job, -> { joins(:job).includes(:job) }
2021-03-08 18:12:59 +05:30
2018-12-05 23:21:45 +05:30
scope :with_file_types, -> (file_types) do
types = self.file_types.select { |file_type| file_types.include?(file_type) }.values
where(file_type: types)
end
2019-09-04 21:01:54 +05:30
scope :with_reports, -> do
with_file_types(REPORT_TYPES.keys.map(&:to_s))
end
2021-03-11 19:13:27 +05:30
scope :sast_reports, -> do
with_file_types(SAST_REPORT_TYPES)
end
scope :secret_detection_reports, -> do
with_file_types(SECRET_DETECTION_REPORT_TYPES)
end
2018-11-18 11:00:15 +05:30
scope :test_reports, -> do
2018-12-05 23:21:45 +05:30
with_file_types(TEST_REPORT_FILE_TYPES)
end
2020-05-24 23:13:21 +05:30
scope :accessibility_reports, -> do
with_file_types(ACCESSIBILITY_REPORT_FILE_TYPES)
end
2020-04-08 14:13:33 +05:30
scope :coverage_reports, -> do
with_file_types(COVERAGE_REPORT_FILE_TYPES)
end
2021-02-22 17:27:13 +05:30
scope :codequality_reports, -> do
with_file_types(CODEQUALITY_REPORT_FILE_TYPES)
end
2020-05-24 23:13:21 +05:30
scope :terraform_reports, -> do
with_file_types(TERRAFORM_REPORT_FILE_TYPES)
end
2018-12-05 23:21:45 +05:30
scope :erasable, -> do
types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values
2018-11-18 11:00:15 +05:30
where(file_type: types)
end
2020-06-23 00:09:42 +05:30
scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) }
2021-03-08 18:12:59 +05:30
scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked) }
scope :order_expired_desc, -> { order(expire_at: :desc) }
2021-01-29 00:20:46 +05:30
scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) }
2019-03-02 22:35:43 +05:30
2019-12-04 20:38:33 +05:30
scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') }
2018-11-20 20:47:30 +05:30
delegate :filename, :exists?, :open, to: :file
2018-03-17 18:26:18 +05:30
enum file_type: {
archive: 1,
metadata: 2,
2018-11-18 11:00:15 +05:30
trace: 3,
2018-12-05 23:21:45 +05:30
junit: 4,
sast: 5, ## EE-specific
dependency_scanning: 6, ## EE-specific
container_scanning: 7, ## EE-specific
dast: 8, ## EE-specific
2018-12-13 13:39:08 +05:30
codequality: 9, ## EE-specific
license_management: 10, ## EE-specific
2020-03-13 15:44:24 +05:30
license_scanning: 101, ## EE-specific till 13.0
2020-07-28 23:09:34 +05:30
performance: 11, ## EE-specific till 13.2
2020-03-13 15:44:24 +05:30
metrics: 12, ## EE-specific
metrics_referee: 13, ## runner referees
network_referee: 14, ## runner referees
2020-04-08 14:13:33 +05:30
lsif: 15, # LSIF data for code navigation
dotenv: 16,
2020-04-22 19:07:51 +05:30
cobertura: 17,
2020-05-24 23:13:21 +05:30
terraform: 18, # Transformed json
accessibility: 19,
2020-06-23 00:09:42 +05:30
cluster_applications: 20,
secret_detection: 21, ## EE-specific
2020-07-28 23:09:34 +05:30
requirements: 22, ## EE-specific
coverage_fuzzing: 23, ## EE-specific
browser_performance: 24, ## EE-specific
2021-01-03 14:25:43 +05:30
load_performance: 25, ## EE-specific
api_fuzzing: 26 ## EE-specific
2018-11-18 11:00:15 +05:30
}
2018-11-20 20:47:30 +05:30
# `file_location` indicates where actual files are stored.
# Ideally, actual files should be stored in the same directory, and use the same
# convention to generate its path. However, sometimes we can't do so due to backward-compatibility.
#
# legacy_path ... The actual file is stored at a path consists of a timestamp
# and raw project/model IDs. Those rows were migrated from
# `ci_builds.artifacts_file` and `ci_builds.artifacts_metadata`
# hashed_path ... The actual file is stored at a path consists of a SHA2 based on the project ID.
# This is the default value.
enum file_location: {
legacy_path: 1,
hashed_path: 2
}
2020-05-24 23:13:21 +05:30
def validate_supported_file_format!
return if Feature.disabled?(:drop_license_management_artifact, project, default_enabled: true)
if UNSUPPORTED_FILE_TYPES.include?(self.file_type&.to_sym)
errors.add(:base, _("File format is no longer supported"))
end
end
def validate_file_format!
2018-11-18 11:00:15 +05:30
unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym
2020-04-08 14:13:33 +05:30
errors.add(:base, _('Invalid file format with specified file type'))
2018-11-18 11:00:15 +05:30
end
end
2020-07-28 23:09:34 +05:30
def self.associated_file_types_for(file_type)
return unless file_types.include?(file_type)
[file_type]
end
2019-12-21 20:55:43 +05:30
def self.total_size
self.sum(:size)
end
2018-03-17 18:26:18 +05:30
def self.artifacts_size_for(project)
self.where(project: project).sum(:size)
end
2018-05-09 12:01:36 +05:30
def local_store?
[nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store)
end
2018-11-20 20:47:30 +05:30
def hashed_path?
return true if trace? # ArchiveLegacyTraces background migration might not have `file_location` column
super || self.file_location.nil?
end
2020-06-23 00:09:42 +05:30
def expired?
expire_at.present? && expire_at < Time.current
end
def expiring?
expire_at.present? && expire_at > Time.current
end
2018-03-17 18:26:18 +05:30
def expire_in
2020-06-23 00:09:42 +05:30
expire_at - Time.current if expire_at
2018-03-17 18:26:18 +05:30
end
def expire_in=(value)
self.expire_at =
if value
2020-10-24 23:57:45 +05:30
::Gitlab::Ci::Build::Artifacts::ExpireInParser.new(value).seconds_from_now
2018-03-17 18:26:18 +05:30
end
end
2018-10-15 14:42:47 +05:30
2019-10-12 21:52:04 +05:30
def self.archived_trace_exists_for?(job_id)
where(job_id: job_id).trace.take&.file&.file&.exists?
end
2020-07-28 23:09:34 +05:30
def self.max_artifact_size(type:, project:)
2020-10-24 23:57:45 +05:30
limit_name = "#{PLAN_LIMIT_PREFIX}#{type}"
max_size = project.actual_limits.limit_for(
limit_name,
alternate_limit: -> { project.closest_setting(:max_artifacts_size) }
)
2020-07-28 23:09:34 +05:30
max_size&.megabytes.to_i
end
2021-04-17 20:07:23 +05:30
def to_deleted_object_attrs(pick_up_at = nil)
2021-01-03 14:25:43 +05:30
{
file_store: file_store,
store_dir: file.store_dir.to_s,
file: file_identifier,
2021-04-17 20:07:23 +05:30
pick_up_at: pick_up_at || expire_at || Time.current
2021-01-03 14:25:43 +05:30
}
end
2018-10-15 14:42:47 +05:30
private
def set_size
self.size = file.size
end
def project_destroyed?
# Use job.project to avoid extra DB query for project
job.project.pending_delete?
end
2018-03-17 18:26:18 +05:30
end
end
2019-12-04 20:38:33 +05:30
Ci::JobArtifact.prepend_if_ee('EE::Ci::JobArtifact')