580 lines
18 KiB
Ruby
580 lines
18 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'spec_helper'
|
|
|
|
RSpec.describe ObjectStorage::DirectUpload do
|
|
let(:region) { 'us-east-1' }
|
|
let(:path_style) { false }
|
|
let(:use_iam_profile) { false }
|
|
let(:consolidated_settings) { false }
|
|
let(:credentials) do
|
|
{
|
|
provider: 'AWS',
|
|
aws_access_key_id: 'AWS_ACCESS_KEY_ID',
|
|
aws_secret_access_key: 'AWS_SECRET_ACCESS_KEY',
|
|
region: region,
|
|
path_style: path_style,
|
|
use_iam_profile: use_iam_profile
|
|
}
|
|
end
|
|
|
|
let(:storage_options) { {} }
|
|
let(:raw_config) do
|
|
{
|
|
enabled: true,
|
|
connection: credentials,
|
|
remote_directory: bucket_name,
|
|
storage_options: storage_options,
|
|
consolidated_settings: consolidated_settings
|
|
}
|
|
end
|
|
|
|
let(:config) { ObjectStorage::Config.new(raw_config) }
|
|
let(:storage_url) { 'https://uploads.s3.amazonaws.com/' }
|
|
|
|
let(:bucket_name) { 'uploads' }
|
|
let(:object_name) { 'tmp/uploads/my-file' }
|
|
let(:maximum_size) { 1.gigabyte }
|
|
|
|
let(:direct_upload) { described_class.new(config, object_name, has_length: has_length, maximum_size: maximum_size) }
|
|
|
|
before do
|
|
Fog.unmock!
|
|
end
|
|
|
|
describe '#has_length' do
|
|
context 'is known' do
|
|
let(:has_length) { true }
|
|
let(:maximum_size) { nil }
|
|
|
|
it "maximum size is not required" do
|
|
expect { direct_upload }.not_to raise_error
|
|
end
|
|
end
|
|
|
|
context 'is unknown' do
|
|
let(:has_length) { false }
|
|
|
|
context 'and maximum size is specified' do
|
|
let(:maximum_size) { 1.gigabyte }
|
|
|
|
it "does not raise an error" do
|
|
expect { direct_upload }.not_to raise_error
|
|
end
|
|
end
|
|
|
|
context 'and maximum size is not specified' do
|
|
let(:maximum_size) { nil }
|
|
|
|
it "raises an error" do
|
|
expect { direct_upload }.to raise_error /maximum_size has to be specified if length is unknown/
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
describe '#get_url' do
|
|
subject { described_class.new(config, object_name, has_length: true) }
|
|
|
|
context 'when AWS is used' do
|
|
it 'calls the proper method' do
|
|
expect_next_instance_of(::Fog::Storage, credentials) do |connection|
|
|
expect(connection).to receive(:get_object_url).once
|
|
end
|
|
|
|
subject.get_url
|
|
end
|
|
end
|
|
|
|
context 'when Google is used' do
|
|
let(:credentials) do
|
|
{
|
|
provider: 'Google',
|
|
google_storage_access_key_id: 'GOOGLE_ACCESS_KEY_ID',
|
|
google_storage_secret_access_key: 'GOOGLE_SECRET_ACCESS_KEY'
|
|
}
|
|
end
|
|
|
|
it 'calls the proper method' do
|
|
expect_next_instance_of(::Fog::Storage, credentials) do |connection|
|
|
expect(connection).to receive(:get_object_https_url).once
|
|
end
|
|
|
|
subject.get_url
|
|
end
|
|
end
|
|
end
|
|
|
|
describe '#to_hash', :aggregate_failures do
|
|
subject { direct_upload.to_hash }
|
|
|
|
shared_examples 'a valid S3 upload' do
|
|
it_behaves_like 'a valid upload'
|
|
|
|
it 'sets Workhorse client data' do
|
|
expect(subject[:UseWorkhorseClient]).to eq(use_iam_profile)
|
|
expect(subject[:RemoteTempObjectID]).to eq(object_name)
|
|
|
|
object_store_config = subject[:ObjectStorage]
|
|
expect(object_store_config[:Provider]).to eq 'AWS'
|
|
|
|
s3_config = object_store_config[:S3Config]
|
|
expect(s3_config[:Bucket]).to eq(bucket_name)
|
|
expect(s3_config[:Region]).to eq(region)
|
|
expect(s3_config[:PathStyle]).to eq(path_style)
|
|
expect(s3_config[:UseIamProfile]).to eq(use_iam_profile)
|
|
expect(s3_config.keys).not_to include(%i(ServerSideEncryption SSEKMSKeyID))
|
|
end
|
|
|
|
context 'when no region is specified' do
|
|
before do
|
|
raw_config.delete(:region)
|
|
end
|
|
|
|
it 'defaults to us-east-1' do
|
|
expect(subject[:ObjectStorage][:S3Config][:Region]).to eq('us-east-1')
|
|
end
|
|
end
|
|
|
|
context 'when V2 signatures are used' do
|
|
before do
|
|
credentials[:aws_signature_version] = 2
|
|
end
|
|
|
|
it 'does not enable Workhorse client' do
|
|
expect(subject[:UseWorkhorseClient]).to be false
|
|
end
|
|
end
|
|
|
|
context 'when V4 signatures are used' do
|
|
before do
|
|
credentials[:aws_signature_version] = 4
|
|
end
|
|
|
|
it 'enables the Workhorse client for instance profiles' do
|
|
expect(subject[:UseWorkhorseClient]).to eq(use_iam_profile)
|
|
end
|
|
end
|
|
|
|
context 'when consolidated settings are used' do
|
|
let(:consolidated_settings) { true }
|
|
|
|
it 'enables the Workhorse client' do
|
|
expect(subject[:UseWorkhorseClient]).to be true
|
|
end
|
|
end
|
|
|
|
context 'when only server side encryption is used' do
|
|
let(:storage_options) { { server_side_encryption: 'AES256' } }
|
|
|
|
it 'sends server side encryption settings' do
|
|
s3_config = subject[:ObjectStorage][:S3Config]
|
|
|
|
expect(s3_config[:ServerSideEncryption]).to eq('AES256')
|
|
expect(s3_config.keys).not_to include(:SSEKMSKeyID)
|
|
end
|
|
end
|
|
|
|
context 'when SSE-KMS is used' do
|
|
let(:storage_options) do
|
|
{
|
|
server_side_encryption: 'AES256',
|
|
server_side_encryption_kms_key_id: 'arn:aws:12345'
|
|
}
|
|
end
|
|
|
|
it 'sends server side encryption settings' do
|
|
s3_config = subject[:ObjectStorage][:S3Config]
|
|
|
|
expect(s3_config[:ServerSideEncryption]).to eq('AES256')
|
|
expect(s3_config[:SSEKMSKeyID]).to eq('arn:aws:12345')
|
|
end
|
|
end
|
|
end
|
|
|
|
shared_examples 'a valid Google upload' do |use_workhorse_client: true|
|
|
let(:gocloud_url) { "gs://#{bucket_name}" }
|
|
|
|
it_behaves_like 'a valid upload'
|
|
|
|
if use_workhorse_client
|
|
it 'enables the Workhorse client' do
|
|
expect(subject[:UseWorkhorseClient]).to be true
|
|
expect(subject[:RemoteTempObjectID]).to eq(object_name)
|
|
expect(subject[:ObjectStorage][:Provider]).to eq('Google')
|
|
expect(subject[:ObjectStorage][:GoCloudConfig]).to eq({ URL: gocloud_url })
|
|
end
|
|
end
|
|
|
|
context 'with workhorse_google_client disabled' do
|
|
before do
|
|
stub_feature_flags(workhorse_google_client: false)
|
|
end
|
|
|
|
it 'does not set Workhorse client data' do
|
|
expect(subject.keys).not_to include(:UseWorkhorseClient, :RemoteTempObjectID, :ObjectStorage)
|
|
end
|
|
end
|
|
end
|
|
|
|
shared_examples 'a valid AzureRM upload' do
|
|
it_behaves_like 'a valid upload'
|
|
|
|
it 'enables the Workhorse client' do
|
|
expect(subject[:UseWorkhorseClient]).to be true
|
|
expect(subject[:RemoteTempObjectID]).to eq(object_name)
|
|
expect(subject[:ObjectStorage][:Provider]).to eq('AzureRM')
|
|
expect(subject[:ObjectStorage][:GoCloudConfig]).to eq({ URL: gocloud_url })
|
|
end
|
|
end
|
|
|
|
shared_examples 'a valid upload' do
|
|
it "returns valid structure" do
|
|
expect(subject).to have_key(:Timeout)
|
|
expect(subject[:GetURL]).to start_with(storage_url)
|
|
expect(subject[:StoreURL]).to start_with(storage_url)
|
|
expect(subject[:DeleteURL]).to start_with(storage_url)
|
|
expect(subject[:CustomPutHeaders]).to be_truthy
|
|
expect(subject[:PutHeaders]).to eq({})
|
|
end
|
|
|
|
context 'with an object with UTF-8 characters' do
|
|
let(:object_name) { 'tmp/uploads/テスト' }
|
|
|
|
it 'returns an escaped path' do
|
|
expect(subject[:GetURL]).to start_with(storage_url)
|
|
|
|
uri = Addressable::URI.parse(subject[:GetURL])
|
|
expect(uri.path).to include("tmp/uploads/#{CGI.escape("テスト")}")
|
|
end
|
|
end
|
|
end
|
|
|
|
shared_examples 'a valid upload with multipart data' do
|
|
before do
|
|
stub_object_storage_multipart_init(storage_url, "myUpload")
|
|
end
|
|
|
|
it_behaves_like 'a valid upload'
|
|
|
|
it "returns valid structure" do
|
|
expect(subject).to have_key(:MultipartUpload)
|
|
expect(subject[:MultipartUpload]).to have_key(:PartSize)
|
|
expect(subject[:MultipartUpload][:PartURLs]).to all(start_with(storage_url))
|
|
expect(subject[:MultipartUpload][:PartURLs]).to all(include('uploadId=myUpload'))
|
|
expect(subject[:MultipartUpload][:CompleteURL]).to start_with(storage_url)
|
|
expect(subject[:MultipartUpload][:CompleteURL]).to include('uploadId=myUpload')
|
|
expect(subject[:MultipartUpload][:AbortURL]).to start_with(storage_url)
|
|
expect(subject[:MultipartUpload][:AbortURL]).to include('uploadId=myUpload')
|
|
end
|
|
|
|
it 'uses only strings in query parameters' do
|
|
expect(direct_upload.send(:connection)).to receive(:signed_url).at_least(:once) do |params|
|
|
if params[:query]
|
|
expect(params[:query].keys.all? { |key| key.is_a?(String) }).to be_truthy
|
|
end
|
|
end
|
|
|
|
subject
|
|
end
|
|
end
|
|
|
|
shared_examples 'a valid S3 upload without multipart data' do
|
|
it_behaves_like 'a valid S3 upload'
|
|
it_behaves_like 'a valid upload without multipart data'
|
|
end
|
|
|
|
shared_examples 'a valid S3 upload with multipart data' do
|
|
it_behaves_like 'a valid S3 upload'
|
|
it_behaves_like 'a valid upload with multipart data'
|
|
end
|
|
|
|
shared_examples 'a valid upload without multipart data' do
|
|
it_behaves_like 'a valid upload'
|
|
|
|
it "returns valid structure" do
|
|
expect(subject).not_to have_key(:MultipartUpload)
|
|
end
|
|
end
|
|
|
|
context 'when AWS is used' do
|
|
context 'when length is known' do
|
|
let(:has_length) { true }
|
|
|
|
it_behaves_like 'a valid S3 upload without multipart data'
|
|
|
|
context 'when path style is true' do
|
|
let(:path_style) { true }
|
|
let(:storage_url) { 'https://s3.amazonaws.com/uploads' }
|
|
|
|
before do
|
|
stub_object_storage_multipart_init(storage_url, "myUpload")
|
|
end
|
|
|
|
it_behaves_like 'a valid S3 upload without multipart data'
|
|
end
|
|
|
|
context 'when IAM profile is true' do
|
|
let(:use_iam_profile) { true }
|
|
let(:iam_credentials_v2_url) { "http://169.254.169.254/latest/api/token" }
|
|
let(:iam_credentials_url) { "http://169.254.169.254/latest/meta-data/iam/security-credentials/" }
|
|
let(:iam_credentials) do
|
|
{
|
|
'AccessKeyId' => 'dummykey',
|
|
'SecretAccessKey' => 'dummysecret',
|
|
'Token' => 'dummytoken',
|
|
'Expiration' => 1.day.from_now.xmlschema
|
|
}
|
|
end
|
|
|
|
before do
|
|
# If IMDSv2 is disabled, we should still fall back to IMDSv1
|
|
stub_request(:put, iam_credentials_v2_url)
|
|
.to_return(status: 404)
|
|
stub_request(:get, iam_credentials_url)
|
|
.to_return(status: 200, body: "somerole", headers: {})
|
|
stub_request(:get, "#{iam_credentials_url}somerole")
|
|
.to_return(status: 200, body: iam_credentials.to_json, headers: {})
|
|
end
|
|
|
|
it_behaves_like 'a valid S3 upload without multipart data'
|
|
|
|
context 'when IMSDv2 is available' do
|
|
let(:iam_token) { 'mytoken' }
|
|
|
|
before do
|
|
stub_request(:put, iam_credentials_v2_url)
|
|
.to_return(status: 200, body: iam_token)
|
|
stub_request(:get, iam_credentials_url).with(headers: { "X-aws-ec2-metadata-token" => iam_token })
|
|
.to_return(status: 200, body: "somerole", headers: {})
|
|
stub_request(:get, "#{iam_credentials_url}somerole").with(headers: { "X-aws-ec2-metadata-token" => iam_token })
|
|
.to_return(status: 200, body: iam_credentials.to_json, headers: {})
|
|
end
|
|
|
|
it_behaves_like 'a valid S3 upload without multipart data'
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when length is unknown' do
|
|
let(:has_length) { false }
|
|
|
|
it_behaves_like 'a valid S3 upload with multipart data' do
|
|
before do
|
|
stub_object_storage_multipart_init(storage_url, "myUpload")
|
|
end
|
|
|
|
context 'when maximum upload size is 0' do
|
|
let(:maximum_size) { 0 }
|
|
|
|
it 'returns maximum number of parts' do
|
|
expect(subject[:MultipartUpload][:PartURLs].length).to eq(100)
|
|
end
|
|
|
|
it 'part size is minimum, 5MB' do
|
|
expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte)
|
|
end
|
|
end
|
|
|
|
context 'when maximum upload size is < 5 MB' do
|
|
let(:maximum_size) { 1024 }
|
|
|
|
it 'returns only 1 part' do
|
|
expect(subject[:MultipartUpload][:PartURLs].length).to eq(1)
|
|
end
|
|
|
|
it 'part size is minimum, 5MB' do
|
|
expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte)
|
|
end
|
|
end
|
|
|
|
context 'when maximum upload size is 10MB' do
|
|
let(:maximum_size) { 10.megabyte }
|
|
|
|
it 'returns only 2 parts' do
|
|
expect(subject[:MultipartUpload][:PartURLs].length).to eq(2)
|
|
end
|
|
|
|
it 'part size is minimum, 5MB' do
|
|
expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte)
|
|
end
|
|
end
|
|
|
|
context 'when maximum upload size is 12MB' do
|
|
let(:maximum_size) { 12.megabyte }
|
|
|
|
it 'returns only 3 parts' do
|
|
expect(subject[:MultipartUpload][:PartURLs].length).to eq(3)
|
|
end
|
|
|
|
it 'part size is rounded-up to 5MB' do
|
|
expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte)
|
|
end
|
|
end
|
|
|
|
context 'when maximum upload size is 49GB' do
|
|
let(:maximum_size) { 49.gigabyte }
|
|
|
|
it 'returns maximum, 100 parts' do
|
|
expect(subject[:MultipartUpload][:PartURLs].length).to eq(100)
|
|
end
|
|
|
|
it 'part size is rounded-up to 5MB' do
|
|
expect(subject[:MultipartUpload][:PartSize]).to eq(505.megabyte)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when Google is used' do
|
|
let(:consolidated_settings) { true }
|
|
|
|
# We need to use fog mocks as using google_application_default
|
|
# will trigger network requests which we don't want in this spec.
|
|
# In turn, using fog mocks will don't use a specific storage endpoint,
|
|
# hence the storage_url with the empty host.
|
|
let(:storage_url) { 'https:///uploads/' }
|
|
|
|
before do
|
|
Fog.mock!
|
|
end
|
|
|
|
context 'with google_application_default' do
|
|
let(:credentials) do
|
|
{
|
|
provider: 'Google',
|
|
google_project: 'GOOGLE_PROJECT',
|
|
google_application_default: true
|
|
}
|
|
end
|
|
|
|
context 'when length is known' do
|
|
let(:has_length) { true }
|
|
|
|
it_behaves_like 'a valid Google upload'
|
|
it_behaves_like 'a valid upload without multipart data'
|
|
end
|
|
|
|
context 'when length is unknown' do
|
|
let(:has_length) { false }
|
|
|
|
it_behaves_like 'a valid Google upload'
|
|
it_behaves_like 'a valid upload without multipart data'
|
|
end
|
|
end
|
|
|
|
context 'with google_json_key_location' do
|
|
let(:credentials) do
|
|
{
|
|
provider: 'Google',
|
|
google_project: 'GOOGLE_PROJECT',
|
|
google_json_key_location: 'LOCATION'
|
|
}
|
|
end
|
|
|
|
context 'when length is known' do
|
|
let(:has_length) { true }
|
|
|
|
it_behaves_like 'a valid Google upload', use_workhorse_client: true
|
|
it_behaves_like 'a valid upload without multipart data'
|
|
end
|
|
|
|
context 'when length is unknown' do
|
|
let(:has_length) { false }
|
|
|
|
it_behaves_like 'a valid Google upload', use_workhorse_client: true
|
|
it_behaves_like 'a valid upload without multipart data'
|
|
end
|
|
end
|
|
|
|
context 'with google_json_key_string' do
|
|
let(:credentials) do
|
|
{
|
|
provider: 'Google',
|
|
google_project: 'GOOGLE_PROJECT',
|
|
google_json_key_string: 'STRING'
|
|
}
|
|
end
|
|
|
|
context 'when length is known' do
|
|
let(:has_length) { true }
|
|
|
|
it_behaves_like 'a valid Google upload', use_workhorse_client: true
|
|
it_behaves_like 'a valid upload without multipart data'
|
|
end
|
|
|
|
context 'when length is unknown' do
|
|
let(:has_length) { false }
|
|
|
|
it_behaves_like 'a valid Google upload', use_workhorse_client: true
|
|
it_behaves_like 'a valid upload without multipart data'
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when AzureRM is used' do
|
|
let(:credentials) do
|
|
{
|
|
provider: 'AzureRM',
|
|
azure_storage_account_name: 'azuretest',
|
|
azure_storage_access_key: 'ABCD1234'
|
|
}
|
|
end
|
|
|
|
let(:has_length) { false }
|
|
let(:storage_domain) { nil }
|
|
let(:storage_url) { 'https://azuretest.blob.core.windows.net' }
|
|
let(:gocloud_url) { "azblob://#{bucket_name}" }
|
|
|
|
it_behaves_like 'a valid AzureRM upload'
|
|
it_behaves_like 'a valid upload without multipart data'
|
|
|
|
context 'when a custom storage domain is used' do
|
|
let(:storage_domain) { 'blob.core.chinacloudapi.cn' }
|
|
let(:storage_url) { "https://azuretest.#{storage_domain}" }
|
|
let(:gocloud_url) { "azblob://#{bucket_name}?domain=#{storage_domain}" }
|
|
|
|
before do
|
|
credentials[:azure_storage_domain] = storage_domain
|
|
end
|
|
|
|
it_behaves_like 'a valid AzureRM upload'
|
|
end
|
|
end
|
|
end
|
|
|
|
describe '#use_workhorse_google_client?' do
|
|
let(:direct_upload) { described_class.new(config, object_name, has_length: true) }
|
|
|
|
subject { direct_upload.use_workhorse_google_client? }
|
|
|
|
context 'with consolidated_settings' do
|
|
let(:consolidated_settings) { true }
|
|
|
|
[
|
|
{ google_application_default: true },
|
|
{ google_json_key_string: 'TEST' },
|
|
{ google_json_key_location: 'PATH' }
|
|
].each do |google_config|
|
|
context "with #{google_config.each_key.first}" do
|
|
let(:credentials) { google_config }
|
|
|
|
it { is_expected.to be_truthy }
|
|
end
|
|
end
|
|
|
|
context 'without any google setting' do
|
|
let(:credentials) { {} }
|
|
|
|
it { is_expected.to be_falsey }
|
|
end
|
|
end
|
|
|
|
context 'without consolidated_settings' do
|
|
let(:consolidated_settings) { true }
|
|
|
|
it { is_expected.to be_falsey }
|
|
end
|
|
end
|
|
end
|