2019-10-12 21:52:04 +05:30
# frozen_string_literal: true
2017-08-17 22:00:37 +05:30
require " spec_helper "
2020-07-28 23:09:34 +05:30
RSpec . describe Gitlab :: EncodingHelper do
2021-10-27 15:23:28 +05:30
using RSpec :: Parameterized :: TableSyntax
2017-09-10 17:25:29 +05:30
let ( :ext_class ) { Class . new { extend Gitlab :: EncodingHelper } }
let ( :binary_string ) { File . read ( Rails . root + " spec/fixtures/dk.png " ) }
2017-08-17 22:00:37 +05:30
describe '#encode!' do
[
2018-03-17 18:26:18 +05:30
[ " nil " , nil , nil ] ,
[ " empty string " , " " . encode ( " ASCII-8BIT " ) , " " . encode ( " UTF-8 " ) ] ,
2019-10-12 21:52:04 +05:30
[ " invalid utf-8 encoded string " , ( + " my bad string \xE5 " ) . force_encoding ( " UTF-8 " ) , " my bad string " ] ,
[ " frozen non-ascii string " , ( + " é " ) . force_encoding ( " ASCII-8BIT " ) . freeze , " é " . encode ( " UTF-8 " ) ] ,
2017-08-17 22:00:37 +05:30
[
'leaves ascii only string as is' ,
'ascii only string' ,
'ascii only string'
] ,
[
'leaves valid utf8 string as is' ,
'multibyte string №∑∉' ,
'multibyte string №∑∉'
] ,
[
'removes invalid bytes from ASCII-8bit encoded multibyte string. This can occur when a git diff match line truncates in the middle of a multibyte character. This occurs after the second word in this example. The test string is as short as we can get while still triggering the error condition when not looking at `detect[:confidence]`.' ,
2019-10-12 21:52:04 +05:30
( + " mu ns \xC3 \n Lorem ipsum dolor sit amet, consectetur adipisicing ut \xC3 \xA0 y \xC3 \xB9 abcd \xC3 \xB9 efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p \n {: .normal_pn} \n \n -Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in \n # *Lorem ipsum \xC3 \xB9 l \xC3 \xB9 l \xC3 \xA0 dolor \xC3 \xB9 k \xC3 \xB9 sit \xC3 \xA8 b \xC3 \xA8 N \xC3 \xA8 amet b \xC3 \xA0 d \xC3 \xAC * \n + # *consectetur \xC3 \xB9 l \xC3 \xB9 l \xC3 \xA0 adipisicing \xC3 \xB9 k \xC3 \xB9 elit \xC3 \xA8 b \xC3 \xA8 N \xC3 \xA8 sed do \xC3 \xA0 d \xC3 \xAC *{: .italic .smcaps} \n \n \xEF \x9B \xA1 eiusmod tempor incididunt, ut \xC3 \xAA n \xC3 \xB9 labore et dolore. Tw \xC4 \x83 nj \xC3 \xAC magna aliqua. Ut enim ad minim veniam \n {: .normal} \n @@ -9,5 +9,5 @@ quis nostrud \xC3 \xAA t \xC3 \xB9 exercitiation ullamco laboris m \xC3 \xB9 s \xC3 \xB9 k \xC3 \xB9 abc \xC3 \xB9 nisi " ) . force_encoding ( 'ASCII-8BIT' ) ,
2017-09-10 17:25:29 +05:30
" mu ns \n Lorem ipsum dolor sit amet, consectetur adipisicing ut \xC3 \xA0 y \xC3 \xB9 abcd \xC3 \xB9 efg kia elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non p \n {: .normal_pn} \n \n -Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in \n # *Lorem ipsum \xC3 \xB9 l \xC3 \xB9 l \xC3 \xA0 dolor \xC3 \xB9 k \xC3 \xB9 sit \xC3 \xA8 b \xC3 \xA8 N \xC3 \xA8 amet b \xC3 \xA0 d \xC3 \xAC * \n + # *consectetur \xC3 \xB9 l \xC3 \xB9 l \xC3 \xA0 adipisicing \xC3 \xB9 k \xC3 \xB9 elit \xC3 \xA8 b \xC3 \xA8 N \xC3 \xA8 sed do \xC3 \xA0 d \xC3 \xAC *{: .italic .smcaps} \n \n \xEF \x9B \xA1 eiusmod tempor incididunt, ut \xC3 \xAA n \xC3 \xB9 labore et dolore. Tw \xC4 \x83 nj \xC3 \xAC magna aliqua. Ut enim ad minim veniam \n {: .normal} \n @@ -9,5 +9,5 @@ quis nostrud \xC3 \xAA t \xC3 \xB9 exercitiation ullamco laboris m \xC3 \xB9 s \xC3 \xB9 k \xC3 \xB9 abc \xC3 \xB9 nisi "
2018-03-27 19:54:05 +05:30
] ,
[
'string with detected encoding that is not supported in Ruby' ,
" \xFF e,i \xFF , \xB8 oi,' \xB8 , \xFF ,- " ,
" --broken encoding: IBM420_ltr "
2017-09-10 17:25:29 +05:30
]
2017-08-17 22:00:37 +05:30
] . each do | description , test_string , xpect |
it description do
expect ( ext_class . encode! ( test_string ) ) . to eq ( xpect )
end
end
it 'leaves binary string as is' do
expect ( ext_class . encode! ( binary_string ) ) . to eq ( binary_string )
end
2017-09-10 17:25:29 +05:30
context 'with corrupted diff' do
let ( :corrupted_diff ) do
with_empty_bare_repository do | repo |
content = File . read ( Rails . root . join (
'spec/fixtures/encoding/Japanese.md' ) . to_s )
commit_a = commit ( repo , 'Japanese.md' , content )
commit_b = commit ( repo , 'Japanese.md' ,
content . sub ( '[TODO: Link]' , '[現在作業中です: Link]' ) )
repo . diff ( commit_a , commit_b ) . each_line . map ( & :content ) . join
end
end
let ( :cleaned_diff ) do
corrupted_diff . dup . force_encoding ( 'UTF-8' )
. encode! ( 'UTF-8' , invalid : :replace , replace : '' )
end
let ( :encoded_diff ) do
described_class . encode! ( corrupted_diff . dup )
end
it 'does not corrupt data but remove invalid characters' do
expect ( encoded_diff ) . to eq ( cleaned_diff )
end
def commit ( repo , path , content )
oid = repo . write ( content , :blob )
index = repo . index
index . read_tree ( repo . head . target . tree ) unless repo . empty?
index . add ( path : path , oid : oid , mode : 0100644 )
user = { name : 'Test' , email : 'test@example.com' }
Rugged :: Commit . create (
repo ,
tree : index . write_tree ( repo ) ,
author : user ,
committer : user ,
message : " Update #{ path } " ,
parents : repo . empty? ? [ ] : [ repo . head . target ] . compact ,
update_ref : 'HEAD'
)
end
end
2017-08-17 22:00:37 +05:30
end
2021-10-27 15:23:28 +05:30
describe '#encode_utf8_no_detect' do
where ( :input , :expected ) do
" abcd " | " abcd "
" DzDzDz " | " DzDzDz "
" \xC7 \xB2 \xC7 DzDzDz " | " Dz<EFBFBD> DzDzDz "
" 🐤🐤🐤🐤 \xF0 \x9F \x90 " | " 🐤🐤🐤🐤<EFBFBD> "
end
with_them do
it 'drops invalid UTF-8' do
expect ( ext_class . encode_utf8_no_detect ( input . dup . force_encoding ( Encoding :: ASCII_8BIT ) ) ) . to eq ( expected )
expect ( ext_class . encode_utf8_no_detect ( input ) ) . to eq ( expected )
end
end
end
2017-08-17 22:00:37 +05:30
describe '#encode_utf8' do
[
2018-03-17 18:26:18 +05:30
[ " nil " , nil , nil ] ,
[ " empty string " , " " . encode ( " ASCII-8BIT " ) , " " . encode ( " UTF-8 " ) ] ,
2019-10-12 21:52:04 +05:30
[ " invalid utf-8 encoded string " , ( + " my bad string \xE5 " ) . force_encoding ( " UTF-8 " ) , " my bad stringå " ] ,
2017-08-17 22:00:37 +05:30
[
" encodes valid utf8 encoded string to utf8 " ,
" λ, λ, λ " . encode ( " UTF-8 " ) ,
2017-09-10 17:25:29 +05:30
" λ, λ, λ " . encode ( " UTF-8 " )
2017-08-17 22:00:37 +05:30
] ,
[
" encodes valid ASCII-8BIT encoded string to utf8 " ,
" ascii only " . encode ( " ASCII-8BIT " ) ,
2017-09-10 17:25:29 +05:30
" ascii only " . encode ( " UTF-8 " )
2017-08-17 22:00:37 +05:30
] ,
[
" encodes valid ISO-8859-1 encoded string to utf8 " ,
" Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann. " . encode ( " ISO-8859-1 " , " UTF-8 " ) ,
2017-09-10 17:25:29 +05:30
" Rüby ist eine Programmiersprache. Wir verlängern den text damit ICU die Sprache erkennen kann. " . encode ( " UTF-8 " )
2018-03-17 18:26:18 +05:30
] ,
[
2019-12-04 20:38:33 +05:30
# Test case from https://gitlab.com/gitlab-org/gitlab-foss/issues/39227
2018-03-17 18:26:18 +05:30
" Equifax branch name " ,
" refs/heads/Equifax " . encode ( " UTF-8 " ) ,
" refs/heads/Equifax " . encode ( " UTF-8 " )
2017-09-10 17:25:29 +05:30
]
2017-08-17 22:00:37 +05:30
] . each do | description , test_string , xpect |
it description do
2018-03-17 18:26:18 +05:30
r = ext_class . encode_utf8 ( test_string )
2017-08-17 22:00:37 +05:30
expect ( r ) . to eq ( xpect )
2018-03-17 18:26:18 +05:30
expect ( r . encoding . name ) . to eq ( 'UTF-8' ) if xpect
2017-08-17 22:00:37 +05:30
end
end
it 'returns empty string on conversion errors' do
2019-02-15 15:39:39 +05:30
expect { ext_class . encode_utf8 ( '' ) } . not_to raise_error
2017-08-17 22:00:37 +05:30
end
2018-03-17 18:26:18 +05:30
2020-04-08 14:13:33 +05:30
it 'replaces invalid and undefined chars with the replace argument' do
str = 'hællo' . encode ( Encoding :: UTF_16LE ) . force_encoding ( Encoding :: ASCII_8BIT )
expect ( ext_class . encode_utf8 ( str , replace : " \ u{FFFD} " ) ) . to eq ( " h<EFBFBD> llo " )
end
2018-03-17 18:26:18 +05:30
context 'with strings that can be forcefully encoded into utf8' do
let ( :test_string ) do
" refs/heads/FixSymbolsTitleDropdown " . encode ( " ASCII-8BIT " )
end
2020-10-24 23:57:45 +05:30
2018-03-17 18:26:18 +05:30
let ( :expected_string ) do
" refs/heads/FixSymbolsTitleDropdown " . encode ( " UTF-8 " )
end
subject { ext_class . encode_utf8 ( test_string ) }
it " doesn't use CharlockHolmes if the encoding can be forced into utf_8 " do
expect ( CharlockHolmes :: EncodingDetector ) . not_to receive ( :detect )
expect ( subject ) . to eq ( expected_string )
expect ( subject . encoding . name ) . to eq ( 'UTF-8' )
end
end
2017-08-17 22:00:37 +05:30
end
describe '#clean' do
[
[
'leaves ascii only string as is' ,
'ascii only string' ,
'ascii only string'
] ,
[
'leaves valid utf8 string as is' ,
'multibyte string №∑∉' ,
'multibyte string №∑∉'
] ,
[
'removes invalid bytes from ASCII-8bit encoded multibyte string.' ,
2019-10-12 21:52:04 +05:30
( + " Lorem ipsum \xC3 \n dolor sit amet, xy \xC3 \xA0 y \xC3 \xB9 abcd \xC3 \xB9 efg " ) . force_encoding ( 'ASCII-8BIT' ) ,
2017-09-10 17:25:29 +05:30
" Lorem ipsum \n dolor sit amet, xyàyùabcdùefg "
2018-04-04 21:44:52 +05:30
] ,
[
'handles UTF-16BE encoded strings' ,
2019-10-12 21:52:04 +05:30
( + " \xFE \xFF \x00 \x41 " ) . force_encoding ( 'ASCII-8BIT' ) , # An "A" prepended with UTF-16 BOM
2018-04-04 21:44:52 +05:30
" \xEF \xBB \xBF A " # An "A" prepended with UTF-8 BOM
2017-09-10 17:25:29 +05:30
]
2017-08-17 22:00:37 +05:30
] . each do | description , test_string , xpect |
it description do
expect ( ext_class . encode! ( test_string ) ) . to eq ( xpect )
end
end
end
2018-03-17 18:26:18 +05:30
describe 'encode_binary' do
[
[ nil , " " ] ,
[ " " , " " ] ,
[ " " , " " ] ,
%w( a1 a1 ) ,
[ " 编码 " , " \xE7 \xBC \x96 \xE7 \xA0 \x81 " . b ]
] . each do | input , result |
it " encodes #{ input . inspect } to #{ result . inspect } " do
expect ( ext_class . encode_binary ( input ) ) . to eq ( result )
end
end
end
2018-11-20 20:47:30 +05:30
2019-07-07 11:18:12 +05:30
describe '#binary_io' do
2018-11-20 20:47:30 +05:30
it 'does not mutate the original string encoding' do
test = 'my-test'
2019-07-07 11:18:12 +05:30
io_stream = ext_class . binary_io ( test )
2018-11-20 20:47:30 +05:30
expect ( io_stream . external_encoding . name ) . to eq ( 'ASCII-8BIT' )
expect ( test . encoding . name ) . to eq ( 'UTF-8' )
end
2019-07-07 11:18:12 +05:30
it 'returns a copy of the IO with the correct encoding' do
test = fixture_file_upload ( 'spec/fixtures/doc_sample.txt' ) . to_io
io_stream = ext_class . binary_io ( test )
expect ( io_stream . external_encoding . name ) . to eq ( 'ASCII-8BIT' )
expect ( test ) . not_to eq ( io_stream )
end
2018-11-20 20:47:30 +05:30
end
2021-06-08 01:23:25 +05:30
describe '#detect_encoding' do
subject { ext_class . detect_encoding ( data , ** kwargs ) }
let ( :data ) { binary_string }
let ( :kwargs ) { { } }
shared_examples 'detects encoding' do
it { is_expected . to be_a ( Hash ) }
it 'correctly detects the binary' do
expect ( subject [ :type ] ) . to eq ( :binary )
end
context 'data is nil' do
let ( :data ) { nil }
it { is_expected . to be_nil }
end
context 'limit is provided' do
let ( :kwargs ) do
{ limit : 10 }
end
it 'correctly detects the binary' do
expect ( subject [ :type ] ) . to eq ( :binary )
end
end
end
context 'cached_encoding_detection is enabled' do
before do
stub_feature_flags ( cached_encoding_detection : true )
end
it_behaves_like 'detects encoding'
context 'cache_key is provided' do
let ( :kwargs ) do
{ cache_key : %w( foo bar ) }
end
it 'uses that cache_key to serve from the cache' do
expect ( Rails . cache ) . to receive ( :fetch ) . with ( [ :detect_binary , CharlockHolmes :: VERSION , %w( foo bar ) ] , expires_in : 1 . week ) . and_call_original
expect ( subject [ :type ] ) . to eq ( :binary )
end
end
end
context 'cached_encoding_detection is disabled' do
before do
stub_feature_flags ( cached_encoding_detection : false )
end
it_behaves_like 'detects encoding'
end
end
2017-08-17 22:00:37 +05:30
end