debian-mirror-gitlab/lib/gitlab/encoding_helper.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

182 lines
6 KiB
Ruby
Raw Normal View History

2018-12-13 13:39:08 +05:30
# frozen_string_literal: true
2017-09-10 17:25:29 +05:30
module Gitlab
module EncodingHelper
extend self
# This threshold is carefully tweaked to prevent usage of encodings detected
# by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
# we're better off sticking with utf8 encoding.
# Reason: git diff can return strings with invalid utf8 byte sequences if it
# truncates a diff in the middle of a multibyte character. In this case
# CharlockHolmes will try to guess the encoding and will likely suggest an
# obscure encoding with low confidence.
# There is a lot more info with this merge request:
# https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
ENCODING_CONFIDENCE_THRESHOLD = 50
2022-06-21 17:19:12 +05:30
UNICODE_REPLACEMENT_CHARACTER = "<EFBFBD>"
2017-09-10 17:25:29 +05:30
def encode!(message)
2018-03-17 18:26:18 +05:30
message = force_encode_utf8(message)
2017-09-10 17:25:29 +05:30
return message if message.valid_encoding?
# return message if message type is binary
2021-06-08 01:23:25 +05:30
detect = detect_encoding(message)
2018-03-17 18:26:18 +05:30
return message.force_encoding("BINARY") if detect_binary?(message, detect)
2017-09-10 17:25:29 +05:30
if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
2018-03-17 18:26:18 +05:30
# force detected encoding if we have sufficient confidence.
2017-09-10 17:25:29 +05:30
message.force_encoding(detect[:encoding])
end
# encode and clean the bad chars
message.replace clean(message)
2018-03-27 19:54:05 +05:30
rescue ArgumentError => e
return unless e.message.include?('unknown encoding name')
2017-09-10 17:25:29 +05:30
encoding = detect ? detect[:encoding] : "unknown"
"--broken encoding: #{encoding}"
end
2021-06-08 01:23:25 +05:30
def detect_encoding(data, limit: CharlockHolmes::EncodingDetector::DEFAULT_BINARY_SCAN_LEN, cache_key: nil)
return if data.nil?
2021-11-11 11:23:49 +05:30
CharlockHolmes::EncodingDetector.new(limit).detect(data)
2021-06-08 01:23:25 +05:30
end
2018-03-17 18:26:18 +05:30
def detect_binary?(data, detect = nil)
2021-06-08 01:23:25 +05:30
detect ||= detect_encoding(data)
2018-03-17 18:26:18 +05:30
detect && detect[:type] == :binary && detect[:confidence] == 100
end
2021-06-08 01:23:25 +05:30
# EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
# only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
# which is what we use below to keep a consistent behavior.
def detect_libgit2_binary?(data, cache_key: nil)
detect = detect_encoding(data, limit: 8000, cache_key: cache_key)
2018-03-17 18:26:18 +05:30
detect && detect[:type] == :binary
end
2021-10-27 15:23:28 +05:30
# This is like encode_utf8 except we skip autodetection of the encoding. We
# assume the data must be interpreted as UTF-8.
def encode_utf8_no_detect(message)
message = force_encode_utf8(message)
return message if message.valid_encoding?
message.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
end
2022-06-21 17:19:12 +05:30
def encode_utf8_with_replacement_character(data)
encode_utf8(data, replace: UNICODE_REPLACEMENT_CHARACTER)
end
2022-10-11 01:57:18 +05:30
# This method escapes unsupported UTF-8 characters instead of deleting them
def encode_utf8_with_escaping!(message)
message = force_encode_utf8(message)
return message if message.valid_encoding?
unless message.valid_encoding?
message = message.chars.map { |char| char.valid_encoding? ? char : escape_chars(char) }.join
end
# encode and clean the bad chars
message.replace clean(message)
end
2020-04-08 14:13:33 +05:30
def encode_utf8(message, replace: "")
2018-03-17 18:26:18 +05:30
message = force_encode_utf8(message)
return message if message.valid_encoding?
2021-06-08 01:23:25 +05:30
detect = detect_encoding(message)
2017-09-10 17:25:29 +05:30
if detect && detect[:encoding]
begin
CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
rescue ArgumentError => e
2020-11-24 15:15:51 +05:30
Gitlab::AppLogger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")
2017-09-10 17:25:29 +05:30
''
end
else
2020-04-08 14:13:33 +05:30
clean(message, replace: replace)
2017-09-10 17:25:29 +05:30
end
2018-03-17 18:26:18 +05:30
rescue ArgumentError
2018-11-18 11:00:15 +05:30
nil
2018-03-17 18:26:18 +05:30
end
2018-11-18 11:00:15 +05:30
def encode_binary(str)
return "" if str.nil?
2018-03-17 18:26:18 +05:30
2018-11-18 11:00:15 +05:30
str.dup.force_encoding(Encoding::ASCII_8BIT)
2018-03-17 18:26:18 +05:30
end
2019-07-07 11:18:12 +05:30
def binary_io(str_or_io)
io = str_or_io.to_io.dup if str_or_io.respond_to?(:to_io)
io ||= StringIO.new(str_or_io.to_s.freeze)
io.tap { |io| io.set_encoding(Encoding::ASCII_8BIT) }
2017-09-10 17:25:29 +05:30
end
2022-06-21 17:19:12 +05:30
ESCAPED_CHARS = {
"a" => "\a", "b" => "\b", "e" => "\e", "f" => "\f",
"n" => "\n", "r" => "\r", "t" => "\t", "v" => "\v",
"\"" => "\""
}.freeze
# rubocop:disable Style/AsciiComments
# `unquote_path` decode filepaths that are returned by some git commands.
# The path may be returned in double-quotes if it contains special characters,
# that are encoded in octal. Also, some characters (see `ESCAPED_CHARS`) are escaped.
# eg. "\311\240\304\253\305\247\305\200\310\247\306\200" (quotes included) is decoded as ɠīŧŀȧƀ
#
# Based on `unquote_c_style` from git source
# https://github.com/git/git/blob/v2.35.1/quote.c#L399
# rubocop:enable Style/AsciiComments
def unquote_path(filename)
return filename unless filename[0] == '"'
filename = filename[1..-2].gsub(/\\(?:([#{ESCAPED_CHARS.keys.join}\\])|(\d{3}))/) do
if c = Regexp.last_match(1)
c == "\\" ? "\\" : ESCAPED_CHARS[c]
elsif c = Regexp.last_match(2)
c.to_i(8).chr
end
end
filename.force_encoding("UTF-8")
end
2017-09-10 17:25:29 +05:30
private
2018-03-17 18:26:18 +05:30
def force_encode_utf8(message)
raise ArgumentError unless message.respond_to?(:force_encoding)
return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
message = message.dup if message.respond_to?(:frozen?) && message.frozen?
message.force_encoding("UTF-8")
end
2022-10-11 01:57:18 +05:30
# Escapes \x80 - \xFF characters not supported by UTF-8
def escape_chars(char)
bytes = char.bytes
return char unless bytes.one?
"%#{bytes.first.to_s(16).upcase}"
end
2020-04-08 14:13:33 +05:30
def clean(message, replace: "")
message.encode(
"UTF-16BE",
undef: :replace,
invalid: :replace,
replace: replace.encode("UTF-16BE")
)
2017-09-10 17:25:29 +05:30
.encode("UTF-8")
.gsub("\0".encode("UTF-8"), "")
end
end
end