diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/.gitignore b/debian/gems-compat/gitlab-labkit-0.2.0/.gitignore new file mode 100644 index 0000000000..f8051c1313 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/.gitignore @@ -0,0 +1,3 @@ +Gemfile.lock +*.gem +node_modules diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/.gitlab-ci.yml b/debian/gems-compat/gitlab-labkit-0.2.0/.gitlab-ci.yml new file mode 100644 index 0000000000..6175b791f3 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/.gitlab-ci.yml @@ -0,0 +1,24 @@ +.test_template: &test_definition + stage: test + script: + - bundle install + - bundle exec rake verify build install + +test:2.6: + image: ruby:2.6 + <<: *test_definition + +test:2.5: + image: ruby:2.5 + <<: *test_definition + +test:2.4: + image: ruby:2.4 + <<: *test_definition + +deploy: + stage: deploy + script: + - tools/deploy-rubygem.sh + only: + - tags diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/.rspec b/debian/gems-compat/gitlab-labkit-0.2.0/.rspec new file mode 100644 index 0000000000..9fba4148ee --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/.rspec @@ -0,0 +1,2 @@ +--color +--require ./spec/spec_helper diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/.rubocop.yml b/debian/gems-compat/gitlab-labkit-0.2.0/.rubocop.yml new file mode 100644 index 0000000000..177fe98beb --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/.rubocop.yml @@ -0,0 +1,173 @@ +AllCops: + TargetRubyVersion: 2.4 + +require: + - rubocop-rspec + +Style/HashSyntax: + EnforcedStyle: no_mixed_keys + +Style/SymbolLiteral: + Enabled: No + +Style/TrailingCommaInArguments: + Enabled: No # Delegated to rufo + +Style/TrailingCommaInHashLiteral: + Enabled: No # Delegated to rufo + +Style/FrozenStringLiteralComment: + EnforcedStyle: always + +Style/TrailingCommaInArrayLiteral: + EnforcedStyleForMultiline: comma + +Style/StringLiterals: + EnforcedStyle: double_quotes + +Style/StringLiteralsInInterpolation: + EnforcedStyle: double_quotes + +Layout/MultilineMethodCallIndentation: + Enabled: No + +Layout/SpaceInLambdaLiteral: + Enabled: No + +Layout/SpaceInsideBlockBraces: + Enabled: No + +Layout/FirstParameterIndentation: + Enabled: No + +Metrics/AbcSize: + Enabled: true + Max: 54.28 + +Metrics/BlockLength: + Enabled: false + +Metrics/BlockNesting: + Enabled: true + Max: 4 + +Metrics/ClassLength: + Enabled: false + +Metrics/CyclomaticComplexity: + Enabled: true + Max: 13 + +Metrics/LineLength: + Enabled: false + +Metrics/MethodLength: + Enabled: false + +Metrics/ModuleLength: + Enabled: false + +Metrics/ParameterLists: + Enabled: true + Max: 8 + +Metrics/PerceivedComplexity: + Enabled: true + Max: 14 + +RSpec/AnyInstance: + Enabled: false + +RSpec/BeEql: + Enabled: true + +RSpec/BeforeAfterAll: + Enabled: false + +RSpec/DescribeClass: + Enabled: false + +RSpec/DescribeMethod: + Enabled: false + +RSpec/DescribeSymbol: + Enabled: true + +RSpec/DescribedClass: + Enabled: true + +RSpec/EmptyExampleGroup: + Enabled: true + +RSpec/ExampleLength: + Enabled: false + Max: 5 + +RSpec/ExampleWording: + Enabled: false + CustomTransform: + be: is + have: has + not: does not + IgnoredWords: [] + +RSpec/ExpectActual: + Enabled: true + +RSpec/ExpectOutput: + Enabled: true + +RSpec/FilePath: + Enabled: true + IgnoreMethods: true + +RSpec/Focus: + Enabled: true + +RSpec/HookArgument: + Enabled: true + EnforcedStyle: implicit + +RSpec/ImplicitExpect: + Enabled: true + EnforcedStyle: is_expected + +RSpec/InstanceVariable: + Enabled: false + +RSpec/LeadingSubject: + Enabled: false + +RSpec/LetSetup: + Enabled: false + +RSpec/MessageChain: + Enabled: false + +RSpec/MessageSpies: + Enabled: false + +RSpec/MultipleDescribes: + Enabled: false + +RSpec/MultipleExpectations: + Enabled: false + +RSpec/NamedSubject: + Enabled: false + +RSpec/NestedGroups: + Enabled: false + +RSpec/NotToNot: + EnforcedStyle: not_to + Enabled: true + +RSpec/RepeatedDescription: + Enabled: false + +RSpec/SubjectStub: + Enabled: false + +RSpec/VerifiedDoubles: + Enabled: false diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/.ruby-version b/debian/gems-compat/gitlab-labkit-0.2.0/.ruby-version new file mode 100644 index 0000000000..d4bcea9584 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/.ruby-version @@ -0,0 +1 @@ +ruby-2.5.3 diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/.rufo b/debian/gems-compat/gitlab-labkit-0.2.0/.rufo new file mode 100644 index 0000000000..84b926d98f --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/.rufo @@ -0,0 +1,3 @@ +align_chained_calls true +parens_in_def :dynamic +trailing_commas true diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/Gemfile b/debian/gems-compat/gitlab-labkit-0.2.0/Gemfile new file mode 100644 index 0000000000..be173b205f --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/Gemfile @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gemspec diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/LICENSE b/debian/gems-compat/gitlab-labkit-0.2.0/LICENSE new file mode 100644 index 0000000000..662504449b --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2019 GitLab B.V. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/README.md b/debian/gems-compat/gitlab-labkit-0.2.0/README.md new file mode 100644 index 0000000000..4a7d809612 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/README.md @@ -0,0 +1,39 @@ +# LabKit-Ruby 🔬🔬🔬🔬🔬 + +LabKit-Ruby is minimalist library to provide functionality for Ruby services at GitLab. + +LabKit-Ruby is the Ruby companion for [LabKit](https://gitlab.com/gitlab-org/labkit), a minimalist library to provide functionality for Go services at GitLab. + +LabKit-Ruby and LabKit are intended to provide similar functionality, but use the semantics of their respective languages, so are not intended to provide identical APIS. + +## Documentation + +API Documentation is available at [the Rubydoc site](https://www.rubydoc.info/gems/gitlab-labkit/). + +## Functionality + +LabKit-Ruby provides functionality in three areas: + +1. `Labkit::Correlation` for handling and propagating Correlation-IDs. +1. `Labkit::Logging` for sanitizing log messages. +1. `Labkit::Tracing` for handling and propagating distributed traces. + +## Developing + +Anyone can contribute! + +```console +$ git clone git@gitlab.com:gitlab-org/labkit-ruby.git +$ cd labkit-ruby +$ bundle install + +$ # Autoformat code and auto-correct linters +$ bundle exec rake fix + +$ # Run tests, linters +$ bundle exec rake verify +``` + +Note that LabKit-Ruby uses the [`rufo`](https://github.com/ruby-formatter/rufo) for auto-formatting. Please run `bundle exec rake fix` to auto-format your code before pushing. + +Please also review the [development section of the LabKit (go) README](https://gitlab.com/gitlab-org/labkit#developing-labkit) for details of the LabKit architectural philosophy. diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/Rakefile b/debian/gems-compat/gitlab-labkit-0.2.0/Rakefile new file mode 100644 index 0000000000..9a7206715f --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/Rakefile @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require "bundler/gem_tasks" +require "rufo" + +begin + require "rspec/core/rake_task" + RSpec::Core::RakeTask.new(:spec) +end + +require "rubocop/rake_task" +RuboCop::RakeTask.new + +desc "Alias for `rake rufo:run`" +task :format => ["rufo:run"] + +namespace :rufo do + require "rufo" + + def rufo_command(*switches, rake_args) + files_or_dirs = rake_args[:files_or_dirs] || "." + args = switches + files_or_dirs.split(" ") + Rufo::Command.run(args) + end + + desc "Format Ruby code in current directory" + task :run, [:files_or_dirs] do |_task, rake_args| + rufo_command(rake_args) + end + + desc "Check that no formatting changes are produced" + task :check, [:files_or_dirs] do |_task, rake_args| + rufo_command("--check", rake_args) + end +end + +task :fix => %w[rufo:run rubocop:auto_correct] + +task :verify => %w[spec rufo:check rubocop] + +task :default => %w[verify build] diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/gitlab-labkit.gemspec b/debian/gems-compat/gitlab-labkit-0.2.0/gitlab-labkit.gemspec new file mode 100644 index 0000000000..10392ca162 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/gitlab-labkit.gemspec @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +lib = File.expand_path("lib", __dir__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) + +Gem::Specification.new do |spec| + spec.name = "gitlab-labkit" + spec.version = "0.2.0" + spec.authors = ["Andrew Newdigate"] + spec.email = ["andrew@gitlab.com"] + + spec.summary = "Instrumentation for GitLab" + spec.homepage = "http://about.gitlab.com" + spec.license = "MIT" + + spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|tools)/}) } + spec.require_paths = ["lib"] + spec.required_ruby_version = ">= 2.4.0" + + spec.add_runtime_dependency "actionpack", "~> 5" + spec.add_runtime_dependency "activesupport", "~> 5" + spec.add_runtime_dependency "grpc", "~> 1.15" + spec.add_runtime_dependency "jaeger-client", "~> 0.10" + spec.add_runtime_dependency "opentracing", "~> 0.4" + + spec.add_development_dependency "rack", "~> 2.0" + spec.add_development_dependency "rake", "~> 12.3" + spec.add_development_dependency "rspec", "~> 3.6.0" + spec.add_development_dependency "rspec-parameterized", "~> 0.4" + spec.add_development_dependency "rubocop", "~> 0.65.0" + spec.add_development_dependency "rubocop-rspec", "~> 1.22.1" + spec.add_development_dependency "rufo", "~> 0.6" +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/gitlab-labkit.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/gitlab-labkit.rb new file mode 100644 index 0000000000..975c0aa931 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/gitlab-labkit.rb @@ -0,0 +1,15 @@ +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +require "active_support/all" + +# LabKit is a module for handling cross-project +# infrastructural concerns, partcularly related to +# observability. +module Labkit + autoload :Correlation, "labkit/correlation" + autoload :Tracing, "labkit/tracing" + autoload :Logging, "labkit/logging" +end + +# rubocop:enable Naming/FileName diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/correlation.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/correlation.rb new file mode 100644 index 0000000000..3e0bc20e10 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/correlation.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Labkit + # Correlation provides correlation functionality + module Correlation + autoload :CorrelationId, "labkit/correlation/correlation_id" + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/correlation/correlation_id.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/correlation/correlation_id.rb new file mode 100644 index 0000000000..f993537518 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/correlation/correlation_id.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Labkit + module Correlation + # CorrelationId module provides access the Correlation-ID + # of the current request + module CorrelationId + LOG_KEY = "correlation_id" + + class << self + def use_id(correlation_id, &_blk) + # always generate a id if null is passed + correlation_id ||= new_id + + ids.push(correlation_id || new_id) + + begin + yield(current_id) + ensure + ids.pop + end + end + + def current_id + ids.last + end + + def current_or_new_id + current_id || new_id + end + + private + + def ids + Thread.current[:correlation_id] ||= [] + end + + def new_id + SecureRandom.uuid + end + end + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/logging.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/logging.rb new file mode 100644 index 0000000000..97a2146966 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/logging.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Labkit + # Logging provides functionality for logging, such as + # sanitization + module Logging + autoload :Sanitizer, "labkit/logging/sanitizer" + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/logging/sanitizer.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/logging/sanitizer.rb new file mode 100644 index 0000000000..9b03300710 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/logging/sanitizer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Labkit + module Logging + # Sanitizer provides log message sanitization, removing + # confidential information from log messages + class Sanitizer + SCP_URL_REGEXP = %r{ + (?:((?:[\-_.!~*()a-zA-Z\d;&=+$,]|%[a-fA-F\d]{2})+)(:(?:(?:[\-_.!~*()a-zA-Z\d;:&=+$,]|%[a-fA-F\d]{2})*))?@) (?# 1: username, 2: password) + (?:((?:(?:[a-zA-Z0-9\-._])+|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|\[(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:(?:[a-fA-F\d]{1,4}:)*[a-fA-F\d]{1,4})?::(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))?)\]))) (?# 3: host) + : + ((?:[\-_.!~*'()a-zA-Z\d:@&=+$,]|%[a-fA-F\d]{2})*(?:;(?:[\-_.!~*'()a-zA-Z\d:@&=+$,]|%[a-fA-F\d]{2})*)*(?:\/(?:[\-_.!~*'()a-zA-Z\d:@&=+$,]|%[a-fA-F\d]{2})*(?:;(?:[\-_.!~*'()a-zA-Z\d:@&=+$,]|%[a-fA-F\d]{2})*)*)*)? (?# 4: path) + }x.freeze + SCP_ANCHORED_URL_REGEXP = /^#{SCP_URL_REGEXP}$/x.freeze + ALLOWED_SCHEMES = %w[http https ssh git].freeze + URL_REGEXP = URI::DEFAULT_PARSER.make_regexp(ALLOWED_SCHEMES).freeze + + def self.sanitize_field(content) + content = content.gsub(URL_REGEXP) { |url| mask_url(url) } + content = content.gsub(SCP_URL_REGEXP) { |scp_url| mask_scp_url(scp_url) } + + content + end + + # Ensures that URLS are sanitized to hide credentials + def self.mask_url(url) + url = url.to_s.strip + p = URI::DEFAULT_PARSER.parse(url) + + p.password = "*****" if p.password.present? + p.user = "*****" if p.user.present? + p.to_s + rescue URI::InvalidURIError + "" + end + + # Ensures that URLs of the form user:password@hostname:project.git are + # sanitized to hide credentials + def self.mask_scp_url(scp_url) + scp_url = scp_url.to_s.strip + m = SCP_ANCHORED_URL_REGEXP.match(scp_url) + return "" unless m + + password = m[2] + host = m[3] + path = m[4] + + return "*****@#{host}:#{path}" unless password.present? + + "*****:*****@#{host}:#{path}" + end + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing.rb new file mode 100644 index 0000000000..ab34bf0d9b --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require "active_support/all" + +module Labkit + # Tracing provides distributed tracing functionality + module Tracing + autoload :Factory, "labkit/tracing/factory" + autoload :GRPCInterceptor, "labkit/tracing/grpc_interceptor" + autoload :JaegerFactory, "labkit/tracing/jaeger_factory" + autoload :RackMiddleware, "labkit/tracing/rack_middleware" + autoload :Rails, "labkit/tracing/rails" + autoload :Sidekiq, "labkit/tracing/sidekiq" + autoload :TracingUtils, "labkit/tracing/tracing_utils" + + # Tracing is only enabled when the `GITLAB_TRACING` env var is configured. + def self.enabled? + connection_string.present? + end + + def self.connection_string + ENV["GITLAB_TRACING"] + end + + def self.tracing_url_template + ENV["GITLAB_TRACING_URL"] + end + + def self.tracing_url_enabled? + enabled? && tracing_url_template.present? + end + + # This will provide a link into the distributed tracing for the current trace, + # if it has been captured. + def self.tracing_url(service_name) + return unless tracing_url_enabled? + + correlation_id = Labkit::Correlation::CorrelationId.current_id.to_s + + # Avoid using `format` since it can throw TypeErrors + # which we want to avoid on unsanitised env var input + tracing_url_template.to_s + .gsub("{{ correlation_id }}", correlation_id) + .gsub("{{ service }}", service_name) + end + + # This will run a block with a span + # @param operation_name [String] The operation name for the span + # @param tags [Hash] Tags to assign to the span + # @param child_of [SpanContext, Span] SpanContext that acts as a parent to + # the newly-started span. If a span instance is provided, its + # context is automatically substituted. + def self.with_tracing(**kwargs, &block) + TracingUtils.with_tracing(**kwargs, &block) + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/factory.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/factory.rb new file mode 100644 index 0000000000..81f39ecfec --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/factory.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require "cgi" + +module Labkit + module Tracing + # Factory provides tools for setting up and configuring the + # distributed tracing system within the process, given the + # tracing connection string + class Factory + OPENTRACING_SCHEME = "opentracing" + + def self.create_tracer(service_name, connection_string) + return unless connection_string.present? + + begin + opentracing_details = parse_connection_string(connection_string) + driver_name = opentracing_details[:driver_name] + + case driver_name + when "jaeger" + JaegerFactory.create_tracer(service_name, opentracing_details[:options]) + else + raise "Unknown driver: #{driver_name}" + end + + # Can't create the tracer? Warn and continue sans tracer + rescue StandardError => e + warn "Unable to instantiate tracer: #{e}" + nil + end + end + + def self.parse_connection_string(connection_string) + parsed = URI.parse(connection_string) + + raise "Invalid tracing connection string" unless valid_uri?(parsed) + + { driver_name: parsed.host, options: parse_query(parsed.query) } + end + private_class_method :parse_connection_string + + def self.parse_query(query) + return {} unless query + + CGI.parse(query).symbolize_keys.transform_values(&:first) + end + private_class_method :parse_query + + def self.valid_uri?(uri) + return false unless uri + + uri.scheme == OPENTRACING_SCHEME && uri.host.to_s =~ /^[a-z0-9_]+$/ && uri.path.empty? + end + private_class_method :valid_uri? + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/grpc_interceptor.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/grpc_interceptor.rb new file mode 100644 index 0000000000..d6d0f45535 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/grpc_interceptor.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# rubocop:disable Lint/UnusedMethodArgument +require "opentracing" +require "grpc" + +module Labkit + module Tracing + # GRPCInterceptor is a client-side GRPC interceptor + # for instrumenting GRPC calls with distributed tracing + class GRPCInterceptor < GRPC::ClientInterceptor + include Singleton + + def request_response(request:, call:, method:, metadata:) + wrap_with_tracing(method, "unary", metadata) { yield } + end + + def client_streamer(requests:, call:, method:, metadata:) + wrap_with_tracing(method, "client_stream", metadata) { yield } + end + + def server_streamer(request:, call:, method:, metadata:) + wrap_with_tracing(method, "server_stream", metadata) { yield } + end + + def bidi_streamer(requests:, call:, method:, metadata:) + wrap_with_tracing(method, "bidi_stream", metadata) { yield } + end + + private + + def wrap_with_tracing(method, grpc_type, metadata) + tags = { "component" => "grpc", "span.kind" => "client", "grpc.method" => method, "grpc.type" => grpc_type } + + TracingUtils.with_tracing(operation_name: "grpc:#{method}", tags: tags) do |span| + OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata) + + yield + end + end + end + end +end + +# rubocop:enable Lint/UnusedMethodArgument diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/jaeger_factory.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/jaeger_factory.rb new file mode 100644 index 0000000000..e5bf500bfa --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/jaeger_factory.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require "jaeger/client" + +module Labkit + module Tracing + # JaegerFactory will configure Jaeger distributed tracing + class JaegerFactory + # When the probabilistic sampler is used, by default 0.1% of requests will be traced + DEFAULT_PROBABILISTIC_RATE = 0.001 + + # The default port for the Jaeger agent UDP listener + DEFAULT_UDP_PORT = 6831 + + # Reduce this from default of 10 seconds as the Ruby jaeger + # client doesn't have overflow control, leading to very large + # messages which fail to send over UDP (max packet = 64k) + # Flush more often, with smaller packets + FLUSH_INTERVAL = 5 + + def self.create_tracer(service_name, options) + kwargs = { + service_name: service_name, + sampler: get_sampler(options[:sampler], options[:sampler_param]), + reporter: get_reporter(service_name, options[:http_endpoint], options[:udp_endpoint]), + }.compact + + extra_params = options.except(:sampler, :sampler_param, :http_endpoint, :udp_endpoint, :strict_parsing, :debug) + if extra_params.present? + message = "jaeger tracer: invalid option: #{extra_params.keys.join(", ")}" + + raise message if options[:strict_parsing] + + warn message + end + + Jaeger::Client.build(kwargs) + end + + def self.get_sampler(sampler_type, sampler_param) + case sampler_type + when "probabilistic" + sampler_rate = sampler_param ? sampler_param.to_f : DEFAULT_PROBABILISTIC_RATE + Jaeger::Samplers::Probabilistic.new(rate: sampler_rate) + when "const" + const_value = sampler_param == "1" + Jaeger::Samplers::Const.new(const_value) + end + end + private_class_method :get_sampler + + def self.get_reporter(service_name, http_endpoint, udp_endpoint) + encoder = Jaeger::Encoders::ThriftEncoder.new(service_name: service_name) + + if http_endpoint.present? + sender = get_http_sender(encoder, http_endpoint) + elsif udp_endpoint.present? + sender = get_udp_sender(encoder, udp_endpoint) + else + return nil + end + + Jaeger::Reporters::RemoteReporter.new(sender: sender, flush_interval: FLUSH_INTERVAL) + end + private_class_method :get_reporter + + def self.get_http_sender(encoder, address) + Jaeger::HttpSender.new(url: address, encoder: encoder, logger: Logger.new(STDOUT)) + end + private_class_method :get_http_sender + + def self.get_udp_sender(encoder, address) + pair = address.split(":", 2) + host = pair[0] + port = pair[1] ? pair[1].to_i : DEFAULT_UDP_PORT + + Jaeger::UdpSender.new(host: host, port: port, encoder: encoder, logger: Logger.new(STDOUT)) + end + private_class_method :get_udp_sender + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rack_middleware.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rack_middleware.rb new file mode 100644 index 0000000000..b8ae410c9d --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rack_middleware.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require "opentracing" +require "active_support/all" +require "action_dispatch" + +module Labkit + module Tracing + # RackMiddleware is a rack middleware component for + # instrumenting incoming http requests into a Rails/Rack + # server + class RackMiddleware + REQUEST_METHOD = "REQUEST_METHOD" + + def initialize(app) + @app = app + end + + def call(env) + method = env[REQUEST_METHOD] + + context = TracingUtils.tracer.extract(OpenTracing::FORMAT_RACK, env) + tags = { "component" => "rack", "span.kind" => "server", "http.method" => method, "http.url" => self.class.build_sanitized_url_from_env(env) } + + TracingUtils.with_tracing(operation_name: "http:#{method}", child_of: context, tags: tags) do |span| + @app.call(env).tap { |status_code, _headers, _body| span.set_tag("http.status_code", status_code) } + end + end + + # Generate a sanitized (safe) request URL from the rack environment + def self.build_sanitized_url_from_env(env) + request = ::ActionDispatch::Request.new(env) + + original_url = request.original_url + uri = URI.parse(original_url) + uri.query = request.filtered_parameters.to_query if uri.query.present? + + uri.to_s + end + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails.rb new file mode 100644 index 0000000000..860011e59b --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Labkit + module Tracing + # Rails provides classes for instrumenting Rails events + module Rails + autoload :ActionViewSubscriber, "labkit/tracing/rails/action_view_subscriber" + autoload :ActiveRecordSubscriber, "labkit/tracing/rails/active_record_subscriber" + autoload :RailsCommon, "labkit/tracing/rails/rails_common" + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails/action_view_subscriber.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails/action_view_subscriber.rb new file mode 100644 index 0000000000..9eb69d5845 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails/action_view_subscriber.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Labkit + module Tracing + module Rails + # ActionViewSubscriber bridges action view notifications to + # the distributed tracing subsystem + class ActionViewSubscriber + include RailsCommon + + COMPONENT_TAG = "ActionView" + RENDER_TEMPLATE_NOTIFICATION_TOPIC = "render_template.action_view" + RENDER_COLLECTION_NOTIFICATION_TOPIC = "render_collection.action_view" + RENDER_PARTIAL_NOTIFICATION_TOPIC = "render_partial.action_view" + + # Instruments Rails ActionView events for opentracing. + # Returns a lambda, which, when called will unsubscribe from the notifications + def self.instrument + subscriber = new + + subscriptions = [ + ActiveSupport::Notifications.subscribe(RENDER_TEMPLATE_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify_render_template(start, finish, payload) + end, + ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify_render_collection(start, finish, payload) + end, + ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify_render_partial(start, finish, payload) + end, + ] + + create_unsubscriber subscriptions + end + + # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html + def notify_render_template(start, finish, payload) + generate_span_for_notification("render_template", start, finish, payload, tags_for_render_template(payload)) + end + + def notify_render_collection(start, finish, payload) + generate_span_for_notification("render_collection", start, finish, payload, tags_for_render_collection(payload)) + end + + def notify_render_partial(start, finish, payload) + generate_span_for_notification("render_partial", start, finish, payload, tags_for_render_partial(payload)) + end + + private + + def tags_for_render_template(payload) + { "component" => COMPONENT_TAG, "template.id" => payload[:identifier], "template.layout" => payload[:layout] } + end + + def tags_for_render_collection(payload) + { + "component" => COMPONENT_TAG, + "template.id" => payload[:identifier], + "template.count" => payload[:count] || 0, + "template.cache.hits" => payload[:cache_hits] || 0, + } + end + + def tags_for_render_partial(payload) + { "component" => COMPONENT_TAG, "template.id" => payload[:identifier] } + end + end + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails/active_record_subscriber.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails/active_record_subscriber.rb new file mode 100644 index 0000000000..f4cd04b0c9 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails/active_record_subscriber.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Labkit + module Tracing + module Rails + # ActiveRecordSubscriber bridges active record notifications to + # the distributed tracing subsystem + class ActiveRecordSubscriber + include RailsCommon + + ACTIVE_RECORD_NOTIFICATION_TOPIC = "sql.active_record" + OPERATION_NAME_PREFIX = "active_record:" + DEFAULT_OPERATION_NAME = "sqlquery" + + # Instruments Rails ActiveRecord events for opentracing. + # Returns a lambda, which, when called will unsubscribe from the notifications + def self.instrument + subscriber = new + + subscription = + ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify(start, finish, payload) + end + + create_unsubscriber [subscription] + end + + # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html + def notify(start, finish, payload) + generate_span_for_notification(notification_name(payload), start, finish, payload, tags_for_notification(payload)) + end + + private + + def notification_name(payload) + OPERATION_NAME_PREFIX + (payload[:name].presence || DEFAULT_OPERATION_NAME) + end + + def tags_for_notification(payload) + { + "component" => "ActiveRecord", + "span.kind" => "client", + "db.type" => "sql", + "db.connection_id" => payload[:connection_id], + "db.cached" => payload[:cached] || false, + "db.statement" => payload[:sql], + } + end + end + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails/rails_common.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails/rails_common.rb new file mode 100644 index 0000000000..bb6c56e4ab --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/rails/rails_common.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "active_support/all" + +module Labkit + module Tracing + module Rails + # RailsCommon is a mixin for providing instrumentation + # functionality for the rails instrumentation classes + module RailsCommon + extend ActiveSupport::Concern + + class_methods do + def create_unsubscriber(subscriptions) + -> { subscriptions.each { |subscriber| ActiveSupport::Notifications.unsubscribe(subscriber) } } + end + end + + def generate_span_for_notification(operation_name, start, finish, payload, tags) + exception = payload[:exception] + + TracingUtils.postnotify_span(operation_name, start, finish, tags: tags, exception: exception) + end + end + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq.rb new file mode 100644 index 0000000000..32bf32f8b1 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Labkit + module Tracing + # Sidekiq provides classes for instrumenting Sidekiq client and server + # functionality + module Sidekiq + autoload :ClientMiddleware, "labkit/tracing/sidekiq/client_middleware" + autoload :ServerMiddleware, "labkit/tracing/sidekiq/server_middleware" + autoload :SidekiqCommon, "labkit/tracing/sidekiq/sidekiq_common" + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq/client_middleware.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq/client_middleware.rb new file mode 100644 index 0000000000..af34c9c29b --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq/client_middleware.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "opentracing" + +module Labkit + module Tracing + module Sidekiq + # ClientMiddleware provides a sidekiq client middleware for + # instrumenting distributed tracing calls made from the client + # application + class ClientMiddleware + include SidekiqCommon + + SPAN_KIND = "client" + + def call(_worker_class, job, _queue, _redis_pool) + TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", tags: tags_from_job(job, SPAN_KIND)) do |span| + # Inject the details directly into the job + TracingUtils.tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job) + + yield + end + end + end + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq/server_middleware.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq/server_middleware.rb new file mode 100644 index 0000000000..94eb807ddd --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq/server_middleware.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require "opentracing" + +module Labkit + module Tracing + module Sidekiq + # ServerMiddleware provides a sidekiq server middleware for + # instrumenting distributed tracing calls when they are + # executed by the Sidekiq server + class ServerMiddleware + include SidekiqCommon + + SPAN_KIND = "server" + + def call(_worker, job, _queue) + context = TracingUtils.tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job) + + TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", child_of: context, tags: tags_from_job(job, SPAN_KIND)) { |_span| yield } + end + end + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq/sidekiq_common.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq/sidekiq_common.rb new file mode 100644 index 0000000000..18e8e38eb2 --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/sidekiq/sidekiq_common.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Labkit + module Tracing + module Sidekiq + # SidekiqCommon is a mixin for the sidekiq middleware components + module SidekiqCommon + def tags_from_job(job, kind) + { + "component" => "sidekiq", + "span.kind" => kind, + "sidekiq.queue" => job["queue"], + "sidekiq.jid" => job["jid"], + "sidekiq.retry" => job["retry"].to_s, + "sidekiq.args" => job["args"]&.join(", "), + } + end + end + end + end +end diff --git a/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/tracing_utils.rb b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/tracing_utils.rb new file mode 100644 index 0000000000..3538c4aa8b --- /dev/null +++ b/debian/gems-compat/gitlab-labkit-0.2.0/lib/labkit/tracing/tracing_utils.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require "opentracing" + +module Labkit + module Tracing + # Internal methods for tracing. This is not part of the LabKit public API. + # For internal usage only + class TracingUtils + # Convience method for running a block with a span + def self.with_tracing(operation_name:, tags:, child_of: nil) + scope = tracer.start_active_span(operation_name, child_of: child_of, tags: tags) + span = scope.span + + # Add correlation details to the span if we have them + correlation_id = Labkit::Correlation::CorrelationId.current_id + span.set_tag("correlation_id", correlation_id) if correlation_id + + begin + yield span + rescue StandardError => e + log_exception_on_span(span, e) + raise e + ensure + scope.close + end + end + + # Obtain a tracer instance + def self.tracer + OpenTracing.global_tracer + end + + # Generate a span retrospectively + def self.postnotify_span(operation_name, start_time, end_time, tags: nil, child_of: nil, exception: nil) + span = OpenTracing.start_span(operation_name, start_time: start_time, tags: tags, child_of: child_of) + + log_exception_on_span(span, exception) if exception + + span.finish(end_time: end_time) + end + + # Add exception logging to a span + def self.log_exception_on_span(span, exception) + span.set_tag("error", true) + span.log_kv(kv_tags_for_exception(exception)) + end + + # Generate key-value tags for an exception + def self.kv_tags_for_exception(exception) + case exception + when Exception + { + :"event" => "error", + :"error.kind" => exception.class.to_s, + :"message" => Labkit::Logging::Sanitizer.sanitize_field(exception.message), + :"stack" => exception.backtrace&.join('\n'), + } + else + { :"event" => "error", :"error.kind" => exception.class.to_s, :"error.object" => Labkit::Logging::Sanitizer.sanitize_field(exception.to_s) } + end + end + end + end +end diff --git a/debian/patches/0840-embed-gitlab-labkit.patch b/debian/patches/0840-embed-gitlab-labkit.patch new file mode 100644 index 0000000000..5b3046eb2d --- /dev/null +++ b/debian/patches/0840-embed-gitlab-labkit.patch @@ -0,0 +1,11 @@ +--- a/Gemfile ++++ b/Gemfile +@@ -282,7 +282,7 @@ + gem 'premailer-rails', '~> 1.9', '>= 1.9.7' + + # LabKit: Tracing and Correlation +-gem 'gitlab-labkit', '~> 0.2.0' ++gem 'gitlab-labkit', '~> 0.2.0', path: 'vendor/gems/gitlab-labkit-0.2.0' + + # I18n + gem 'ruby_parser', '~> 3.8', require: false diff --git a/debian/patches/series b/debian/patches/series index 54ce80a0ac..88cdb48559 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -33,3 +33,4 @@ bump-devise-to-4-6.patch 0810-embed-omniauth-salesforce.patch 0820-embed-apollo-upload-server.patch 0830-embed-sassc-rails.patch +0840-embed-gitlab-labkit.patch