debian-mirror-gitlab/public/robots.txt

84 lines
2.2 KiB
Text
Raw Normal View History

2015-09-11 14:41:01 +05:30
# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
2014-09-02 18:07:02 +05:30
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /
2015-09-11 14:41:01 +05:30
# Add a 1 second delay between successive requests to the same server, limits resources used by crawler
# Only some crawlers respect this setting, e.g. Googlebot does not
# Crawl-delay: 1
2020-07-28 23:09:34 +05:30
# Based on details in https://gitlab.com/gitlab-org/gitlab/blob/master/config/routes.rb,
# https://gitlab.com/gitlab-org/gitlab/blob/master/spec/routing, and using application
# Global routes
2017-08-17 22:00:37 +05:30
User-Agent: *
2015-09-11 14:41:01 +05:30
Disallow: /autocomplete/users
2021-01-29 00:20:46 +05:30
Disallow: /autocomplete/projects
2015-09-11 14:41:01 +05:30
Disallow: /search
Disallow: /admin
Disallow: /profile
Disallow: /dashboard
Disallow: /users
2018-05-09 12:01:36 +05:30
Disallow: /help
2020-07-28 23:09:34 +05:30
Disallow: /s/
2021-01-29 00:20:46 +05:30
Disallow: /-/profile
Disallow: /-/ide/
# Restrict allowed routes to avoid very ugly search results
2018-11-20 20:47:30 +05:30
Allow: /users/sign_in
2021-01-29 00:20:46 +05:30
Allow: /users/*/snippets
2015-09-11 14:41:01 +05:30
2020-07-28 23:09:34 +05:30
# Generic resource routes like new, edit, raw
# This will block routes like:
# - /projects/new
# - /gitlab-org/gitlab-foss/issues/123/-/edit
2017-08-17 22:00:37 +05:30
User-Agent: *
2020-07-28 23:09:34 +05:30
Disallow: /*/new
Disallow: /*/edit
Disallow: /*/raw
2021-01-29 00:20:46 +05:30
Disallow: /*/realtime_changes
2020-07-28 23:09:34 +05:30
# Group details
User-Agent: *
Disallow: /groups/*/analytics
Disallow: /groups/*/contribution_analytics
Disallow: /groups/*/group_members
2015-09-11 14:41:01 +05:30
# Project details
2017-08-17 22:00:37 +05:30
User-Agent: *
2021-01-29 00:20:46 +05:30
Disallow: /*/*.git$
2020-07-28 23:09:34 +05:30
Disallow: /*/archive/
Disallow: /*/repository/archive*
Disallow: /*/activity
Disallow: /*/blame
Disallow: /*/commits
Disallow: /*/commit
Disallow: /*/commit/*.patch
Disallow: /*/commit/*.diff
Disallow: /*/compare
Disallow: /*/network
Disallow: /*/graphs
Disallow: /*/merge_requests/*.patch
Disallow: /*/merge_requests/*.diff
Disallow: /*/merge_requests/*/diffs
Disallow: /*/deploy_keys
Disallow: /*/hooks
Disallow: /*/services
Disallow: /*/protected_branches
Disallow: /*/uploads/
2019-02-15 15:39:39 +05:30
Disallow: /*/project_members
2020-07-28 23:09:34 +05:30
Disallow: /*/settings
2021-01-29 00:20:46 +05:30
Disallow: /*/-/import
Disallow: /*/-/environments
Disallow: /*/-/jobs
Disallow: /*/-/requirements_management
Disallow: /*/-/pipelines
Disallow: /*/-/pipeline_schedules
Disallow: /*/-/dependencies
Disallow: /*/-/licenses
Disallow: /*/-/metrics
Disallow: /*/-/incidents
Disallow: /*/-/value_stream_analytics
Disallow: /*/-/analytics
Disallow: /*/insights