debian-mirror-gitlab/public/robots.txt

73 lines
2 KiB
Text
Raw Normal View History

2015-09-11 14:41:01 +05:30
# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
2014-09-02 18:07:02 +05:30
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /
2015-09-11 14:41:01 +05:30
# Add a 1 second delay between successive requests to the same server, limits resources used by crawler
# Only some crawlers respect this setting, e.g. Googlebot does not
# Crawl-delay: 1
# Based on details in https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/routes.rb, https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/routing, and using application
2017-08-17 22:00:37 +05:30
User-Agent: *
2015-09-11 14:41:01 +05:30
Disallow: /autocomplete/users
Disallow: /search
Disallow: /api
Disallow: /admin
Disallow: /profile
Disallow: /dashboard
Disallow: /projects/new
Disallow: /groups/new
Disallow: /groups/*/edit
Disallow: /users
2018-05-09 12:01:36 +05:30
Disallow: /help
2018-11-20 20:47:30 +05:30
# Only specifically allow the Sign In page to avoid very ugly search results
Allow: /users/sign_in
2015-09-11 14:41:01 +05:30
# Global snippets
2017-08-17 22:00:37 +05:30
User-Agent: *
2016-11-03 12:29:30 +05:30
Disallow: /s/
2015-09-11 14:41:01 +05:30
Disallow: /snippets/new
Disallow: /snippets/*/edit
Disallow: /snippets/*/raw
# Project details
2017-08-17 22:00:37 +05:30
User-Agent: *
2015-09-11 14:41:01 +05:30
Disallow: /*/*.git
Disallow: /*/*/fork/new
Disallow: /*/*/repository/archive*
Disallow: /*/*/activity
Disallow: /*/*/new
Disallow: /*/*/edit
Disallow: /*/*/raw
Disallow: /*/*/blame
Disallow: /*/*/commits/*/*
2015-10-24 18:46:33 +05:30
Disallow: /*/*/commit/*.patch
Disallow: /*/*/commit/*.diff
2015-09-11 14:41:01 +05:30
Disallow: /*/*/compare
Disallow: /*/*/branches/new
Disallow: /*/*/tags/new
Disallow: /*/*/network
Disallow: /*/*/graphs
Disallow: /*/*/milestones/new
Disallow: /*/*/milestones/*/edit
Disallow: /*/*/issues/new
Disallow: /*/*/issues/*/edit
Disallow: /*/*/merge_requests/new
Disallow: /*/*/merge_requests/*.patch
Disallow: /*/*/merge_requests/*.diff
Disallow: /*/*/merge_requests/*/edit
Disallow: /*/*/merge_requests/*/diffs
Disallow: /*/*/project_members/import
Disallow: /*/*/labels/new
Disallow: /*/*/labels/*/edit
Disallow: /*/*/wikis/*/edit
Disallow: /*/*/snippets/new
Disallow: /*/*/snippets/*/edit
Disallow: /*/*/snippets/*/raw
Disallow: /*/*/deploy_keys
Disallow: /*/*/hooks
Disallow: /*/*/services
Disallow: /*/*/protected_branches
2016-06-02 11:05:42 +05:30
Disallow: /*/*/uploads/