74 lines
2.1 KiB
Text
74 lines
2.1 KiB
Text
# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
|
|
#
|
|
# To ban all spiders from the entire site uncomment the next two lines:
|
|
# User-Agent: *
|
|
# Disallow: /
|
|
|
|
# Add a 1 second delay between successive requests to the same server, limits resources used by crawler
|
|
# Only some crawlers respect this setting, e.g. Googlebot does not
|
|
# Crawl-delay: 1
|
|
|
|
# Based on details in https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/routes.rb, https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/routing, and using application
|
|
User-Agent: *
|
|
Disallow: /autocomplete/users
|
|
Disallow: /search
|
|
Disallow: /api
|
|
Disallow: /admin
|
|
Disallow: /profile
|
|
Disallow: /dashboard
|
|
Disallow: /projects/new
|
|
Disallow: /groups/new
|
|
Disallow: /groups/*/edit
|
|
Disallow: /users
|
|
Disallow: /help
|
|
# Only specifically allow the Sign In page to avoid very ugly search results
|
|
Allow: /users/sign_in
|
|
|
|
# Global snippets
|
|
User-Agent: *
|
|
Disallow: /s/
|
|
Disallow: /snippets/new
|
|
Disallow: /snippets/*/edit
|
|
Disallow: /snippets/*/raw
|
|
|
|
# Project details
|
|
User-Agent: *
|
|
Disallow: /*/*.git
|
|
Disallow: /*/*/fork/new
|
|
Disallow: /*/*/repository/archive*
|
|
Disallow: /*/*/activity
|
|
Disallow: /*/*/new
|
|
Disallow: /*/*/edit
|
|
Disallow: /*/*/raw
|
|
Disallow: /*/*/blame
|
|
Disallow: /*/*/commits/*/*
|
|
Disallow: /*/*/commit/*.patch
|
|
Disallow: /*/*/commit/*.diff
|
|
Disallow: /*/*/compare
|
|
Disallow: /*/*/branches/new
|
|
Disallow: /*/*/tags/new
|
|
Disallow: /*/*/network
|
|
Disallow: /*/*/graphs
|
|
Disallow: /*/*/milestones/new
|
|
Disallow: /*/*/milestones/*/edit
|
|
Disallow: /*/*/issues/new
|
|
Disallow: /*/*/issues/*/edit
|
|
Disallow: /*/*/merge_requests/new
|
|
Disallow: /*/*/merge_requests/*.patch
|
|
Disallow: /*/*/merge_requests/*.diff
|
|
Disallow: /*/*/merge_requests/*/edit
|
|
Disallow: /*/*/merge_requests/*/diffs
|
|
Disallow: /*/*/project_members/import
|
|
Disallow: /*/*/labels/new
|
|
Disallow: /*/*/labels/*/edit
|
|
Disallow: /*/*/wikis/*/edit
|
|
Disallow: /*/*/snippets/new
|
|
Disallow: /*/*/snippets/*/edit
|
|
Disallow: /*/*/snippets/*/raw
|
|
Disallow: /*/*/deploy_keys
|
|
Disallow: /*/*/hooks
|
|
Disallow: /*/*/services
|
|
Disallow: /*/*/protected_branches
|
|
Disallow: /*/*/uploads/
|
|
Disallow: /*/-/group_members
|
|
Disallow: /*/project_members
|