From 143f88d9757303dc742102b0f4cba5bff6143d63 Mon Sep 17 00:00:00 2001 From: 8ctopus <13252042+8ctopus@users.noreply.github.com> Date: Tue, 7 Jan 2020 12:38:22 +0500 Subject: [PATCH] More extensive robots.txt docs (#9566) --- .../doc/advanced/customizing-gitea.en-us.md | 6 --- .../search-engines-indexation.en-us.md | 39 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 docs/content/doc/advanced/search-engines-indexation.en-us.md diff --git a/docs/content/doc/advanced/customizing-gitea.en-us.md b/docs/content/doc/advanced/customizing-gitea.en-us.md index 7ae9b74fa..52d57f676 100644 --- a/docs/content/doc/advanced/customizing-gitea.en-us.md +++ b/docs/content/doc/advanced/customizing-gitea.en-us.md @@ -44,12 +44,6 @@ environment variable; this can be used to override the default path to something **Note:** Gitea must perform a full restart to see configuration changes. -## Customizing /robots.txt - -To make Gitea serve a custom `/robots.txt` (default: empty 404), create a file called -`robots.txt` in the `custom` folder (or `CustomPath`) with -[expected contents](http://www.robotstxt.org/). - ## Serving custom public files To make Gitea serve custom public files (like pages and images), use the folder diff --git a/docs/content/doc/advanced/search-engines-indexation.en-us.md b/docs/content/doc/advanced/search-engines-indexation.en-us.md new file mode 100644 index 000000000..f6dc498e1 --- /dev/null +++ b/docs/content/doc/advanced/search-engines-indexation.en-us.md @@ -0,0 +1,39 @@ +--- +date: "2019-12-31T13:55:00+05:00" +title: "Advanced: Search Engines Indexation" +slug: "search-engines-indexation" +weight: 30 +toc: true +draft: false +menu: + sidebar: + parent: "advanced" + name: "Search Engines Indexation" + weight: 60 + identifier: "search-engines-indexation" +--- + +# Search engines indexation of your Gitea installation + +By default your Gitea installation will be indexed by search engines. +If you don't want your repository to be visible for search engines read further. + +## Block search engines indexation using robots.txt + +To make Gitea serve a custom `robots.txt` (default: empty 404) for top level installations, +create a file called `robots.txt` in the [`custom` folder or `CustomPath`]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}}) + +Examples on how to configure the `robots.txt` can be found at [https://moz.com/learn/seo/robotstxt](https://moz.com/learn/seo/robotstxt). + + +```txt +User-agent: * +Disallow: / +``` + +If you installed Gitea in a subdirectory, you will need to create or edit the `robots.txt` in the top level directory. + +```txt +User-agent: * +Disallow: /gitea/ +```