realaravinth/forgejo-federation
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity

Fix sitemap (#22272)

Browse source 
Fix #22270.

Related to #18407.

The old code treated both sitemap and sitemap index as the format like:

```xml
...
<url>
  <loc>http://localhost:3000/explore/users/sitemap-1.xml</loc>
</url>
...
```

Actually, it's incorrect for sitemap index, it should be:

```xml
...
<sitemap>
  <loc>http://localhost:3000/explore/users/sitemap-1.xml</loc>
</sitemap>
...
```

See https://www.sitemaps.org/protocol.html

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: delvh <dev.lh@web.de>
This commit is contained in:
Jason Song 2022-12-30 23:31:00 +08:00 committed by GitHub
parent 9dcaf14a14
commit d34f3a2213
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 171 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
modules/sitemap/sitemap.go
View file

@ -11,48 +11,62 @@ import (
"time" "time"
) )
// sitemapFileLimit contains the maximum size of a sitemap file const (
const sitemapFileLimit = 50 * 1024 * 1024 sitemapFileLimit = 50 * 1024 * 1024 // the maximum size of a sitemap file
urlsLimit = 50000
// Url represents a single sitemap entry schemaURL = "http://www.sitemaps.org/schemas/sitemap/0.9"
urlsetName = "urlset"
sitemapindexName = "sitemapindex"
)
// URL represents a single sitemap entry
type URL struct { type URL struct {
URL string `xml:"loc"` URL string `xml:"loc"`
LastMod *time.Time `xml:"lastmod,omitempty"` LastMod *time.Time `xml:"lastmod,omitempty"`
} }
// SitemapUrl represents a sitemap // Sitemap represents a sitemap
type Sitemap struct { type Sitemap struct {
XMLName xml.Name XMLName xml.Name
Namespace string `xml:"xmlns,attr"` Namespace string `xml:"xmlns,attr"`
URLs []URL `xml:"url"` URLs []URL `xml:"url"`
Sitemaps []URL `xml:"sitemap"`
} }
// NewSitemap creates a sitemap // NewSitemap creates a sitemap
func NewSitemap() *Sitemap { func NewSitemap() *Sitemap {
return &Sitemap{ return &Sitemap{
XMLName: xml.Name{Local: "urlset"}, XMLName: xml.Name{Local: urlsetName},
Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9", Namespace: schemaURL,
} }
} }
// NewSitemap creates a sitemap index. // NewSitemapIndex creates a sitemap index.
func NewSitemapIndex() *Sitemap { func NewSitemapIndex() *Sitemap {
return &Sitemap{ return &Sitemap{
XMLName: xml.Name{Local: "sitemapindex"}, XMLName: xml.Name{Local: sitemapindexName},
Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9", Namespace: schemaURL,
} }
} }
// Add adds a URL to the sitemap // Add adds a URL to the sitemap
func (s *Sitemap) Add(u URL) { func (s *Sitemap) Add(u URL) {
if s.XMLName.Local == sitemapindexName {
s.Sitemaps = append(s.Sitemaps, u)
} else {
s.URLs = append(s.URLs, u) s.URLs = append(s.URLs, u)
}
} }
// Write writes the sitemap to a response // WriteTo writes the sitemap to a response
func (s *Sitemap) WriteTo(w io.Writer) (int64, error) { func (s *Sitemap) WriteTo(w io.Writer) (int64, error) {
if len(s.URLs) > 50000 { if l := len(s.URLs); l > urlsLimit {
return 0, fmt.Errorf("The sitemap contains too many URLs: %d", len(s.URLs)) return 0, fmt.Errorf("The sitemap contains %d URLs, but only %d are allowed", l, urlsLimit)
}
if l := len(s.Sitemaps); l > urlsLimit {
return 0, fmt.Errorf("The sitemap contains %d sub-sitemaps, but only %d are allowed", l, urlsLimit)
} }
buf := bytes.NewBufferString(xml.Header) buf := bytes.NewBufferString(xml.Header)
if err := xml.NewEncoder(buf).Encode(s); err != nil { if err := xml.NewEncoder(buf).Encode(s); err != nil {
@ -62,7 +76,7 @@ func (s *Sitemap) WriteTo(w io.Writer) (int64, error) {
return 0, err return 0, err
} }
if buf.Len() > sitemapFileLimit { if buf.Len() > sitemapFileLimit {
return 0, fmt.Errorf("The sitemap is too big: %d", buf.Len()) return 0, fmt.Errorf("The sitemap has %d bytes, but only %d are allowed", buf.Len(), sitemapFileLimit)
} }
return buf.WriteTo(w) return buf.WriteTo(w)
} }

174
modules/sitemap/sitemap_test.go
View file

@ -6,7 +6,6 @@ package sitemap
import ( import (
"bytes" "bytes"
"encoding/xml" "encoding/xml"
"fmt"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -14,63 +13,154 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func TestOk(t *testing.T) { func TestNewSitemap(t *testing.T) {
testReal := func(s *Sitemap, name string, urls []URL, expected string) { ts := time.Unix(1651322008, 0).UTC()
for _, url := range urls {
tests := []struct {
name string
urls []URL
want string
wantErr string
}{
{
name: "empty",
urls: []URL{},
want: xml.Header + `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
"" +
"</urlset>\n",
},
{
name: "regular",
urls: []URL{
{URL: "https://gitea.io/test1", LastMod: &ts},
},
want: xml.Header + `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>" +
"</urlset>\n",
},
{
name: "without lastmod",
urls: []URL{
{URL: "https://gitea.io/test1"},
},
want: xml.Header + `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
"<url><loc>https://gitea.io/test1</loc></url>" +
"</urlset>\n",
},
{
name: "multiple",
urls: []URL{
{URL: "https://gitea.io/test1", LastMod: &ts},
{URL: "https://gitea.io/test2", LastMod: nil},
},
want: xml.Header + `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>" +
"<url><loc>https://gitea.io/test2</loc></url>" +
"</urlset>\n",
},
{
name: "too many urls",
urls: make([]URL, 50001),
wantErr: "The sitemap contains 50001 URLs, but only 50000 are allowed",
},
{
name: "too big file",
urls: []URL{
{URL: strings.Repeat("b", 50*1024*1024+1)},
},
wantErr: "The sitemap has 52428932 bytes, but only 52428800 are allowed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := NewSitemap()
for _, url := range tt.urls {
s.Add(url) s.Add(url)
} }
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
_, err := s.WriteTo(buf) _, err := s.WriteTo(buf)
assert.NoError(t, nil, err) if tt.wantErr != "" {
assert.Equal(t, xml.Header+"<"+name+" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"+expected+"</"+name+">\n", buf.String()) assert.EqualError(t, err, tt.wantErr)
} else {
assert.NoError(t, err)
assert.Equalf(t, tt.want, buf.String(), "NewSitemap()")
} }
test := func(urls []URL, expected string) { })
testReal(NewSitemap(), "urlset", urls, expected)
testReal(NewSitemapIndex(), "sitemapindex", urls, expected)
} }
}
func TestNewSitemapIndex(t *testing.T) {
ts := time.Unix(1651322008, 0).UTC() ts := time.Unix(1651322008, 0).UTC()
test( tests := []struct {
[]URL{}, name string
"", urls []URL
) want string
test( wantErr string
[]URL{ }{
{
name: "empty",
urls: []URL{},
want: xml.Header + `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
"" +
"</sitemapindex>\n",
},
{
name: "regular",
urls: []URL{
{URL: "https://gitea.io/test1", LastMod: &ts}, {URL: "https://gitea.io/test1", LastMod: &ts},
}, },
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>", want: xml.Header + `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
) "<sitemap><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></sitemap>" +
test( "</sitemapindex>\n",
[]URL{
{URL: "https://gitea.io/test2", LastMod: nil},
}, },
"<url><loc>https://gitea.io/test2</loc></url>", {
) name: "without lastmod",
test( urls: []URL{
[]URL{ {URL: "https://gitea.io/test1"},
},
want: xml.Header + `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
"<sitemap><loc>https://gitea.io/test1</loc></sitemap>" +
"</sitemapindex>\n",
},
{
name: "multiple",
urls: []URL{
{URL: "https://gitea.io/test1", LastMod: &ts}, {URL: "https://gitea.io/test1", LastMod: &ts},
{URL: "https://gitea.io/test2", LastMod: nil}, {URL: "https://gitea.io/test2", LastMod: nil},
}, },
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>"+ want: xml.Header + `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
"<url><loc>https://gitea.io/test2</loc></url>", "<sitemap><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></sitemap>" +
) "<sitemap><loc>https://gitea.io/test2</loc></sitemap>" +
} "</sitemapindex>\n",
},
func TestTooManyURLs(t *testing.T) { {
s := NewSitemap() name: "too many sitemaps",
for i := 0; i < 50001; i++ { urls: make([]URL, 50001),
s.Add(URL{URL: fmt.Sprintf("https://gitea.io/test%d", i)}) wantErr: "The sitemap contains 50001 sub-sitemaps, but only 50000 are allowed",
},
{
name: "too big file",
urls: []URL{
{URL: strings.Repeat("b", 50*1024*1024+1)},
},
wantErr: "The sitemap has 52428952 bytes, but only 52428800 are allowed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := NewSitemapIndex()
for _, url := range tt.urls {
s.Add(url)
} }
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
_, err := s.WriteTo(buf) _, err := s.WriteTo(buf)
assert.EqualError(t, err, "The sitemap contains too many URLs: 50001") if tt.wantErr != "" {
} assert.EqualError(t, err, tt.wantErr)
} else {
func TestSitemapTooBig(t *testing.T) { assert.NoError(t, err)
s := NewSitemap() assert.Equalf(t, tt.want, buf.String(), "NewSitemapIndex()")
s.Add(URL{URL: strings.Repeat("b", sitemapFileLimit)}) }
buf := &bytes.Buffer{} })
_, err := s.WriteTo(buf) }
assert.EqualError(t, err, "The sitemap is too big: 52428931")
} }