realaravinth/bench-forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Replace mahonia with the standard functions in the Golang Sub-repositories

Browse source
This commit is contained in:
Peter Smit 2014-12-22 11:01:52 +02:00
parent 0f77ad219c
commit 3af757ed77
2 changed files with 31 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
models/git_diff.go
View file

@ -14,12 +14,14 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/net/html/charset"
"golang.org/x/text/transform"
"github.com/Unknwon/com" "github.com/Unknwon/com"
"github.com/gogits/gogs/modules/base" "github.com/gogits/gogs/modules/base"
"github.com/gogits/gogs/modules/git" "github.com/gogits/gogs/modules/git"
"github.com/gogits/gogs/modules/log" "github.com/gogits/gogs/modules/log"
"github.com/gogits/gogs/modules/mahonia"
"github.com/gogits/gogs/modules/process" "github.com/gogits/gogs/modules/process"
) )
@ -192,14 +194,18 @@ func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff
} }
// FIXME: use first 30 lines to detect file encoding. // FIXME: use first 30 lines to detect file encoding.
charset, err := base.DetectEncoding(buf.Bytes()) charsetLabel, err := base.DetectEncoding(buf.Bytes())
if charset != "utf8" && err == nil { if charsetLabel != "utf8" && err == nil {
decoder := mahonia.NewDecoder(charset) encoding, _ := charset.Lookup(charsetLabel)
if decoder != nil {
if encoding != nil {
d := encoding.NewDecoder()
for _, f := range diff.Files { for _, f := range diff.Files {
for _, sec := range f.Sections { for _, sec := range f.Sections {
for _, l := range sec.Lines { for _, l := range sec.Lines {
l.Content = decoder.ConvertString(l.Content) if c, _, err := transform.String(d, l.Content); err == nil {
l.Content = c
}
} }
} }
} }

27
modules/base/template.go
View file

@ -7,14 +7,15 @@ package base
import ( import (
"container/list" "container/list"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"html/template" "html/template"
"runtime" "runtime"
"strings" "strings"
"time" "time"
"github.com/gogits/gogs/modules/mahonia" "golang.org/x/net/html/charset"
"golang.org/x/text/transform"
"github.com/gogits/gogs/modules/setting" "github.com/gogits/gogs/modules/setting"
"github.com/saintfish/chardet" "github.com/saintfish/chardet"
) )
@ -54,20 +55,30 @@ func DetectEncoding(content []byte) (string, error) {
} }
func ToUtf8WithErr(content []byte) (error, string) { func ToUtf8WithErr(content []byte) (error, string) {
charset, err := DetectEncoding(content) charsetLabel, err := DetectEncoding(content)
if err != nil { if err != nil {
return err, "" return err, ""
} }
if charset == "utf8" { if charsetLabel == "utf8" {
return nil, string(content) return nil, string(content)
} }
decoder := mahonia.NewDecoder(charset) encoding, _ := charset.Lookup(charsetLabel)
if decoder != nil {
return nil, decoder.ConvertString(string(content)) if encoding == nil {
return fmt.Errorf("unknow char decoder %s", charsetLabel), string(content)
} }
return errors.New("unknow char decoder"), string(content)
result, n, err := transform.String(encoding.NewDecoder(), string(content))
// If there is an error, we concatenate the nicely decoded part and the
// original left over. This way we won't loose data.
if err != nil {
result = result + string(content[n:])
}
return err, result
} }
func ToUtf8(content string) string { func ToUtf8(content string) string {