From 812cfd0ad9bb85b13ce77f611b3c80dad371d1ef Mon Sep 17 00:00:00 2001 From: zeripath Date: Fri, 24 Apr 2020 14:22:36 +0100 Subject: [PATCH] Use markdown frontmatter to provide Table of contents, language and frontmatter rendering (#11047) * Add control for the rendering of the frontmatter * Add control to include a TOC * Add control to set language - allows control of ToC header and CJK glyph choice. Signed-off-by: Andrew Thornton art27@cantab.net --- go.mod | 1 + modules/markup/html.go | 21 +++ modules/markup/markdown/ast.go | 107 +++++++++++++++ modules/markup/markdown/goldmark.go | 172 ++++++++++++++++++++++-- modules/markup/markdown/markdown.go | 7 +- modules/markup/markdown/renderconfig.go | 163 ++++++++++++++++++++++ modules/markup/markdown/toc.go | 49 +++++++ modules/markup/sanitizer.go | 3 + options/locale/locale_en-US.ini | 1 + vendor/modules.txt | 1 + 10 files changed, 509 insertions(+), 16 deletions(-) create mode 100644 modules/markup/markdown/ast.go create mode 100644 modules/markup/markdown/renderconfig.go create mode 100644 modules/markup/markdown/toc.go diff --git a/go.mod b/go.mod index e391da492..8ed0fe928 100644 --- a/go.mod +++ b/go.mod @@ -124,6 +124,7 @@ require ( gopkg.in/ini.v1 v1.52.0 gopkg.in/ldap.v3 v3.0.2 gopkg.in/testfixtures.v2 v2.5.0 + gopkg.in/yaml.v2 v2.2.8 mvdan.cc/xurls/v2 v2.1.0 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 xorm.io/builder v0.3.7 diff --git a/modules/markup/html.go b/modules/markup/html.go index 51d161ecc..294b870d8 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -351,6 +351,27 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { visitText = false } else if node.Data == "code" || node.Data == "pre" { return + } else if node.Data == "i" { + for _, attr := range node.Attr { + if attr.Key != "class" { + continue + } + classes := strings.Split(attr.Val, " ") + for i, class := range classes { + if class == "icon" { + classes[0], classes[i] = classes[i], classes[0] + attr.Val = strings.Join(classes, " ") + + // Remove all children of icons + child := node.FirstChild + for child != nil { + node.RemoveChild(child) + child = node.FirstChild + } + break + } + } + } } for n := node.FirstChild; n != nil; n = n.NextSibling { ctx.visitNode(n, visitText) diff --git a/modules/markup/markdown/ast.go b/modules/markup/markdown/ast.go new file mode 100644 index 000000000..f79d12435 --- /dev/null +++ b/modules/markup/markdown/ast.go @@ -0,0 +1,107 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package markdown + +import "github.com/yuin/goldmark/ast" + +// Details is a block that contains Summary and details +type Details struct { + ast.BaseBlock +} + +// Dump implements Node.Dump . +func (n *Details) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// KindDetails is the NodeKind for Details +var KindDetails = ast.NewNodeKind("Details") + +// Kind implements Node.Kind. +func (n *Details) Kind() ast.NodeKind { + return KindDetails +} + +// NewDetails returns a new Paragraph node. +func NewDetails() *Details { + return &Details{ + BaseBlock: ast.BaseBlock{}, + } +} + +// IsDetails returns true if the given node implements the Details interface, +// otherwise false. +func IsDetails(node ast.Node) bool { + _, ok := node.(*Details) + return ok +} + +// Summary is a block that contains the summary of details block +type Summary struct { + ast.BaseBlock +} + +// Dump implements Node.Dump . +func (n *Summary) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// KindSummary is the NodeKind for Summary +var KindSummary = ast.NewNodeKind("Summary") + +// Kind implements Node.Kind. +func (n *Summary) Kind() ast.NodeKind { + return KindSummary +} + +// NewSummary returns a new Summary node. +func NewSummary() *Summary { + return &Summary{ + BaseBlock: ast.BaseBlock{}, + } +} + +// IsSummary returns true if the given node implements the Summary interface, +// otherwise false. +func IsSummary(node ast.Node) bool { + _, ok := node.(*Summary) + return ok +} + +// Icon is an inline for a fomantic icon +type Icon struct { + ast.BaseInline + Name []byte +} + +// Dump implements Node.Dump . +func (n *Icon) Dump(source []byte, level int) { + m := map[string]string{} + m["Name"] = string(n.Name) + ast.DumpHelper(n, source, level, m, nil) +} + +// KindIcon is the NodeKind for Icon +var KindIcon = ast.NewNodeKind("Icon") + +// Kind implements Node.Kind. +func (n *Icon) Kind() ast.NodeKind { + return KindIcon +} + +// NewIcon returns a new Paragraph node. +func NewIcon(name string) *Icon { + return &Icon{ + BaseInline: ast.BaseInline{}, + Name: []byte(name), + } +} + +// IsIcon returns true if the given node implements the Icon interface, +// otherwise false. +func IsIcon(node ast.Node) bool { + _, ok := node.(*Icon) + return ok +} diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 70f47e289..6edb3e697 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -7,12 +7,16 @@ package markdown import ( "bytes" "fmt" + "regexp" "strings" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/common" + "code.gitea.io/gitea/modules/setting" giteautil "code.gitea.io/gitea/modules/util" + meta "github.com/yuin/goldmark-meta" "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" "github.com/yuin/goldmark/parser" @@ -24,17 +28,56 @@ import ( var byteMailto = []byte("mailto:") -// GiteaASTTransformer is a default transformer of the goldmark tree. -type GiteaASTTransformer struct{} +// Header holds the data about a header. +type Header struct { + Level int + Text string + ID string +} + +// ASTTransformer is a default transformer of the goldmark tree. +type ASTTransformer struct{} // Transform transforms the given AST tree. -func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { +func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { + metaData := meta.GetItems(pc) + firstChild := node.FirstChild() + createTOC := false + var toc = []Header{} + rc := &RenderConfig{ + Meta: "table", + Icon: "table", + Lang: "", + } + if metaData != nil { + rc.ToRenderConfig(metaData) + + metaNode := rc.toMetaNode(metaData) + if metaNode != nil { + node.InsertBefore(node, firstChild, metaNode) + } + createTOC = rc.TOC + toc = make([]Header, 0, 100) + } + _ = ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } switch v := n.(type) { + case *ast.Heading: + if createTOC { + text := n.Text(reader.Source()) + header := Header{ + Text: util.BytesToReadOnlyString(text), + Level: v.Level, + } + if id, found := v.AttributeString("id"); found { + header.ID = util.BytesToReadOnlyString(id.([]byte)) + } + toc = append(toc, header) + } case *ast.Image: // Images need two things: // @@ -91,6 +134,21 @@ func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, } return ast.WalkContinue, nil }) + + if createTOC && len(toc) > 0 { + lang := rc.Lang + if len(lang) == 0 { + lang = setting.Langs[0] + } + tocNode := createTOCNode(toc, lang) + if tocNode != nil { + node.InsertBefore(node, firstChild, tocNode) + } + } + + if len(rc.Lang) > 0 { + node.SetAttributeString("lang", []byte(rc.Lang)) + } } type prefixedIDs struct { @@ -139,10 +197,10 @@ func newPrefixedIDs() *prefixedIDs { } } -// NewTaskCheckBoxHTMLRenderer creates a TaskCheckBoxHTMLRenderer to render tasklists +// NewHTMLRenderer creates a HTMLRenderer to render // in the gitea form. -func NewTaskCheckBoxHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { - r := &TaskCheckBoxHTMLRenderer{ +func NewHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &HTMLRenderer{ Config: html.NewConfig(), } for _, opt := range opts { @@ -151,19 +209,109 @@ func NewTaskCheckBoxHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { return r } -// TaskCheckBoxHTMLRenderer is a renderer.NodeRenderer implementation that -// renders checkboxes in list items. -// Overrides the default goldmark one to present the gitea format -type TaskCheckBoxHTMLRenderer struct { +// HTMLRenderer is a renderer.NodeRenderer implementation that +// renders gitea specific features. +type HTMLRenderer struct { html.Config } // RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. -func (r *TaskCheckBoxHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { +func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindDocument, r.renderDocument) + reg.Register(KindDetails, r.renderDetails) + reg.Register(KindSummary, r.renderSummary) + reg.Register(KindIcon, r.renderIcon) reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox) } -func (r *TaskCheckBoxHTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + log.Info("renderDocument %v", node) + n := node.(*ast.Document) + + if val, has := n.AttributeString("lang"); has { + var err error + if entering { + _, err = w.WriteString("') + } + } else { + _, err = w.WriteString("") + } + + if err != nil { + return ast.WalkStop, err + } + } + + return ast.WalkContinue, nil +} + +func (r *HTMLRenderer) renderDetails(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + var err error + if entering { + _, err = w.WriteString("
") + } else { + _, err = w.WriteString("
") + } + + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} + +func (r *HTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + var err error + if entering { + _, err = w.WriteString("") + } else { + _, err = w.WriteString("") + } + + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} + +var validNameRE = regexp.MustCompile("^[a-z ]+$") + +func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + + n := node.(*Icon) + + name := strings.TrimSpace(strings.ToLower(string(n.Name))) + + if len(name) == 0 { + // skip this + return ast.WalkContinue, nil + } + + if !validNameRE.MatchString(name) { + // skip this + return ast.WalkContinue, nil + } + + var err error + _, err = w.WriteString(fmt.Sprintf(``, name)) + + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} + +func (r *HTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index c48bbab30..e50301ffe 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -54,13 +54,13 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { extension.Ellipsis: nil, }), ), - meta.New(meta.WithTable()), + meta.Meta, ), goldmark.WithParserOptions( parser.WithAttribute(), parser.WithAutoHeadingID(), parser.WithASTTransformers( - util.Prioritized(&GiteaASTTransformer{}, 10000), + util.Prioritized(&ASTTransformer{}, 10000), ), ), goldmark.WithRendererOptions( @@ -71,7 +71,7 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { // Override the original Tasklist renderer! converter.Renderer().AddOptions( renderer.WithNodeRenderers( - util.Prioritized(NewTaskCheckBoxHTMLRenderer(), 1000), + util.Prioritized(NewHTMLRenderer(), 10), ), ) @@ -85,7 +85,6 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { if err := converter.Convert(giteautil.NormalizeEOL(body), &buf, parser.WithContext(pc)); err != nil { log.Error("Unable to render: %v", err) } - return markup.SanitizeReader(&buf).Bytes() } diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go new file mode 100644 index 000000000..bef67e9e5 --- /dev/null +++ b/modules/markup/markdown/renderconfig.go @@ -0,0 +1,163 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package markdown + +import ( + "fmt" + "strings" + + "github.com/yuin/goldmark/ast" + east "github.com/yuin/goldmark/extension/ast" + "gopkg.in/yaml.v2" +) + +// RenderConfig represents rendering configuration for this file +type RenderConfig struct { + Meta string + Icon string + TOC bool + Lang string +} + +// ToRenderConfig converts a yaml.MapSlice to a RenderConfig +func (rc *RenderConfig) ToRenderConfig(meta yaml.MapSlice) { + if meta == nil { + return + } + found := false + var giteaMetaControl yaml.MapItem + for _, item := range meta { + strKey, ok := item.Key.(string) + if !ok { + continue + } + strKey = strings.TrimSpace(strings.ToLower(strKey)) + switch strKey { + case "gitea": + giteaMetaControl = item + found = true + case "include_toc": + val, ok := item.Value.(bool) + if !ok { + continue + } + rc.TOC = val + case "lang": + val, ok := item.Value.(string) + if !ok { + continue + } + val = strings.TrimSpace(val) + if len(val) == 0 { + continue + } + rc.Lang = val + } + } + + if found { + switch v := giteaMetaControl.Value.(type) { + case string: + switch v { + case "none": + rc.Meta = "none" + case "table": + rc.Meta = "table" + default: // "details" + rc.Meta = "details" + } + case yaml.MapSlice: + for _, item := range v { + strKey, ok := item.Key.(string) + if !ok { + continue + } + strKey = strings.TrimSpace(strings.ToLower(strKey)) + switch strKey { + case "meta": + val, ok := item.Value.(string) + if !ok { + continue + } + switch strings.TrimSpace(strings.ToLower(val)) { + case "none": + rc.Meta = "none" + case "table": + rc.Meta = "table" + default: // "details" + rc.Meta = "details" + } + case "details_icon": + val, ok := item.Value.(string) + if !ok { + continue + } + rc.Icon = strings.TrimSpace(strings.ToLower(val)) + case "include_toc": + val, ok := item.Value.(bool) + if !ok { + continue + } + rc.TOC = val + case "lang": + val, ok := item.Value.(string) + if !ok { + continue + } + val = strings.TrimSpace(val) + if len(val) == 0 { + continue + } + rc.Lang = val + } + } + } + } +} + +func (rc *RenderConfig) toMetaNode(meta yaml.MapSlice) ast.Node { + switch rc.Meta { + case "table": + return metaToTable(meta) + case "details": + return metaToDetails(meta, rc.Icon) + default: + return nil + } +} + +func metaToTable(meta yaml.MapSlice) ast.Node { + table := east.NewTable() + alignments := []east.Alignment{} + for range meta { + alignments = append(alignments, east.AlignNone) + } + row := east.NewTableRow(alignments) + for _, item := range meta { + cell := east.NewTableCell() + cell.AppendChild(cell, ast.NewString([]byte(fmt.Sprintf("%v", item.Key)))) + row.AppendChild(row, cell) + } + table.AppendChild(table, east.NewTableHeader(row)) + + row = east.NewTableRow(alignments) + for _, item := range meta { + cell := east.NewTableCell() + cell.AppendChild(cell, ast.NewString([]byte(fmt.Sprintf("%v", item.Value)))) + row.AppendChild(row, cell) + } + table.AppendChild(table, row) + return table +} + +func metaToDetails(meta yaml.MapSlice, icon string) ast.Node { + details := NewDetails() + summary := NewSummary() + summary.AppendChild(summary, NewIcon(icon)) + details.AppendChild(details, summary) + details.AppendChild(details, metaToTable(meta)) + + return details +} diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go new file mode 100644 index 000000000..189821c34 --- /dev/null +++ b/modules/markup/markdown/toc.go @@ -0,0 +1,49 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package markdown + +import ( + "fmt" + "net/url" + + "github.com/unknwon/i18n" + "github.com/yuin/goldmark/ast" +) + +func createTOCNode(toc []Header, lang string) ast.Node { + details := NewDetails() + summary := NewSummary() + + summary.AppendChild(summary, ast.NewString([]byte(i18n.Tr(lang, "toc")))) + details.AppendChild(details, summary) + ul := ast.NewList('-') + details.AppendChild(details, ul) + currentLevel := 6 + for _, header := range toc { + if header.Level < currentLevel { + currentLevel = header.Level + } + } + for _, header := range toc { + for currentLevel > header.Level { + ul = ul.Parent().(*ast.List) + currentLevel-- + } + for currentLevel < header.Level { + newL := ast.NewList('-') + ul.AppendChild(ul, newL) + currentLevel++ + ul = newL + } + li := ast.NewListItem(currentLevel * 2) + a := ast.NewLink() + a.Destination = []byte(fmt.Sprintf("#%s", url.PathEscape(header.ID))) + a.AppendChild(a, ast.NewString([]byte(header.Text))) + li.AppendChild(li, a) + ul.AppendChild(ul, li) + } + + return details +} diff --git a/modules/markup/sanitizer.go b/modules/markup/sanitizer.go index b5c6dc25f..95c6eb0dc 100644 --- a/modules/markup/sanitizer.go +++ b/modules/markup/sanitizer.go @@ -56,6 +56,9 @@ func ReplaceSanitizer() { // Allow classes for task lists sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`task-list`)).OnElements("ul") + // Allow icons + sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^icon(\s+[\p{L}\p{N}_-]+)+$`)).OnElements("i", "span") + // Allow generally safe attributes generalSafeAttrs := []string{"abbr", "accept", "accept-charset", "accesskey", "action", "align", "alt", diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 848cb05a8..2a4789e22 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -19,6 +19,7 @@ create_new = Create… user_profile_and_more = Profile and Settings… signed_in_as = Signed in as enable_javascript = This website works better with JavaScript. +toc = Table of Contents username = Username email = Email Address diff --git a/vendor/modules.txt b/vendor/modules.txt index 545b100d6..d6d4d6c8a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -844,6 +844,7 @@ gopkg.in/toqueteos/substring.v1 # gopkg.in/warnings.v0 v0.1.2 gopkg.in/warnings.v0 # gopkg.in/yaml.v2 v2.2.8 +## explicit gopkg.in/yaml.v2 # mvdan.cc/xurls/v2 v2.1.0 ## explicit