Update markdown parsing library to github.com/gomarkdown/markdown (#944)

vendor/github.com/gomarkdown/markdown/parser/aside.go

@@ -0,0 +1,73 @@
+package parser
+
+import (
+	"bytes"
+
+	"github.com/gomarkdown/markdown/ast"
+)
+
+// returns aisde prefix length
+func (p *Parser) asidePrefix(data []byte) int {
+	i := 0
+	n := len(data)
+	for i < 3 && i < n && data[i] == ' ' {
+		i++
+	}
+	if i+1 < n && data[i] == 'A' && data[i+1] == '>' {
+		if i+2 < n && data[i+2] == ' ' {
+			return i + 3
+		}
+		return i + 2
+	}
+	return 0
+}
+
+// aside ends with at least one blank line
+// followed by something without a aside prefix
+func (p *Parser) terminateAside(data []byte, beg, end int) bool {
+	if p.isEmpty(data[beg:]) <= 0 {
+		return false
+	}
+	if end >= len(data) {
+		return true
+	}
+	return p.asidePrefix(data[end:]) == 0 && p.isEmpty(data[end:]) == 0
+}
+
+// parse a aside fragment
+func (p *Parser) aside(data []byte) int {
+	var raw bytes.Buffer
+	beg, end := 0, 0
+	// identical to quote
+	for beg < len(data) {
+		end = beg
+		// Step over whole lines, collecting them. While doing that, check for
+		// fenced code and if one's found, incorporate it altogether,
+		// irregardless of any contents inside it
+		for end < len(data) && data[end] != '\n' {
+			if p.extensions&FencedCode != 0 {
+				if i := p.fencedCodeBlock(data[end:], false); i > 0 {
+					// -1 to compensate for the extra end++ after the loop:
+					end += i - 1
+					break
+				}
+			}
+			end++
+		}
+		end = skipCharN(data, end, '\n', 1)
+		if pre := p.asidePrefix(data[beg:]); pre > 0 {
+			// skip the prefix
+			beg += pre
+		} else if p.terminateAside(data, beg, end) {
+			break
+		}
+		// this line is part of the aside
+		raw.Write(data[beg:end])
+		beg = end
+	}
+
+	block := p.addBlock(&ast.Aside{})
+	p.block(raw.Bytes())
+	p.finalize(block)
+	return end
+}

vendor/github.com/gomarkdown/markdown/parser/attribute.go

@@ -0,0 +1,116 @@
+package parser
+
+import (
+	"bytes"
+
+	"github.com/gomarkdown/markdown/ast"
+)
+
+// attribute parses a (potential) block attribute and adds it to p.
+func (p *Parser) attribute(data []byte) []byte {
+	if len(data) < 3 {
+		return data
+	}
+	i := 0
+	if data[i] != '{' {
+		return data
+	}
+	i++
+
+	// last character must be a } otherwise it's not an attribute
+	end := skipUntilChar(data, i, '\n')
+	if data[end-1] != '}' {
+		return data
+	}
+
+	i = skipSpace(data, i)
+	b := &ast.Attribute{Attrs: make(map[string][]byte)}
+
+	esc := false
+	quote := false
+	trail := 0
+Loop:
+	for ; i < len(data); i++ {
+		switch data[i] {
+		case ' ', '\t', '\f', '\v':
+			if quote {
+				continue
+			}
+			chunk := data[trail+1 : i]
+			if len(chunk) == 0 {
+				trail = i
+				continue
+			}
+			switch {
+			case chunk[0] == '.':
+				b.Classes = append(b.Classes, chunk[1:])
+			case chunk[0] == '#':
+				b.ID = chunk[1:]
+			default:
+				k, v := keyValue(chunk)
+				if k != nil && v != nil {
+					b.Attrs[string(k)] = v
+				} else {
+					// this is illegal in an attribute
+					return data
+				}
+			}
+			trail = i
+		case '"':
+			if esc {
+				esc = !esc
+				continue
+			}
+			quote = !quote
+		case '\\':
+			esc = !esc
+		case '}':
+			if esc {
+				esc = !esc
+				continue
+			}
+			chunk := data[trail+1 : i]
+			if len(chunk) == 0 {
+				return data
+			}
+			switch {
+			case chunk[0] == '.':
+				b.Classes = append(b.Classes, chunk[1:])
+			case chunk[0] == '#':
+				b.ID = chunk[1:]
+			default:
+				k, v := keyValue(chunk)
+				if k != nil && v != nil {
+					b.Attrs[string(k)] = v
+				} else {
+					return data
+				}
+			}
+			i++
+			break Loop
+		default:
+			esc = false
+		}
+	}
+
+	p.attr = b
+	return data[i:]
+}
+
+// key="value" quotes are mandatory.
+func keyValue(data []byte) ([]byte, []byte) {
+	chunk := bytes.SplitN(data, []byte{'='}, 2)
+	if len(chunk) != 2 {
+		return nil, nil
+	}
+	key := chunk[0]
+	value := chunk[1]
+
+	if len(value) < 3 || len(key) == 0 {
+		return nil, nil
+	}
+	if value[0] != '"' || value[len(value)-1] != '"' {
+		return key, nil
+	}
+	return key, value[1 : len(value)-1]
+}

vendor/github.com/gomarkdown/markdown/parser/callout.go

@@ -0,0 +1,29 @@
+package parser
+
+import (
+	"bytes"
+	"strconv"
+)
+
+// IsCallout detects a callout in the following format: <<N>> Where N is a integer > 0.
+func IsCallout(data []byte) (id []byte, consumed int) {
+	if !bytes.HasPrefix(data, []byte("<<")) {
+		return nil, 0
+	}
+	start := 2
+	end := bytes.Index(data[start:], []byte(">>"))
+	if end < 0 {
+		return nil, 0
+	}
+
+	b := data[start : start+end]
+	b = bytes.TrimSpace(b)
+	i, err := strconv.Atoi(string(b))
+	if err != nil {
+		return nil, 0
+	}
+	if i <= 0 {
+		return nil, 0
+	}
+	return b, start + end + 2 // 2 for >>
+}

vendor/github.com/gomarkdown/markdown/parser/caption.go

@@ -0,0 +1,70 @@
+package parser
+
+import (
+	"bytes"
+)
+
+// caption checks for a caption, it returns the caption data and a potential "headingID".
+func (p *Parser) caption(data, caption []byte) ([]byte, string, int) {
+	if !bytes.HasPrefix(data, caption) {
+		return nil, "", 0
+	}
+	j := len(caption)
+	data = data[j:]
+	end := p.linesUntilEmpty(data)
+
+	data = data[:end]
+
+	id, start := captionID(data)
+	if id != "" {
+		return data[:start], id, end + j
+	}
+
+	return data, "", end + j
+}
+
+// linesUntilEmpty scans lines up to the first empty line.
+func (p *Parser) linesUntilEmpty(data []byte) int {
+	line, i := 0, 0
+
+	for line < len(data) {
+		i++
+
+		// find the end of this line
+		for i < len(data) && data[i-1] != '\n' {
+			i++
+		}
+
+		if p.isEmpty(data[line:i]) == 0 {
+			line = i
+			continue
+		}
+
+		break
+	}
+	return i
+}
+
+// captionID checks if the caption *ends* in {#....}. If so the text after {# is taken to be
+// the ID/anchor of the entire figure block.
+func captionID(data []byte) (string, int) {
+	end := len(data)
+
+	j, k := 0, 0
+	// find start/end of heading id
+	for j = 0; j < end-1 && (data[j] != '{' || data[j+1] != '#'); j++ {
+	}
+	for k = j + 1; k < end && data[k] != '}'; k++ {
+	}
+	// remains must be whitespace.
+	for l := k + 1; l < end; l++ {
+		if !isSpace(data[l]) {
+			return "", 0
+		}
+	}
+
+	if j > 0 && k > 0 && j+2 < k {
+		return string(data[j+2 : k]), j
+	}
+	return "", 0
+}

vendor/github.com/gomarkdown/markdown/parser/citation.go

@@ -0,0 +1,86 @@
+package parser
+
+import (
+	"bytes"
+
+	"github.com/gomarkdown/markdown/ast"
+)
+
+// citation parses a citation. In its most simple form [@ref], we allow multiple
+// being separated by semicolons and a sub reference inside ala pandoc: [@ref, p. 23].
+// Each citation can have a modifier: !, ? or - wich mean:
+//
+// ! - normative
+// ? - formative
+// - - suppressed
+//
+// The suffix starts after a comma, we strip any whitespace before and after. If the output
+// allows for it, this can be rendered.
+func citation(p *Parser, data []byte, offset int) (int, ast.Node) {
+	// look for the matching closing bracket
+	i := offset + 1
+	for level := 1; level > 0 && i < len(data); i++ {
+		switch {
+		case data[i] == '\n':
+			// no newlines allowed.
+			return 0, nil
+
+		case data[i-1] == '\\':
+			continue
+
+		case data[i] == '[':
+			level++
+
+		case data[i] == ']':
+			level--
+			if level <= 0 {
+				i-- // compensate for extra i++ in for loop
+			}
+		}
+	}
+
+	if i >= len(data) {
+		return 0, nil
+	}
+
+	node := &ast.Citation{}
+
+	citations := bytes.Split(data[1:i], []byte(";"))
+	for _, citation := range citations {
+		var suffix []byte
+		citation = bytes.TrimSpace(citation)
+		j := 0
+		if citation[j] != '@' {
+			// not a citation, drop out entirely.
+			return 0, nil
+		}
+		if c := bytes.Index(citation, []byte(",")); c > 0 {
+			part := citation[:c]
+			suff := citation[c+1:]
+			part = bytes.TrimSpace(part)
+			suff = bytes.TrimSpace(suff)
+
+			citation = part
+			suffix = suff
+		}
+
+		citeType := ast.CitationTypeInformative
+		j = 1
+		switch citation[j] {
+		case '!':
+			citeType = ast.CitationTypeNormative
+			j++
+		case '?':
+			citeType = ast.CitationTypeInformative
+			j++
+		case '-':
+			citeType = ast.CitationTypeSuppressed
+			j++
+		}
+		node.Destination = append(node.Destination, citation[j:])
+		node.Type = append(node.Type, citeType)
+		node.Suffix = append(node.Suffix, suffix)
+	}
+
+	return i + 1, node
+}

vendor/github.com/gomarkdown/markdown/parser/esc.go

@@ -0,0 +1,20 @@
+package parser
+
+// isEscape returns true if byte i is prefixed by an odd number of backslahses.
+func isEscape(data []byte, i int) bool {
+	if i == 0 {
+		return false
+	}
+	if i == 1 {
+		return data[0] == '\\'
+	}
+	j := i - 1
+	for ; j >= 0; j-- {
+		if data[j] != '\\' {
+			break
+		}
+	}
+	j++
+	// odd number of backslahes means escape
+	return (i-j)%2 != 0
+}

vendor/github.com/gomarkdown/markdown/parser/figures.go

@@ -0,0 +1,119 @@
+package parser
+
+import (
+	"bytes"
+
+	"github.com/gomarkdown/markdown/ast"
+)
+
+// sFigureLine checks if there's a figure line (e.g., !--- ) at the beginning of data,
+// and returns the end index if so, or 0 otherwise.
+func sFigureLine(data []byte, oldmarker string) (end int, marker string) {
+	i, size := 0, 0
+
+	n := len(data)
+	// skip up to three spaces
+	for i < n && i < 3 && data[i] == ' ' {
+		i++
+	}
+
+	// check for the marker characters: !
+	if i+1 >= n {
+		return 0, ""
+	}
+	if data[i] != '!' || data[i+1] != '-' {
+		return 0, ""
+	}
+	i++
+
+	c := data[i] // i.e. the -
+
+	// the whole line must be the same char or whitespace
+	for i < n && data[i] == c {
+		size++
+		i++
+	}
+
+	// the marker char must occur at least 3 times
+	if size < 3 {
+		return 0, ""
+	}
+	marker = string(data[i-size : i])
+
+	// if this is the end marker, it must match the beginning marker
+	if oldmarker != "" && marker != oldmarker {
+		return 0, ""
+	}
+
+	// there is no syntax modifier although it might be an idea to re-use this space for something?
+
+	i = skipChar(data, i, ' ')
+	if i >= n || data[i] != '\n' {
+		if i == n {
+			return i, marker
+		}
+		return 0, ""
+	}
+	return i + 1, marker // Take newline into account.
+}
+
+// figureBlock returns the end index if data contains a figure block at the beginning,
+// or 0 otherwise. It writes to out if doRender is true, otherwise it has no side effects.
+// If doRender is true, a final newline is mandatory to recognize the figure block.
+func (p *Parser) figureBlock(data []byte, doRender bool) int {
+	beg, marker := sFigureLine(data, "")
+	if beg == 0 || beg >= len(data) {
+		return 0
+	}
+
+	var raw bytes.Buffer
+
+	for {
+		// safe to assume beg < len(data)
+
+		// check for the end of the code block
+		figEnd, _ := sFigureLine(data[beg:], marker)
+		if figEnd != 0 {
+			beg += figEnd
+			break
+		}
+
+		// copy the current line
+		end := skipUntilChar(data, beg, '\n') + 1
+
+		// did we reach the end of the buffer without a closing marker?
+		if end >= len(data) {
+			return 0
+		}
+
+		// verbatim copy to the working buffer
+		if doRender {
+			raw.Write(data[beg:end])
+		}
+		beg = end
+	}
+
+	if !doRender {
+		return beg
+	}
+
+	figure := &ast.CaptionFigure{}
+	p.addBlock(figure)
+	p.block(raw.Bytes())
+
+	defer p.finalize(figure)
+
+	if captionContent, id, consumed := p.caption(data[beg:], []byte("Figure: ")); consumed > 0 {
+		caption := &ast.Caption{}
+		p.Inline(caption, captionContent)
+
+		figure.HeadingID = id
+
+		p.addChild(caption)
+
+		beg += consumed
+	}
+
+	p.finalize(figure)
+	return beg
+}

vendor/github.com/gomarkdown/markdown/parser/include.go

@@ -0,0 +1,129 @@
+package parser
+
+import (
+	"bytes"
+	"path"
+	"path/filepath"
+)
+
+// isInclude parses {{...}}[...], that contains a path between the {{, the [...] syntax contains
+// an address to select which lines to include. It is treated as an opaque string and just given
+// to readInclude.
+func (p *Parser) isInclude(data []byte) (filename string, address []byte, consumed int) {
+	i := skipCharN(data, 0, ' ', 3) // start with up to 3 spaces
+	if len(data[i:]) < 3 {
+		return "", nil, 0
+	}
+	if data[i] != '{' || data[i+1] != '{' {
+		return "", nil, 0
+	}
+	start := i + 2
+
+	// find the end delimiter
+	i = skipUntilChar(data, i, '}')
+	if i+1 >= len(data) {
+		return "", nil, 0
+	}
+	end := i
+	i++
+	if data[i] != '}' {
+		return "", nil, 0
+	}
+	filename = string(data[start:end])
+
+	if i+1 < len(data) && data[i+1] == '[' { // potential address specification
+		start := i + 2
+
+		end = skipUntilChar(data, start, ']')
+		if end >= len(data) {
+			return "", nil, 0
+		}
+		address = data[start:end]
+		return filename, address, end + 1
+	}
+
+	return filename, address, i + 1
+}
+
+func (p *Parser) readInclude(from, file string, address []byte) []byte {
+	if p.Opts.ReadIncludeFn != nil {
+		return p.Opts.ReadIncludeFn(from, file, address)
+	}
+
+	return nil
+}
+
+// isCodeInclude parses <{{...}} which is similar to isInclude the returned bytes are, however wrapped in a code block.
+func (p *Parser) isCodeInclude(data []byte) (filename string, address []byte, consumed int) {
+	i := skipCharN(data, 0, ' ', 3) // start with up to 3 spaces
+	if len(data[i:]) < 3 {
+		return "", nil, 0
+	}
+	if data[i] != '<' {
+		return "", nil, 0
+	}
+	start := i
+
+	filename, address, consumed = p.isInclude(data[i+1:])
+	if consumed == 0 {
+		return "", nil, 0
+	}
+	return filename, address, start + consumed + 1
+}
+
+// readCodeInclude acts like include except the returned bytes are wrapped in a fenced code block.
+func (p *Parser) readCodeInclude(from, file string, address []byte) []byte {
+	data := p.readInclude(from, file, address)
+	if data == nil {
+		return nil
+	}
+	ext := path.Ext(file)
+	buf := &bytes.Buffer{}
+	buf.Write([]byte("```"))
+	if ext != "" { // starts with a dot
+		buf.WriteString(" " + ext[1:] + "\n")
+	} else {
+		buf.WriteByte('\n')
+	}
+	buf.Write(data)
+	buf.WriteString("```\n")
+	return buf.Bytes()
+}
+
+// incStack hold the current stack of chained includes. Each value is the containing
+// path of the file being parsed.
+type incStack struct {
+	stack []string
+}
+
+func newIncStack() *incStack {
+	return &incStack{stack: []string{}}
+}
+
+// Push updates i with new.
+func (i *incStack) Push(new string) {
+	if path.IsAbs(new) {
+		i.stack = append(i.stack, path.Dir(new))
+		return
+	}
+	last := ""
+	if len(i.stack) > 0 {
+		last = i.stack[len(i.stack)-1]
+	}
+	i.stack = append(i.stack, path.Dir(filepath.Join(last, new)))
+}
+
+// Pop pops the last value.
+func (i *incStack) Pop() {
+	if len(i.stack) == 0 {
+		return
+	}
+	i.stack = i.stack[:len(i.stack)-1]
+}
+
+func (i *incStack) Last() string {
+	if len(i.stack) == 0 {
+		return ""
+	}
+	return i.stack[len(i.stack)-1]
+}

vendor/github.com/gomarkdown/markdown/parser/matter.go

@@ -0,0 +1,36 @@
+package parser
+
+import (
+	"bytes"
+
+	"github.com/gomarkdown/markdown/ast"
+)
+
+func (p *Parser) documentMatter(data []byte) int {
+	if data[0] != '{' {
+		return 0
+	}
+
+	consumed := 0
+	matter := ast.DocumentMatterNone
+	if bytes.HasPrefix(data, []byte("{frontmatter}")) {
+		consumed = len("{frontmatter}")
+		matter = ast.DocumentMatterFront
+	}
+	if bytes.HasPrefix(data, []byte("{mainmatter}")) {
+		consumed = len("{mainmatter}")
+		matter = ast.DocumentMatterMain
+	}
+	if bytes.HasPrefix(data, []byte("{backmatter}")) {
+		consumed = len("{backmatter}")
+		matter = ast.DocumentMatterBack
+	}
+	if consumed == 0 {
+		return 0
+	}
+	node := &ast.DocumentMatter{Matter: matter}
+	p.addBlock(node)
+	p.finalize(node)
+
+	return consumed
+}

vendor/github.com/gomarkdown/markdown/parser/options.go

@@ -0,0 +1,32 @@
+package parser
+
+import (
+	"github.com/gomarkdown/markdown/ast"
+)
+
+// Flags control optional behavior of parser.
+type Flags int
+
+// Options is a collection of supplementary parameters tweaking the behavior of various parts of the parser.
+type Options struct {
+	ParserHook    BlockFunc
+	ReadIncludeFn ReadIncludeFunc
+
+	Flags Flags // Flags allow customizing parser's behavior
+}
+
+// Parser renderer configuration options.
+const (
+	FlagsNone        Flags = 0
+	SkipFootnoteList Flags = 1 << iota // Skip adding the footnote list (regardless if they are parsed)
+)
+
+// BlockFunc allows to registration of a parser function. If successful it
+// returns an ast.Node, a buffer that should be parsed as a block and the the number of bytes consumed.
+type BlockFunc func(data []byte) (ast.Node, []byte, int)
+
+// ReadIncludeFunc should read the file under path and returns the read bytes,
+// from will be set to the name of the current file being parsed. Initially
+// this will be empty. address is the optional address specifier of which lines
+// of the file to return. If this function is not set no data will be read.
+type ReadIncludeFunc func(from, path string, address []byte) []byte

vendor/github.com/gomarkdown/markdown/parser/parser.go

@@ -0,0 +1,812 @@
+/*
+Package parser implements parser for markdown text that generates AST (abstract syntax tree).
+*/
+package parser
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/gomarkdown/markdown/ast"
+)
+
+// Extensions is a bitmask of enabled parser extensions.
+type Extensions int
+
+// Bit flags representing markdown parsing extensions.
+// Use | (or) to specify multiple extensions.
+const (
+	NoExtensions           Extensions = 0
+	NoIntraEmphasis        Extensions = 1 << iota // Ignore emphasis markers inside words
+	Tables                                        // Parse tables
+	FencedCode                                    // Parse fenced code blocks
+	Autolink                                      // Detect embedded URLs that are not explicitly marked
+	Strikethrough                                 // Strikethrough text using ~~test~~
+	LaxHTMLBlocks                                 // Loosen up HTML block parsing rules
+	SpaceHeadings                                 // Be strict about prefix heading rules
+	HardLineBreak                                 // Translate newlines into line breaks
+	NonBlockingSpace                              // Translate backspace spaces into line non-blocking spaces
+	TabSizeEight                                  // Expand tabs to eight spaces instead of four
+	Footnotes                                     // Pandoc-style footnotes
+	NoEmptyLineBeforeBlock                        // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
+	HeadingIDs                                    // specify heading IDs  with {#id}
+	Titleblock                                    // Titleblock ala pandoc
+	AutoHeadingIDs                                // Create the heading ID from the text
+	BackslashLineBreak                            // Translate trailing backslashes into line breaks
+	DefinitionLists                               // Parse definition lists
+	MathJax                                       // Parse MathJax
+	OrderedListStart                              // Keep track of the first number used when starting an ordered list.
+	Attributes                                    // Block Attributes
+	SuperSubscript                                // Super- and subscript support: 2^10^, H~2~O.
+	EmptyLinesBreakList                           // 2 empty lines break out of list
+	Includes                                      // Support including other files.
+	Mmark                                         // Support Mmark syntax, see https://mmark.nl/syntax
+
+	CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode |
+		Autolink | Strikethrough | SpaceHeadings | HeadingIDs |
+		BackslashLineBreak | DefinitionLists | MathJax
+)
+
+// The size of a tab stop.
+const (
+	tabSizeDefault = 4
+	tabSizeDouble  = 8
+)
+
+// for each character that triggers a response when parsing inline data.
+type inlineParser func(p *Parser, data []byte, offset int) (int, ast.Node)
+
+// ReferenceOverrideFunc is expected to be called with a reference string and
+// return either a valid Reference type that the reference string maps to or
+// nil. If overridden is false, the default reference logic will be executed.
+// See the documentation in Options for more details on use-case.
+type ReferenceOverrideFunc func(reference string) (ref *Reference, overridden bool)
+
+// Parser is a type that holds extensions and the runtime state used by
+// Parse, and the renderer. You can not use it directly, construct it with New.
+type Parser struct {
+
+	// ReferenceOverride is an optional function callback that is called every
+	// time a reference is resolved. It can be set before starting parsing.
+	//
+	// In Markdown, the link reference syntax can be made to resolve a link to
+	// a reference instead of an inline URL, in one of the following ways:
+	//
+	//  * [link text][refid]
+	//  * [refid][]
+	//
+	// Usually, the refid is defined at the bottom of the Markdown document. If
+	// this override function is provided, the refid is passed to the override
+	// function first, before consulting the defined refids at the bottom. If
+	// the override function indicates an override did not occur, the refids at
+	// the bottom will be used to fill in the link details.
+	ReferenceOverride ReferenceOverrideFunc
+
+	Opts Options
+
+	// after parsing, this is AST root of parsed markdown text
+	Doc ast.Node
+
+	extensions Extensions
+
+	refs           map[string]*reference
+	refsRecord     map[string]struct{}
+	inlineCallback [256]inlineParser
+	nesting        int
+	maxNesting     int
+	insideLink     bool
+	indexCnt       int // incremented after every index
+
+	// Footnotes need to be ordered as well as available to quickly check for
+	// presence. If a ref is also a footnote, it's stored both in refs and here
+	// in notes. Slice is nil if footnotes not enabled.
+	notes []*reference
+
+	tip                  ast.Node // = doc
+	oldTip               ast.Node
+	lastMatchedContainer ast.Node // = doc
+	allClosed            bool
+
+	// Attributes are attached to block level elements.
+	attr *ast.Attribute
+
+	includeStack *incStack
+}
+
+// New creates a markdown parser with CommonExtensions.
+//
+// You can then call `doc := p.Parse(markdown)` to parse markdown document
+// and `markdown.Render(doc, renderer)` to convert it to another format with
+// a renderer.
+func New() *Parser {
+	return NewWithExtensions(CommonExtensions)
+}
+
+// NewWithExtensions creates a markdown parser with given extensions.
+func NewWithExtensions(extension Extensions) *Parser {
+	p := Parser{
+		refs:         make(map[string]*reference),
+		refsRecord:   make(map[string]struct{}),
+		maxNesting:   16,
+		insideLink:   false,
+		Doc:          &ast.Document{},
+		extensions:   extension,
+		allClosed:    true,
+		includeStack: newIncStack(),
+	}
+	p.tip = p.Doc
+	p.oldTip = p.Doc
+	p.lastMatchedContainer = p.Doc
+
+	p.inlineCallback[' '] = maybeLineBreak
+	p.inlineCallback['*'] = emphasis
+	p.inlineCallback['_'] = emphasis
+	if p.extensions&Strikethrough != 0 {
+		p.inlineCallback['~'] = emphasis
+	}
+	p.inlineCallback['`'] = codeSpan
+	p.inlineCallback['\n'] = lineBreak
+	p.inlineCallback['['] = link
+	p.inlineCallback['<'] = leftAngle
+	p.inlineCallback['\\'] = escape
+	p.inlineCallback['&'] = entity
+	p.inlineCallback['!'] = maybeImage
+	if p.extensions&Mmark != 0 {
+		p.inlineCallback['('] = maybeShortRefOrIndex
+	}
+	p.inlineCallback['^'] = maybeInlineFootnoteOrSuper
+	if p.extensions&Autolink != 0 {
+		p.inlineCallback['h'] = maybeAutoLink
+		p.inlineCallback['m'] = maybeAutoLink
+		p.inlineCallback['f'] = maybeAutoLink
+		p.inlineCallback['H'] = maybeAutoLink
+		p.inlineCallback['M'] = maybeAutoLink
+		p.inlineCallback['F'] = maybeAutoLink
+	}
+	if p.extensions&MathJax != 0 {
+		p.inlineCallback['$'] = math
+	}
+
+	return &p
+}
+
+func (p *Parser) getRef(refid string) (ref *reference, found bool) {
+	if p.ReferenceOverride != nil {
+		r, overridden := p.ReferenceOverride(refid)
+		if overridden {
+			if r == nil {
+				return nil, false
+			}
+			return &reference{
+				link:     []byte(r.Link),
+				title:    []byte(r.Title),
+				noteID:   0,
+				hasBlock: false,
+				text:     []byte(r.Text)}, true
+		}
+	}
+	// refs are case insensitive
+	ref, found = p.refs[strings.ToLower(refid)]
+	return ref, found
+}
+
+func (p *Parser) isFootnote(ref *reference) bool {
+	_, ok := p.refsRecord[string(ref.link)]
+	return ok
+}
+
+func (p *Parser) finalize(block ast.Node) {
+	p.tip = block.GetParent()
+}
+
+func (p *Parser) addChild(node ast.Node) ast.Node {
+	for !canNodeContain(p.tip, node) {
+		p.finalize(p.tip)
+	}
+	ast.AppendChild(p.tip, node)
+	p.tip = node
+	return node
+}
+
+func canNodeContain(n ast.Node, v ast.Node) bool {
+	switch n.(type) {
+	case *ast.List:
+		return isListItem(v)
+	case *ast.Document, *ast.BlockQuote, *ast.Aside, *ast.ListItem, *ast.CaptionFigure:
+		return !isListItem(v)
+	case *ast.Table:
+		switch v.(type) {
+		case *ast.TableHeader, *ast.TableBody, *ast.TableFooter:
+			return true
+		default:
+			return false
+		}
+	case *ast.TableHeader, *ast.TableBody, *ast.TableFooter:
+		_, ok := v.(*ast.TableRow)
+		return ok
+	case *ast.TableRow:
+		_, ok := v.(*ast.TableCell)
+		return ok
+	}
+	return false
+}
+
+func (p *Parser) closeUnmatchedBlocks() {
+	if p.allClosed {
+		return
+	}
+	for p.oldTip != p.lastMatchedContainer {
+		parent := p.oldTip.GetParent()
+		p.finalize(p.oldTip)
+		p.oldTip = parent
+	}
+	p.allClosed = true
+}
+
+// Reference represents the details of a link.
+// See the documentation in Options for more details on use-case.
+type Reference struct {
+	// Link is usually the URL the reference points to.
+	Link string
+	// Title is the alternate text describing the link in more detail.
+	Title string
+	// Text is the optional text to override the ref with if the syntax used was
+	// [refid][]
+	Text string
+}
+
+// Parse generates AST (abstract syntax tree) representing markdown document.
+//
+// The result is a root of the tree whose underlying type is *ast.Document
+//
+// You can then convert AST to html using html.Renderer, to some other format
+// using a custom renderer or transform the tree.
+func (p *Parser) Parse(input []byte) ast.Node {
+	p.block(input)
+	// Walk the tree and finish up some of unfinished blocks
+	for p.tip != nil {
+		p.finalize(p.tip)
+	}
+	// Walk the tree again and process inline markdown in each block
+	ast.WalkFunc(p.Doc, func(node ast.Node, entering bool) ast.WalkStatus {
+		switch node.(type) {
+		case *ast.Paragraph, *ast.Heading, *ast.TableCell:
+			p.Inline(node, node.AsContainer().Content)
+			node.AsContainer().Content = nil
+		}
+		return ast.GoToNext
+	})
+
+	if p.Opts.Flags&SkipFootnoteList == 0 {
+		p.parseRefsToAST()
+	}
+	return p.Doc
+}
+
+func (p *Parser) parseRefsToAST() {
+	if p.extensions&Footnotes == 0 || len(p.notes) == 0 {
+		return
+	}
+	p.tip = p.Doc
+	list := &ast.List{
+		IsFootnotesList: true,
+		ListFlags:       ast.ListTypeOrdered,
+	}
+	p.addBlock(&ast.Footnotes{})
+	block := p.addBlock(list)
+	flags := ast.ListItemBeginningOfList
+	// Note: this loop is intentionally explicit, not range-form. This is
+	// because the body of the loop will append nested footnotes to p.notes and
+	// we need to process those late additions. Range form would only walk over
+	// the fixed initial set.
+	for i := 0; i < len(p.notes); i++ {
+		ref := p.notes[i]
+		p.addChild(ref.footnote)
+		block := ref.footnote
+		listItem := block.(*ast.ListItem)
+		listItem.ListFlags = flags | ast.ListTypeOrdered
+		listItem.RefLink = ref.link
+		if ref.hasBlock {
+			flags |= ast.ListItemContainsBlock
+			p.block(ref.title)
+		} else {
+			p.Inline(block, ref.title)
+		}
+		flags &^= ast.ListItemBeginningOfList | ast.ListItemContainsBlock
+	}
+	above := list.Parent
+	finalizeList(list)
+	p.tip = above
+
+	ast.WalkFunc(block, func(node ast.Node, entering bool) ast.WalkStatus {
+		switch node.(type) {
+		case *ast.Paragraph, *ast.Heading:
+			p.Inline(node, node.AsContainer().Content)
+			node.AsContainer().Content = nil
+		}
+		return ast.GoToNext
+	})
+}
+
+//
+// Link references
+//
+// This section implements support for references that (usually) appear
+// as footnotes in a document, and can be referenced anywhere in the document.
+// The basic format is:
+//
+//    [1]: http://www.google.com/ "Google"
+//    [2]: http://www.github.com/ "Github"
+//
+// Anywhere in the document, the reference can be linked by referring to its
+// label, i.e., 1 and 2 in this example, as in:
+//
+//    This library is hosted on [Github][2], a git hosting site.
+//
+// Actual footnotes as specified in Pandoc and supported by some other Markdown
+// libraries such as php-markdown are also taken care of. They look like this:
+//
+//    This sentence needs a bit of further explanation.[^note]
+//
+//    [^note]: This is the explanation.
+//
+// Footnotes should be placed at the end of the document in an ordered list.
+// Inline footnotes such as:
+//
+//    Inline footnotes^[Not supported.] also exist.
+//
+// are not yet supported.
+
+// reference holds all information necessary for a reference-style links or
+// footnotes.
+//
+// Consider this markdown with reference-style links:
+//
+//     [link][ref]
+//
+//     [ref]: /url/ "tooltip title"
+//
+// It will be ultimately converted to this HTML:
+//
+//     <p><a href=\"/url/\" title=\"title\">link</a></p>
+//
+// And a reference structure will be populated as follows:
+//
+//     p.refs["ref"] = &reference{
+//         link: "/url/",
+//         title: "tooltip title",
+//     }
+//
+// Alternatively, reference can contain information about a footnote. Consider
+// this markdown:
+//
+//     Text needing a footnote.[^a]
+//
+//     [^a]: This is the note
+//
+// A reference structure will be populated as follows:
+//
+//     p.refs["a"] = &reference{
+//         link: "a",
+//         title: "This is the note",
+//         noteID: <some positive int>,
+//     }
+//
+// TODO: As you can see, it begs for splitting into two dedicated structures
+// for refs and for footnotes.
+type reference struct {
+	link     []byte
+	title    []byte
+	noteID   int // 0 if not a footnote ref
+	hasBlock bool
+	footnote ast.Node // a link to the Item node within a list of footnotes
+
+	text []byte // only gets populated by refOverride feature with Reference.Text
+}
+
+func (r *reference) String() string {
+	return fmt.Sprintf("{link: %q, title: %q, text: %q, noteID: %d, hasBlock: %v}",
+		r.link, r.title, r.text, r.noteID, r.hasBlock)
+}
+
+// Check whether or not data starts with a reference link.
+// If so, it is parsed and stored in the list of references
+// (in the render struct).
+// Returns the number of bytes to skip to move past it,
+// or zero if the first line is not a reference.
+func isReference(p *Parser, data []byte, tabSize int) int {
+	// up to 3 optional leading spaces
+	if len(data) < 4 {
+		return 0
+	}
+	i := 0
+	for i < 3 && data[i] == ' ' {
+		i++
+	}
+
+	noteID := 0
+
+	// id part: anything but a newline between brackets
+	if data[i] != '[' {
+		return 0
+	}
+	i++
+	if p.extensions&Footnotes != 0 {
+		if i < len(data) && data[i] == '^' {
+			// we can set it to anything here because the proper noteIds will
+			// be assigned later during the second pass. It just has to be != 0
+			noteID = 1
+			i++
+		}
+	}
+	idOffset := i
+	for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
+		i++
+	}
+	if i >= len(data) || data[i] != ']' {
+		return 0
+	}
+	idEnd := i
+	// footnotes can have empty ID, like this: [^], but a reference can not be
+	// empty like this: []. Break early if it's not a footnote and there's no ID
+	if noteID == 0 && idOffset == idEnd {
+		return 0
+	}
+	// spacer: colon (space | tab)* newline? (space | tab)*
+	i++
+	if i >= len(data) || data[i] != ':' {
+		return 0
+	}
+	i++
+	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
+		i++
+	}
+	if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
+		i++
+		if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
+			i++
+		}
+	}
+	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
+		i++
+	}
+	if i >= len(data) {
+		return 0
+	}
+
+	var (
+		linkOffset, linkEnd   int
+		titleOffset, titleEnd int
+		lineEnd               int
+		raw                   []byte
+		hasBlock              bool
+	)
+
+	if p.extensions&Footnotes != 0 && noteID != 0 {
+		linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize)
+		lineEnd = linkEnd
+	} else {
+		linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i)
+	}
+	if lineEnd == 0 {
+		return 0
+	}
+
+	// a valid ref has been found
+
+	ref := &reference{
+		noteID:   noteID,
+		hasBlock: hasBlock,
+	}
+
+	if noteID > 0 {
+		// reusing the link field for the id since footnotes don't have links
+		ref.link = data[idOffset:idEnd]
+		// if footnote, it's not really a title, it's the contained text
+		ref.title = raw
+	} else {
+		ref.link = data[linkOffset:linkEnd]
+		ref.title = data[titleOffset:titleEnd]
+	}
+
+	// id matches are case-insensitive
+	id := string(bytes.ToLower(data[idOffset:idEnd]))
+
+	p.refs[id] = ref
+
+	return lineEnd
+}
+
+func scanLinkRef(p *Parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) {
+	// link: whitespace-free sequence, optionally between angle brackets
+	if data[i] == '<' {
+		i++
+	}
+	linkOffset = i
+	for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
+		i++
+	}
+	linkEnd = i
+	if linkEnd < len(data) && data[linkOffset] == '<' && data[linkEnd-1] == '>' {
+		linkOffset++
+		linkEnd--
+	}
+
+	// optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
+	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
+		i++
+	}
+	if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
+		return
+	}
+
+	// compute end-of-line
+	if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
+		lineEnd = i
+	}
+	if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
+		lineEnd++
+	}
+
+	// optional (space|tab)* spacer after a newline
+	if lineEnd > 0 {
+		i = lineEnd + 1
+		for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
+			i++
+		}
+	}
+
+	// optional title: any non-newline sequence enclosed in '"() alone on its line
+	if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
+		i++
+		titleOffset = i
+
+		// look for EOL
+		for i < len(data) && data[i] != '\n' && data[i] != '\r' {
+			i++
+		}
+		if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
+			titleEnd = i + 1
+		} else {
+			titleEnd = i
+		}
+
+		// step back
+		i--
+		for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
+			i--
+		}
+		if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
+			lineEnd = titleEnd
+			titleEnd = i
+		}
+	}
+
+	return
+}
+
+// The first bit of this logic is the same as Parser.listItem, but the rest
+// is much simpler. This function simply finds the entire block and shifts it
+// over by one tab if it is indeed a block (just returns the line if it's not).
+// blockEnd is the end of the section in the input buffer, and contents is the
+// extracted text that was shifted over one tab. It will need to be rendered at
+// the end of the document.
+func scanFootnote(p *Parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) {
+	if i == 0 || len(data) == 0 {
+		return
+	}
+
+	// skip leading whitespace on first line
+	for i < len(data) && data[i] == ' ' {
+		i++
+	}
+
+	blockStart = i
+
+	// find the end of the line
+	blockEnd = i
+	for i < len(data) && data[i-1] != '\n' {
+		i++
+	}
+
+	// get working buffer
+	var raw bytes.Buffer
+
+	// put the first line into the working buffer
+	raw.Write(data[blockEnd:i])
+	blockEnd = i
+
+	// process the following lines
+	containsBlankLine := false
+
+gatherLines:
+	for blockEnd < len(data) {
+		i++
+
+		// find the end of this line
+		for i < len(data) && data[i-1] != '\n' {
+			i++
+		}
+
+		// if it is an empty line, guess that it is part of this item
+		// and move on to the next line
+		if p.isEmpty(data[blockEnd:i]) > 0 {
+			containsBlankLine = true
+			blockEnd = i
+			continue
+		}
+
+		n := 0
+		if n = isIndented(data[blockEnd:i], indentSize); n == 0 {
+			// this is the end of the block.
+			// we don't want to include this last line in the index.
+			break gatherLines
+		}
+
+		// if there were blank lines before this one, insert a new one now
+		if containsBlankLine {
+			raw.WriteByte('\n')
+			containsBlankLine = false
+		}
+
+		// get rid of that first tab, write to buffer
+		raw.Write(data[blockEnd+n : i])
+		hasBlock = true
+
+		blockEnd = i
+	}
+
+	if data[blockEnd-1] != '\n' {
+		raw.WriteByte('\n')
+	}
+
+	contents = raw.Bytes()
+
+	return
+}
+
+// isPunctuation returns true if c is a punctuation symbol.
+func isPunctuation(c byte) bool {
+	for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
+		if c == r {
+			return true
+		}
+	}
+	return false
+}
+
+// isSpace returns true if c is a white-space charactr
+func isSpace(c byte) bool {
+	return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
+}
+
+// isLetter returns true if c is ascii letter
+func isLetter(c byte) bool {
+	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}
+
+// isAlnum returns true if c is a digit or letter
+// TODO: check when this is looking for ASCII alnum and when it should use unicode
+func isAlnum(c byte) bool {
+	return (c >= '0' && c <= '9') || isLetter(c)
+}
+
+// TODO: this is not used
+// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
+// always ends output with a newline
+func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
+	// first, check for common cases: no tabs, or only tabs at beginning of line
+	i, prefix := 0, 0
+	slowcase := false
+	for i = 0; i < len(line); i++ {
+		if line[i] == '\t' {
+			if prefix == i {
+				prefix++
+			} else {
+				slowcase = true
+				break
+			}
+		}
+	}
+
+	// no need to decode runes if all tabs are at the beginning of the line
+	if !slowcase {
+		for i = 0; i < prefix*tabSize; i++ {
+			out.WriteByte(' ')
+		}
+		out.Write(line[prefix:])
+		return
+	}
+
+	// the slow case: we need to count runes to figure out how
+	// many spaces to insert for each tab
+	column := 0
+	i = 0
+	for i < len(line) {
+		start := i
+		for i < len(line) && line[i] != '\t' {
+			_, size := utf8.DecodeRune(line[i:])
+			i += size
+			column++
+		}
+
+		if i > start {
+			out.Write(line[start:i])
+		}
+
+		if i >= len(line) {
+			break
+		}
+
+		for {
+			out.WriteByte(' ')
+			column++
+			if column%tabSize == 0 {
+				break
+			}
+		}
+
+		i++
+	}
+}
+
+// Find if a line counts as indented or not.
+// Returns number of characters the indent is (0 = not indented).
+func isIndented(data []byte, indentSize int) int {
+	if len(data) == 0 {
+		return 0
+	}
+	if data[0] == '\t' {
+		return 1
+	}
+	if len(data) < indentSize {
+		return 0
+	}
+	for i := 0; i < indentSize; i++ {
+		if data[i] != ' ' {
+			return 0
+		}
+	}
+	return indentSize
+}
+
+// Create a url-safe slug for fragments
+func slugify(in []byte) []byte {
+	if len(in) == 0 {
+		return in
+	}
+	out := make([]byte, 0, len(in))
+	sym := false
+
+	for _, ch := range in {
+		if isAlnum(ch) {
+			sym = false
+			out = append(out, ch)
+		} else if sym {
+			continue
+		} else {
+			out = append(out, '-')
+			sym = true
+		}
+	}
+	var a, b int
+	var ch byte
+	for a, ch = range out {
+		if ch != '-' {
+			break
+		}
+	}
+	for b = len(out) - 1; b > 0; b-- {
+		if out[b] != '-' {
+			break
+		}
+	}
+	return out[a : b+1]
+}
+
+func isListItem(d ast.Node) bool {
+	_, ok := d.(*ast.ListItem)
+	return ok
+}

vendor/github.com/gomarkdown/markdown/parser/ref.go

@@ -0,0 +1,89 @@
+package parser
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/gomarkdown/markdown/ast"
+)
+
+// parse '(#r)', where r does not contain spaces. Or.
+// (!item) (!item, subitem), for an index, (!!item) signals primary.
+func maybeShortRefOrIndex(p *Parser, data []byte, offset int) (int, ast.Node) {
+	if len(data[offset:]) < 4 {
+		return 0, nil
+	}
+	// short ref first
+	data = data[offset:]
+	i := 1
+	switch data[i] {
+	case '#': // cross ref
+		i++
+	Loop:
+		for i < len(data) {
+			c := data[i]
+			switch {
+			case c == ')':
+				break Loop
+			case !isAlnum(c):
+				if c == '_' || c == '-' || c == ':' {
+					i++
+					continue
+				}
+				i = 0
+				break Loop
+			}
+			i++
+		}
+		if i >= len(data) {
+			return 0, nil
+		}
+		if data[i] != ')' {
+			return 0, nil
+		}
+
+		id := data[2:i]
+		node := &ast.CrossReference{}
+		node.Destination = id
+
+		return i + 1, node
+
+	case '!': // index
+		i++
+		start := i
+		i = skipUntilChar(data, start, ')')
+
+		// did we reach the end of the buffer without a closing marker?
+		if i >= len(data) {
+			return 0, nil
+		}
+
+		if len(data[start:i]) < 1 {
+			return 0, nil
+		}
+
+		idx := &ast.Index{}
+
+		idx.ID = fmt.Sprintf("idxref:%d", p.indexCnt)
+		p.indexCnt++
+
+		idx.Primary = data[start] == '!'
+		buf := data[start:i]
+
+		if idx.Primary {
+			buf = buf[1:]
+		}
+		items := bytes.Split(buf, []byte(","))
+		switch len(items) {
+		case 1:
+			idx.Item = bytes.TrimSpace(items[0])
+			return i + 1, idx
+		case 2:
+			idx.Item = bytes.TrimSpace(items[0])
+			idx.Subitem = bytes.TrimSpace(items[1])
+			return i + 1, idx
+		}
+	}
+
+	return 0, nil
+}