Add support for markdown to HTML conversion (matrix). Closes #663 (#670)

This uses our own gomatrix lib with the SendHTML function which
adds HTML to formatted_body in matrix.
golang-commonmark is used to convert markdown into valid HTML.

vendor/gitlab.com/golang-commonmark/mdurl/.gitlab-ci.yml

@@ -0,0 +1,16 @@
+image: golang:1.11
+
+stages:
+  - build
+  - test
+
+build:
+  stage: build
+  script:
+    - go build ./...
+
+test:
+  stage: test
+  script:
+    - test -z "$(gofmt -l . | tee /dev/stderr)"
+    - go test ./...

vendor/gitlab.com/golang-commonmark/mdurl/LICENSE

@@ -0,0 +1,10 @@
+Copyright (c) 2015, The Authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/gitlab.com/golang-commonmark/mdurl/README.md

@@ -0,0 +1,8 @@
+mdurl [![License](https://img.shields.io/badge/licence-BSD--2--Clause-blue.svg)](https://opensource.org/licenses/BSD-2-Clause) [![GoDoc](http://godoc.org/gitlab.com/golang-commonmark/mdurl?status.svg)](http://godoc.org/gitlab.com/golang-commonmark/mdurl) [![Pipeline status](https://gitlab.com/golang-commonmark/mdurl/badges/master/pipeline.svg)](https://gitlab.com/golang-commonmark/mdurl/commits/master)
+=====
+
+Package mdurl provides functions for parsing, decoding and encoding URLs.
+
+## Install
+
+    go get -u gitlab.com/golang-commonmark/mdurl

vendor/gitlab.com/golang-commonmark/mdurl/decode.go

@@ -0,0 +1,80 @@
+// Copyright 2015 The Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mdurl
+
+import (
+	"bytes"
+	"unicode/utf8"
+)
+
+func advance(s string, pos int) (byte, int) {
+	if pos >= len(s) {
+		return 0, len(s) + 1
+	}
+	if s[pos] != '%' {
+		return s[pos], pos + 1
+	}
+	if pos+2 < len(s) &&
+		hexDigit(s[pos+1]) &&
+		hexDigit(s[pos+2]) {
+		return unhex(s[pos+1])<<4 | unhex(s[pos+2]), pos + 3
+	}
+	return '%', pos + 1
+}
+
+// Decode decodes a percent-encoded URL.
+// Invalid percent-encoded sequences are left as is.
+// Invalid UTF-8 sequences are replaced with U+FFFD.
+func Decode(rawurl string) string {
+	var buf bytes.Buffer
+	i := 0
+	const replacement = "\xEF\xBF\xBD"
+outer:
+	for i < len(rawurl) {
+		r, rlen := utf8.DecodeRuneInString(rawurl[i:])
+		if r == '%' && i+2 < len(rawurl) &&
+			hexDigit(rawurl[i+1]) &&
+			hexDigit(rawurl[i+2]) {
+			b := unhex(rawurl[i+1])<<4 | unhex(rawurl[i+2])
+			if b < 0x80 {
+				buf.WriteByte(b)
+				i += 3
+				continue
+			}
+			var n int
+			if b&0xe0 == 0xc0 {
+				n = 1
+			} else if b&0xf0 == 0xe0 {
+				n = 2
+			} else if b&0xf8 == 0xf0 {
+				n = 3
+			}
+			if n == 0 {
+				buf.WriteString(replacement)
+				i += 3
+				continue
+			}
+			rb := make([]byte, n+1)
+			rb[0] = b
+			j := i + 3
+			for k := 0; k < n; k++ {
+				b, j = advance(rawurl, j)
+				if j > len(rawurl) || b&0xc0 != 0x80 {
+					buf.WriteString(replacement)
+					i += 3
+					continue outer
+				}
+				rb[k+1] = b
+			}
+			r, _ := utf8.DecodeRune(rb)
+			buf.WriteRune(r)
+			i = j
+			continue
+		}
+		buf.WriteRune(r)
+		i += rlen
+	}
+	return buf.String()
+}

vendor/gitlab.com/golang-commonmark/mdurl/encode.go

@@ -0,0 +1,53 @@
+// Copyright 2015 The Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mdurl
+
+import (
+	"bytes"
+	"strings"
+	"unicode/utf8"
+)
+
+// Encode percent-encodes rawurl, avoiding double encoding.
+// It doesn't touch:
+// - alphanumeric characters ([0-9a-zA-Z]);
+// - percent-encoded characters (%[0-9a-fA-F]{2});
+// - excluded characters ([;/?:@&=+$,-_.!~*'()#]).
+// Invalid UTF-8 sequences are replaced with U+FFFD.
+func Encode(rawurl string) string {
+	const hexdigit = "0123456789ABCDEF"
+	var buf bytes.Buffer
+	i := 0
+	for i < len(rawurl) {
+		r, rlen := utf8.DecodeRuneInString(rawurl[i:])
+		if r >= 0x80 {
+			for j, n := i, i+rlen; j < n; j++ {
+				b := rawurl[j]
+				buf.WriteByte('%')
+				buf.WriteByte(hexdigit[(b>>4)&0xf])
+				buf.WriteByte(hexdigit[b&0xf])
+			}
+		} else if r == '%' {
+			if i+2 < len(rawurl) &&
+				hexDigit(rawurl[i+1]) &&
+				hexDigit(rawurl[i+2]) {
+				buf.WriteByte('%')
+				buf.WriteByte(byteToUpper(rawurl[i+1]))
+				buf.WriteByte(byteToUpper(rawurl[i+2]))
+				i += 2
+			} else {
+				buf.WriteString("%25")
+			}
+		} else if strings.IndexByte("!#$&'()*+,-./0123456789:;=?@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~", byte(r)) == -1 {
+			buf.WriteByte('%')
+			buf.WriteByte(hexdigit[(r>>4)&0xf])
+			buf.WriteByte(hexdigit[r&0xf])
+		} else {
+			buf.WriteByte(byte(r))
+		}
+		i += rlen
+	}
+	return buf.String()
+}

vendor/gitlab.com/golang-commonmark/mdurl/parse.go

@@ -0,0 +1,146 @@
+// Copyright 2015 The Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mdurl
+
+import (
+	"errors"
+	"strings"
+)
+
+// ErrMissingScheme error is returned by Parse if the passed URL starts with a colon.
+var ErrMissingScheme = errors.New("missing protocol scheme")
+
+var slashedProtocol = map[string]bool{
+	"http":   true,
+	"https":  true,
+	"ftp":    true,
+	"gopher": true,
+	"file":   true,
+}
+
+func split(s string, c byte) (string, string, bool) {
+	i := strings.IndexByte(s, c)
+	if i < 0 {
+		return s, "", false
+	}
+	return s[:i], s[i+1:], true
+}
+
+func findScheme(s string) (int, error) {
+	if s == "" {
+		return 0, nil
+	}
+
+	b := s[0]
+	if b == ':' {
+		return 0, ErrMissingScheme
+	}
+	if !letter(b) {
+		return 0, nil
+	}
+
+	for i := 1; i < len(s); i++ {
+		b := s[i]
+		switch {
+		case b == ':':
+			return i, nil
+		case strings.IndexByte("+-.0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", b) != -1:
+			// do nothing
+		default:
+			return 0, nil
+		}
+	}
+
+	return 0, nil
+}
+
+// Parse parses rawurl into a URL structure.
+func Parse(rawurl string) (*URL, error) {
+	n, err := findScheme(rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	var url URL
+	rest := rawurl
+	hostless := false
+	if n > 0 {
+		url.RawScheme = rest[:n]
+		url.Scheme, rest = strings.ToLower(rest[:n]), rest[n+1:]
+		if url.Scheme == "javascript" {
+			hostless = true
+		}
+	}
+
+	if !hostless && strings.HasPrefix(rest, "//") {
+		url.Slashes, rest = true, rest[2:]
+	}
+
+	if !hostless && (url.Slashes || (url.Scheme != "" && !slashedProtocol[url.Scheme])) {
+		hostEnd := strings.IndexAny(rest, "#/?")
+		atSign := -1
+		i := hostEnd
+		if i == -1 {
+			i = len(rest) - 1
+		}
+		for i >= 0 {
+			if rest[i] == '@' {
+				atSign = i
+				break
+			}
+			i--
+		}
+
+		if atSign != -1 {
+			url.Auth, rest = rest[:atSign], rest[atSign+1:]
+		}
+
+		hostEnd = strings.IndexAny(rest, "\t\r\n \"#%'/;<>?\\^`{|}")
+		if hostEnd == -1 {
+			hostEnd = len(rest)
+		}
+		if hostEnd > 0 && hostEnd < len(rest) && rest[hostEnd-1] == ':' {
+			hostEnd--
+		}
+		host := rest[:hostEnd]
+
+		if len(host) > 1 {
+			b := host[hostEnd-1]
+			if digit(b) {
+				for i := len(host) - 2; i >= 0; i-- {
+					b := host[i]
+					if b == ':' {
+						url.Host, url.Port = host[:i], host[i+1:]
+						break
+					}
+					if !digit(b) {
+						break
+					}
+				}
+			} else if b == ':' {
+				host = host[:hostEnd-1]
+				hostEnd--
+			}
+		}
+		if url.Port == "" {
+			url.Host = host
+		}
+		rest = rest[hostEnd:]
+
+		if ipv6 := len(url.Host) > 2 &&
+			url.Host[0] == '[' &&
+			url.Host[len(url.Host)-1] == ']'; ipv6 {
+			url.Host = url.Host[1 : len(url.Host)-1]
+			url.IPv6 = true
+		} else if i := strings.IndexByte(url.Host, ':'); i >= 0 {
+			url.Host, rest = url.Host[:i], url.Host[i:]+rest
+		}
+	}
+
+	rest, url.Fragment, url.HasFragment = split(rest, '#')
+	url.Path, url.RawQuery, url.HasQuery = split(rest, '?')
+
+	return &url, nil
+}

vendor/gitlab.com/golang-commonmark/mdurl/url.go

@@ -0,0 +1,101 @@
+// Copyright 2015 The Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package url provides functions for parsing, decoding and encoding URLs.
+package mdurl
+
+// A URL represents a parsed URL.
+type URL struct {
+	Scheme      string
+	RawScheme   string
+	Slashes     bool
+	Auth        string
+	Host        string
+	Port        string
+	Path        string
+	RawQuery    string
+	HasQuery    bool
+	Fragment    string
+	HasFragment bool
+	IPv6        bool
+}
+
+// String reassembles the URL into a URL string.
+func (u *URL) String() string {
+	size := len(u.Path)
+	if u.Scheme != "" {
+		size += len(u.Scheme) + 1
+	}
+	if u.Slashes {
+		size += 2
+	}
+	if u.Auth != "" {
+		size += len(u.Auth) + 1
+	}
+	if u.Host != "" {
+		size += len(u.Host)
+		if u.IPv6 {
+			size += 2
+		}
+	}
+	if u.Port != "" {
+		size += len(u.Port) + 1
+	}
+	if u.HasQuery {
+		size += len(u.RawQuery) + 1
+	}
+	if u.HasFragment {
+		size += len(u.Fragment) + 1
+	}
+	if size == 0 {
+		return ""
+	}
+
+	buf := make([]byte, size)
+	i := 0
+	if u.Scheme != "" {
+		i += copy(buf, u.Scheme)
+		buf[i] = ':'
+		i++
+	}
+	if u.Slashes {
+		buf[i] = '/'
+		i++
+		buf[i] = '/'
+		i++
+	}
+	if u.Auth != "" {
+		i += copy(buf[i:], u.Auth)
+		buf[i] = '@'
+		i++
+	}
+	if u.Host != "" {
+		if u.IPv6 {
+			buf[i] = '['
+			i++
+			i += copy(buf[i:], u.Host)
+			buf[i] = ']'
+			i++
+		} else {
+			i += copy(buf[i:], u.Host)
+		}
+	}
+	if u.Port != "" {
+		buf[i] = ':'
+		i++
+		i += copy(buf[i:], u.Port)
+	}
+	i += copy(buf[i:], u.Path)
+	if u.HasQuery {
+		buf[i] = '?'
+		i++
+		i += copy(buf[i:], u.RawQuery)
+	}
+	if u.HasFragment {
+		buf[i] = '#'
+		i++
+		i += copy(buf[i:], u.Fragment)
+	}
+	return string(buf)
+}

vendor/gitlab.com/golang-commonmark/mdurl/util.go

@@ -0,0 +1,36 @@
+// Copyright 2015 The Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mdurl
+
+func hexDigit(b byte) bool {
+	return digit(b) || b >= 'a' && b <= 'f' || b >= 'A' && b <= 'F'
+}
+
+func unhex(b byte) byte {
+	switch {
+	case digit(b):
+		return b - '0'
+	case b >= 'a' && b <= 'f':
+		return b - 'a' + 10
+	case b >= 'A' && b <= 'F':
+		return b - 'A' + 10
+	}
+	panic("unhex: not a hex digit")
+}
+
+func letter(b byte) bool {
+	return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z'
+}
+
+func digit(b byte) bool {
+	return b >= '0' && b <= '9'
+}
+
+func byteToUpper(b byte) byte {
+	if b >= 'a' && b <= 'z' {
+		return b - 'a' + 'A'
+	}
+	return b
+}