s/vgo/dep/

This commit is contained in:
Jesse Hallam 2018-07-26 16:25:25 -04:00
parent f387133c21
commit 8788b41ab2
No known key found for this signature in database
GPG key ID: E7959EB6518AF966
321 changed files with 55322 additions and 145 deletions

View file

@ -0,0 +1,56 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package jsonutils
import (
"bytes"
"encoding/json"
"github.com/pkg/errors"
)
type HumanizedJsonError struct {
Err error
Line int
Character int
}
func (e *HumanizedJsonError) Error() string {
return e.Err.Error()
}
// HumanizeJsonError extracts error offsets and annotates the error with useful context
func HumanizeJsonError(err error, data []byte) error {
if syntaxError, ok := err.(*json.SyntaxError); ok {
return NewHumanizedJsonError(syntaxError, data, syntaxError.Offset)
} else if unmarshalError, ok := err.(*json.UnmarshalTypeError); ok {
return NewHumanizedJsonError(unmarshalError, data, unmarshalError.Offset)
} else {
return err
}
}
func NewHumanizedJsonError(err error, data []byte, offset int64) *HumanizedJsonError {
if err == nil {
return nil
}
if offset < 0 || offset > int64(len(data)) {
return &HumanizedJsonError{
Err: errors.Wrapf(err, "invalid offset %d", offset),
}
}
lineSep := []byte{'\n'}
line := bytes.Count(data[:offset], lineSep) + 1
lastLineOffset := bytes.LastIndex(data[:offset], lineSep)
character := int(offset) - (lastLineOffset + 1) + 1
return &HumanizedJsonError{
Line: line,
Character: character,
Err: errors.Wrapf(err, "parsing error at line %d, character %d", line, character),
}
}

View file

@ -0,0 +1,157 @@
// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package utils
import (
"crypto"
"crypto/rsa"
"crypto/sha512"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
var publicKey []byte = []byte(`-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyZmShlU8Z8HdG0IWSZ8r
tSyzyxrXkJjsFUf0Ke7bm/TLtIggRdqOcUF3XEWqQk5RGD5vuq7Rlg1zZqMEBk8N
EZeRhkxyaZW8pLjxwuBUOnXfJew31+gsTNdKZzRjrvPumKr3EtkleuoxNdoatu4E
HrKmR/4Yi71EqAvkhk7ZjQFuF0osSWJMEEGGCSUYQnTEqUzcZSh1BhVpkIkeu8Kk
1wCtptODixvEujgqVe+SrE3UlZjBmPjC/CL+3cYmufpSNgcEJm2mwsdaXp2OPpfn
a0v85XL6i9ote2P+fLZ3wX9EoioHzgdgB7arOxY50QRJO7OyCqpKFKv6lRWTXuSt
hwIDAQAB
-----END PUBLIC KEY-----`)
func ValidateLicense(signed []byte) (bool, string) {
decoded := make([]byte, base64.StdEncoding.DecodedLen(len(signed)))
_, err := base64.StdEncoding.Decode(decoded, signed)
if err != nil {
mlog.Error(fmt.Sprintf("Encountered error decoding license, err=%v", err.Error()))
return false, ""
}
if len(decoded) <= 256 {
mlog.Error("Signed license not long enough")
return false, ""
}
// remove null terminator
for decoded[len(decoded)-1] == byte(0) {
decoded = decoded[:len(decoded)-1]
}
plaintext := decoded[:len(decoded)-256]
signature := decoded[len(decoded)-256:]
block, _ := pem.Decode(publicKey)
public, err := x509.ParsePKIXPublicKey(block.Bytes)
if err != nil {
mlog.Error(fmt.Sprintf("Encountered error signing license, err=%v", err.Error()))
return false, ""
}
rsaPublic := public.(*rsa.PublicKey)
h := sha512.New()
h.Write(plaintext)
d := h.Sum(nil)
err = rsa.VerifyPKCS1v15(rsaPublic, crypto.SHA512, d, signature)
if err != nil {
mlog.Error(fmt.Sprintf("Invalid signature, err=%v", err.Error()))
return false, ""
}
return true, string(plaintext)
}
func GetAndValidateLicenseFileFromDisk(location string) (*model.License, []byte) {
fileName := GetLicenseFileLocation(location)
if _, err := os.Stat(fileName); err != nil {
mlog.Debug(fmt.Sprintf("We could not find the license key in the database or on disk at %v", fileName))
return nil, nil
}
mlog.Info(fmt.Sprintf("License key has not been uploaded. Loading license key from disk at %v", fileName))
licenseBytes := GetLicenseFileFromDisk(fileName)
if success, licenseStr := ValidateLicense(licenseBytes); !success {
mlog.Error(fmt.Sprintf("Found license key at %v but it appears to be invalid.", fileName))
return nil, nil
} else {
return model.LicenseFromJson(strings.NewReader(licenseStr)), licenseBytes
}
}
func GetLicenseFileFromDisk(fileName string) []byte {
file, err := os.Open(fileName)
if err != nil {
mlog.Error(fmt.Sprintf("Failed to open license key from disk at %v err=%v", fileName, err.Error()))
return nil
}
defer file.Close()
licenseBytes, err := ioutil.ReadAll(file)
if err != nil {
mlog.Error(fmt.Sprintf("Failed to read license key from disk at %v err=%v", fileName, err.Error()))
return nil
}
return licenseBytes
}
func GetLicenseFileLocation(fileLocation string) string {
if fileLocation == "" {
configDir, _ := FindDir("config")
return filepath.Join(configDir, "mattermost.mattermost-license")
} else {
return fileLocation
}
}
func GetClientLicense(l *model.License) map[string]string {
props := make(map[string]string)
props["IsLicensed"] = strconv.FormatBool(l != nil)
if l != nil {
props["Id"] = l.Id
props["Users"] = strconv.Itoa(*l.Features.Users)
props["LDAP"] = strconv.FormatBool(*l.Features.LDAP)
props["MFA"] = strconv.FormatBool(*l.Features.MFA)
props["SAML"] = strconv.FormatBool(*l.Features.SAML)
props["Cluster"] = strconv.FormatBool(*l.Features.Cluster)
props["Metrics"] = strconv.FormatBool(*l.Features.Metrics)
props["GoogleOAuth"] = strconv.FormatBool(*l.Features.GoogleOAuth)
props["Office365OAuth"] = strconv.FormatBool(*l.Features.Office365OAuth)
props["Compliance"] = strconv.FormatBool(*l.Features.Compliance)
props["MHPNS"] = strconv.FormatBool(*l.Features.MHPNS)
props["Announcement"] = strconv.FormatBool(*l.Features.Announcement)
props["Elasticsearch"] = strconv.FormatBool(*l.Features.Elasticsearch)
props["DataRetention"] = strconv.FormatBool(*l.Features.DataRetention)
props["IssuedAt"] = strconv.FormatInt(l.IssuedAt, 10)
props["StartsAt"] = strconv.FormatInt(l.StartsAt, 10)
props["ExpiresAt"] = strconv.FormatInt(l.ExpiresAt, 10)
props["Name"] = l.Customer.Name
props["Email"] = l.Customer.Email
props["Company"] = l.Customer.Company
props["PhoneNumber"] = l.Customer.PhoneNumber
props["EmailNotificationContents"] = strconv.FormatBool(*l.Features.EmailNotificationContents)
props["MessageExport"] = strconv.FormatBool(*l.Features.MessageExport)
props["CustomPermissionsSchemes"] = strconv.FormatBool(*l.Features.CustomPermissionsSchemes)
}
return props
}

View file

@ -0,0 +1,253 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
import (
"regexp"
"strings"
"unicode"
"unicode/utf8"
)
// Based off of extensions/autolink.c from https://github.com/github/cmark
var (
DefaultUrlSchemes = []string{"http", "https", "ftp", "mailto", "tel"}
)
// Given a string with a w at the given position, tries to parse and return a link starting with "www."
// if one exists. If the text at the given position isn't a link, returns an empty string. Equivalent to
// www_match from the reference code.
func parseWWWAutolink(data string, position int) string {
// Check that this isn't part of another word
if position > 1 {
prevChar := data[position-1]
if !isWhitespaceByte(prevChar) && !isAllowedBeforeWWWLink(prevChar) {
return ""
}
}
// Check that this starts with www
if len(data)-position < 4 || !regexp.MustCompile(`^www\d{0,3}\.`).MatchString(data[position:]) {
return ""
}
end := checkDomain(data[position:], false)
if end == 0 {
return ""
}
end += position
// Grab all text until the end of the string or the next whitespace character
for end < len(data) && !isWhitespaceByte(data[end]) {
end += 1
}
// Trim trailing punctuation
link := trimTrailingCharactersFromLink(data[position:end])
if link == "" {
return ""
}
return link
}
func isAllowedBeforeWWWLink(c byte) bool {
switch c {
case '*', '_', '~', ')':
return true
default:
return false
}
}
// Given a string with a : at the given position, tried to parse and return a link starting with a URL scheme
// if one exists. If the text around the given position isn't a link, returns an empty string. Equivalent to
// url_match from the reference code.
func parseURLAutolink(data string, position int) string {
// Check that a :// exists. This doesn't match the clients that treat the slashes as optional.
if len(data)-position < 4 || data[position+1] != '/' || data[position+2] != '/' {
return ""
}
start := position - 1
for start > 0 && isAlphanumericByte(data[start-1]) {
start -= 1
}
// Ensure that the URL scheme is allowed and that at least one character after the scheme is valid.
scheme := data[start:position]
if !isSchemeAllowed(scheme) || !isValidHostCharacter(data[position+3:]) {
return ""
}
end := checkDomain(data[position+3:], true)
if end == 0 {
return ""
}
end += position
// Grab all text until the end of the string or the next whitespace character
for end < len(data) && !isWhitespaceByte(data[end]) {
end += 1
}
// Trim trailing punctuation
link := trimTrailingCharactersFromLink(data[start:end])
if link == "" {
return ""
}
return link
}
func isSchemeAllowed(scheme string) bool {
// Note that this doesn't support the custom URL schemes implemented by the client
for _, allowed := range DefaultUrlSchemes {
if strings.EqualFold(allowed, scheme) {
return true
}
}
return false
}
// Given a string starting with a URL, returns the number of valid characters that make up the URL's domain.
// Returns 0 if the string doesn't start with a domain name. allowShort determines whether or not the domain
// needs to contain a period to be considered valid. Equivalent to check_domain from the reference code.
func checkDomain(data string, allowShort bool) int {
foundUnderscore := false
foundPeriod := false
i := 1
for ; i < len(data)-1; i++ {
if data[i] == '_' {
foundUnderscore = true
break
} else if data[i] == '.' {
foundPeriod = true
} else if !isValidHostCharacter(data[i:]) && data[i] != '-' {
break
}
}
if foundUnderscore {
return 0
}
if allowShort {
// If allowShort is set, accept any string of valid domain characters
return i
}
// If allowShort isn't set, a valid domain just requires at least a single period. Note that this
// logic isn't entirely necessary because we already know the string starts with "www." when
// this is called from parseWWWAutolink
if foundPeriod {
return i
} else {
return 0
}
}
// Returns true if the provided link starts with a valid character for a domain name. Equivalent to
// is_valid_hostchar from the reference code.
func isValidHostCharacter(link string) bool {
c, _ := utf8.DecodeRuneInString(link)
if c == utf8.RuneError {
return false
}
return !unicode.IsSpace(c) && !unicode.IsPunct(c)
}
// Removes any trailing characters such as punctuation or stray brackets that shouldn't be part of the link.
// Equivalent to autolink_delim from the reference code.
func trimTrailingCharactersFromLink(link string) string {
runes := []rune(link)
linkEnd := len(runes)
// Cut off the link before an open angle bracket if it contains one
for i, c := range runes {
if c == '<' {
linkEnd = i
break
}
}
for linkEnd > 0 {
c := runes[linkEnd-1]
if !canEndAutolink(c) {
// Trim trailing quotes, periods, etc
linkEnd = linkEnd - 1
} else if c == ';' {
// Trim a trailing HTML entity
newEnd := linkEnd - 2
for newEnd > 0 && ((runes[newEnd] >= 'a' && runes[newEnd] <= 'z') || (runes[newEnd] >= 'A' && runes[newEnd] <= 'Z')) {
newEnd -= 1
}
if newEnd < linkEnd-2 && runes[newEnd] == '&' {
linkEnd = newEnd
} else {
// This isn't actually an HTML entity, so just trim the semicolon
linkEnd = linkEnd - 1
}
} else if c == ')' {
// Only allow an autolink ending with a bracket if that bracket is part of a matching pair of brackets.
// If there are more closing brackets than opening ones, remove the extra bracket
numClosing := 0
numOpening := 0
// Examples (input text => output linked portion):
//
// http://www.pokemon.com/Pikachu_(Electric)
// => http://www.pokemon.com/Pikachu_(Electric)
//
// http://www.pokemon.com/Pikachu_((Electric)
// => http://www.pokemon.com/Pikachu_((Electric)
//
// http://www.pokemon.com/Pikachu_(Electric))
// => http://www.pokemon.com/Pikachu_(Electric)
//
// http://www.pokemon.com/Pikachu_((Electric))
// => http://www.pokemon.com/Pikachu_((Electric))
for i := 0; i < linkEnd; i++ {
if runes[i] == '(' {
numOpening += 1
} else if runes[i] == ')' {
numClosing += 1
}
}
if numClosing <= numOpening {
// There's fewer or equal closing brackets, so we've found the end of the link
break
}
linkEnd -= 1
} else {
// There's no special characters at the end of the link, so we're at the end
break
}
}
return string(runes[:linkEnd])
}
func canEndAutolink(c rune) bool {
switch c {
case '?', '!', '.', ',', ':', '*', '_', '~', '\'', '"':
return false
default:
return true
}
}

View file

@ -0,0 +1,62 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
type BlockQuote struct {
blockBase
markdown string
Children []Block
}
func (b *BlockQuote) Continuation(indentation int, r Range) *continuation {
if indentation > 3 {
return nil
}
s := b.markdown[r.Position:r.End]
if s == "" || s[0] != '>' {
return nil
}
remaining := Range{r.Position + 1, r.End}
indentation, indentationBytes := countIndentation(b.markdown, remaining)
if indentation > 0 {
indentation--
}
return &continuation{
Indentation: indentation,
Remaining: Range{remaining.Position + indentationBytes, remaining.End},
}
}
func (b *BlockQuote) AddChild(openBlocks []Block) []Block {
b.Children = append(b.Children, openBlocks[0])
return openBlocks
}
func blockQuoteStart(markdown string, indent int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
if indent > 3 {
return nil
}
s := markdown[r.Position:r.End]
if s == "" || s[0] != '>' {
return nil
}
block := &BlockQuote{
markdown: markdown,
}
r.Position++
if len(s) > 1 && s[1] == ' ' {
r.Position++
}
indent, bytes := countIndentation(markdown, r)
ret := []Block{block}
if descendants := blockStartOrParagraph(markdown, indent, Range{r.Position + bytes, r.End}, nil, nil); descendants != nil {
block.Children = append(block.Children, descendants[0])
ret = append(ret, descendants...)
}
return ret
}

View file

@ -0,0 +1,153 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
import (
"strings"
)
type continuation struct {
Indentation int
Remaining Range
}
type Block interface {
Continuation(indentation int, r Range) *continuation
AddLine(indentation int, r Range) bool
Close()
AllowsBlockStarts() bool
HasTrailingBlankLine() bool
}
type blockBase struct{}
func (*blockBase) AddLine(indentation int, r Range) bool { return false }
func (*blockBase) Close() {}
func (*blockBase) AllowsBlockStarts() bool { return true }
func (*blockBase) HasTrailingBlankLine() bool { return false }
type ContainerBlock interface {
Block
AddChild(openBlocks []Block) []Block
}
type Range struct {
Position int
End int
}
func closeBlocks(blocks []Block, referenceDefinitions *[]*ReferenceDefinition) {
for _, block := range blocks {
block.Close()
if p, ok := block.(*Paragraph); ok && len(p.ReferenceDefinitions) > 0 {
*referenceDefinitions = append(*referenceDefinitions, p.ReferenceDefinitions...)
}
}
}
func ParseBlocks(markdown string, lines []Line) (*Document, []*ReferenceDefinition) {
document := &Document{}
var referenceDefinitions []*ReferenceDefinition
openBlocks := []Block{document}
for _, line := range lines {
r := line.Range
lastMatchIndex := 0
indentation, indentationBytes := countIndentation(markdown, r)
r = Range{r.Position + indentationBytes, r.End}
for i, block := range openBlocks {
if continuation := block.Continuation(indentation, r); continuation != nil {
indentation = continuation.Indentation
r = continuation.Remaining
additionalIndentation, additionalIndentationBytes := countIndentation(markdown, r)
r = Range{r.Position + additionalIndentationBytes, r.End}
indentation += additionalIndentation
lastMatchIndex = i
} else {
break
}
}
if openBlocks[lastMatchIndex].AllowsBlockStarts() {
if newBlocks := blockStart(markdown, indentation, r, openBlocks[:lastMatchIndex+1], openBlocks[lastMatchIndex+1:]); newBlocks != nil {
didAdd := false
for i := lastMatchIndex; i >= 0; i-- {
if container, ok := openBlocks[i].(ContainerBlock); ok {
if newBlocks := container.AddChild(newBlocks); newBlocks != nil {
closeBlocks(openBlocks[i+1:], &referenceDefinitions)
openBlocks = openBlocks[:i+1]
openBlocks = append(openBlocks, newBlocks...)
didAdd = true
break
}
}
}
if didAdd {
continue
}
}
}
isBlank := strings.TrimSpace(markdown[r.Position:r.End]) == ""
if paragraph, ok := openBlocks[len(openBlocks)-1].(*Paragraph); ok && !isBlank {
paragraph.Text = append(paragraph.Text, r)
continue
}
closeBlocks(openBlocks[lastMatchIndex+1:], &referenceDefinitions)
openBlocks = openBlocks[:lastMatchIndex+1]
if openBlocks[lastMatchIndex].AddLine(indentation, r) {
continue
}
if paragraph := newParagraph(markdown, r); paragraph != nil {
for i := lastMatchIndex; i >= 0; i-- {
if container, ok := openBlocks[i].(ContainerBlock); ok {
if newBlocks := container.AddChild([]Block{paragraph}); newBlocks != nil {
closeBlocks(openBlocks[i+1:], &referenceDefinitions)
openBlocks = openBlocks[:i+1]
openBlocks = append(openBlocks, newBlocks...)
break
}
}
}
}
}
closeBlocks(openBlocks, &referenceDefinitions)
return document, referenceDefinitions
}
func blockStart(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
if r.Position >= r.End {
return nil
}
if start := blockQuoteStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
return start
} else if start := listStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
return start
} else if start := indentedCodeStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
return start
} else if start := fencedCodeStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
return start
}
return nil
}
func blockStartOrParagraph(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
if start := blockStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
return start
}
if paragraph := newParagraph(markdown, r); paragraph != nil {
return []Block{paragraph}
}
return nil
}

View file

@ -0,0 +1,22 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
type Document struct {
blockBase
Children []Block
}
func (b *Document) Continuation(indentation int, r Range) *continuation {
return &continuation{
Indentation: indentation,
Remaining: r,
}
}
func (b *Document) AddChild(openBlocks []Block) []Block {
b.Children = append(b.Children, openBlocks[0])
return openBlocks
}

View file

@ -0,0 +1,112 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
import (
"strings"
)
type FencedCodeLine struct {
Indentation int
Range Range
}
type FencedCode struct {
blockBase
markdown string
didSeeClosingFence bool
Indentation int
OpeningFence Range
RawInfo Range
RawCode []FencedCodeLine
}
func (b *FencedCode) Code() (result string) {
for _, code := range b.RawCode {
result += strings.Repeat(" ", code.Indentation) + b.markdown[code.Range.Position:code.Range.End]
}
return
}
func (b *FencedCode) Info() string {
return Unescape(b.markdown[b.RawInfo.Position:b.RawInfo.End])
}
func (b *FencedCode) Continuation(indentation int, r Range) *continuation {
if b.didSeeClosingFence {
return nil
}
return &continuation{
Indentation: indentation,
Remaining: r,
}
}
func (b *FencedCode) AddLine(indentation int, r Range) bool {
s := b.markdown[r.Position:r.End]
if indentation <= 3 && strings.HasPrefix(s, b.markdown[b.OpeningFence.Position:b.OpeningFence.End]) {
suffix := strings.TrimSpace(s[b.OpeningFence.End-b.OpeningFence.Position:])
isClosingFence := true
for _, c := range suffix {
if c != rune(s[0]) {
isClosingFence = false
break
}
}
if isClosingFence {
b.didSeeClosingFence = true
return true
}
}
if indentation >= b.Indentation {
indentation -= b.Indentation
} else {
indentation = 0
}
b.RawCode = append(b.RawCode, FencedCodeLine{
Indentation: indentation,
Range: r,
})
return true
}
func (b *FencedCode) AllowsBlockStarts() bool {
return false
}
func fencedCodeStart(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
s := markdown[r.Position:r.End]
if !strings.HasPrefix(s, "```") && !strings.HasPrefix(s, "~~~") {
return nil
}
fenceCharacter := rune(s[0])
fenceLength := 3
for _, c := range s[3:] {
if c == fenceCharacter {
fenceLength++
} else {
break
}
}
for i := r.Position + fenceLength; i < r.End; i++ {
if markdown[i] == '`' {
return nil
}
}
return []Block{
&FencedCode{
markdown: markdown,
Indentation: indentation,
RawInfo: trimRightSpace(markdown, Range{r.Position + fenceLength, r.End}),
OpeningFence: Range{r.Position, r.Position + fenceLength},
},
}
}

View file

@ -0,0 +1,192 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
import (
"fmt"
"strings"
)
var htmlEscaper = strings.NewReplacer(
`&`, "&amp;",
`<`, "&lt;",
`>`, "&gt;",
`"`, "&quot;",
)
// RenderHTML produces HTML with the same behavior as the example renderer used in the CommonMark
// reference materials except for one slight difference: for brevity, no unnecessary whitespace is
// inserted between elements. The output is not defined by the CommonMark spec, and it exists
// primarily as an aid in testing.
func RenderHTML(markdown string) string {
return RenderBlockHTML(Parse(markdown))
}
func RenderBlockHTML(block Block, referenceDefinitions []*ReferenceDefinition) (result string) {
return renderBlockHTML(block, referenceDefinitions, false)
}
func renderBlockHTML(block Block, referenceDefinitions []*ReferenceDefinition, isTightList bool) (result string) {
switch v := block.(type) {
case *Document:
for _, block := range v.Children {
result += RenderBlockHTML(block, referenceDefinitions)
}
case *Paragraph:
if len(v.Text) == 0 {
return
}
if !isTightList {
result += "<p>"
}
for _, inline := range v.ParseInlines(referenceDefinitions) {
result += RenderInlineHTML(inline)
}
if !isTightList {
result += "</p>"
}
case *List:
if v.IsOrdered {
if v.OrderedStart != 1 {
result += fmt.Sprintf(`<ol start="%v">`, v.OrderedStart)
} else {
result += "<ol>"
}
} else {
result += "<ul>"
}
for _, block := range v.Children {
result += renderBlockHTML(block, referenceDefinitions, !v.IsLoose)
}
if v.IsOrdered {
result += "</ol>"
} else {
result += "</ul>"
}
case *ListItem:
result += "<li>"
for _, block := range v.Children {
result += renderBlockHTML(block, referenceDefinitions, isTightList)
}
result += "</li>"
case *BlockQuote:
result += "<blockquote>"
for _, block := range v.Children {
result += RenderBlockHTML(block, referenceDefinitions)
}
result += "</blockquote>"
case *FencedCode:
if info := v.Info(); info != "" {
language := strings.Fields(info)[0]
result += `<pre><code class="language-` + htmlEscaper.Replace(language) + `">`
} else {
result += "<pre><code>"
}
result += htmlEscaper.Replace(v.Code()) + "</code></pre>"
case *IndentedCode:
result += "<pre><code>" + htmlEscaper.Replace(v.Code()) + "</code></pre>"
default:
panic(fmt.Sprintf("missing case for type %T", v))
}
return
}
func escapeURL(url string) (result string) {
for i := 0; i < len(url); {
switch b := url[i]; b {
case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '-', '_', '.', '!', '~', '*', '\'', '(', ')', '#':
result += string(b)
i++
default:
if b == '%' && i+2 < len(url) && isHexByte(url[i+1]) && isHexByte(url[i+2]) {
result += url[i : i+3]
i += 3
} else if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') {
result += string(b)
i++
} else {
result += fmt.Sprintf("%%%0X", b)
i++
}
}
}
return
}
func RenderInlineHTML(inline Inline) (result string) {
switch v := inline.(type) {
case *Text:
return htmlEscaper.Replace(v.Text)
case *HardLineBreak:
return "<br />"
case *SoftLineBreak:
return "\n"
case *CodeSpan:
return "<code>" + htmlEscaper.Replace(v.Code) + "</code>"
case *InlineImage:
result += `<img src="` + htmlEscaper.Replace(escapeURL(v.Destination())) + `" alt="` + htmlEscaper.Replace(renderImageAltText(v.Children)) + `"`
if title := v.Title(); title != "" {
result += ` title="` + htmlEscaper.Replace(title) + `"`
}
result += ` />`
case *ReferenceImage:
result += `<img src="` + htmlEscaper.Replace(escapeURL(v.Destination())) + `" alt="` + htmlEscaper.Replace(renderImageAltText(v.Children)) + `"`
if title := v.Title(); title != "" {
result += ` title="` + htmlEscaper.Replace(title) + `"`
}
result += ` />`
case *InlineLink:
result += `<a href="` + htmlEscaper.Replace(escapeURL(v.Destination())) + `"`
if title := v.Title(); title != "" {
result += ` title="` + htmlEscaper.Replace(title) + `"`
}
result += `>`
for _, inline := range v.Children {
result += RenderInlineHTML(inline)
}
result += "</a>"
case *ReferenceLink:
result += `<a href="` + htmlEscaper.Replace(escapeURL(v.Destination())) + `"`
if title := v.Title(); title != "" {
result += ` title="` + htmlEscaper.Replace(title) + `"`
}
result += `>`
for _, inline := range v.Children {
result += RenderInlineHTML(inline)
}
result += "</a>"
case *Autolink:
result += `<a href="` + htmlEscaper.Replace(escapeURL(v.Link)) + `">`
for _, inline := range v.Children {
result += RenderInlineHTML(inline)
}
result += "</a>"
default:
panic(fmt.Sprintf("missing case for type %T", v))
}
return
}
func renderImageAltText(children []Inline) (result string) {
for _, inline := range children {
result += renderImageChildAltText(inline)
}
return
}
func renderImageChildAltText(inline Inline) (result string) {
switch v := inline.(type) {
case *Text:
return v.Text
case *InlineImage:
for _, inline := range v.Children {
result += renderImageChildAltText(inline)
}
case *InlineLink:
for _, inline := range v.Children {
result += renderImageChildAltText(inline)
}
}
return
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,98 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
import (
"strings"
)
type IndentedCodeLine struct {
Indentation int
Range Range
}
type IndentedCode struct {
blockBase
markdown string
RawCode []IndentedCodeLine
}
func (b *IndentedCode) Code() (result string) {
for _, code := range b.RawCode {
result += strings.Repeat(" ", code.Indentation) + b.markdown[code.Range.Position:code.Range.End]
}
return
}
func (b *IndentedCode) Continuation(indentation int, r Range) *continuation {
if indentation >= 4 {
return &continuation{
Indentation: indentation - 4,
Remaining: r,
}
}
s := b.markdown[r.Position:r.End]
if strings.TrimSpace(s) == "" {
return &continuation{
Remaining: r,
}
}
return nil
}
func (b *IndentedCode) AddLine(indentation int, r Range) bool {
b.RawCode = append(b.RawCode, IndentedCodeLine{
Indentation: indentation,
Range: r,
})
return true
}
func (b *IndentedCode) Close() {
for {
last := b.RawCode[len(b.RawCode)-1]
s := b.markdown[last.Range.Position:last.Range.End]
if strings.TrimRight(s, "\r\n") == "" {
b.RawCode = b.RawCode[:len(b.RawCode)-1]
} else {
break
}
}
}
func (b *IndentedCode) AllowsBlockStarts() bool {
return false
}
func indentedCodeStart(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
if len(unmatchedBlocks) > 0 {
if _, ok := unmatchedBlocks[len(unmatchedBlocks)-1].(*Paragraph); ok {
return nil
}
} else if len(matchedBlocks) > 0 {
if _, ok := matchedBlocks[len(matchedBlocks)-1].(*Paragraph); ok {
return nil
}
}
if indentation < 4 {
return nil
}
s := markdown[r.Position:r.End]
if strings.TrimSpace(s) == "" {
return nil
}
return []Block{
&IndentedCode{
markdown: markdown,
RawCode: []IndentedCodeLine{{
Indentation: indentation - 4,
Range: r,
}},
},
}
}

View file

@ -0,0 +1,620 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
import (
"container/list"
"strings"
"unicode"
"unicode/utf8"
)
type Inline interface {
IsInline() bool
}
type inlineBase struct{}
func (inlineBase) IsInline() bool { return true }
type Text struct {
inlineBase
Text string
Range Range
}
type CodeSpan struct {
inlineBase
Code string
}
type HardLineBreak struct {
inlineBase
}
type SoftLineBreak struct {
inlineBase
}
type InlineLinkOrImage struct {
inlineBase
Children []Inline
RawDestination Range
markdown string
rawTitle string
}
func (i *InlineLinkOrImage) Destination() string {
return Unescape(i.markdown[i.RawDestination.Position:i.RawDestination.End])
}
func (i *InlineLinkOrImage) Title() string {
return Unescape(i.rawTitle)
}
type InlineLink struct {
InlineLinkOrImage
}
type InlineImage struct {
InlineLinkOrImage
}
type ReferenceLinkOrImage struct {
inlineBase
*ReferenceDefinition
Children []Inline
}
type ReferenceLink struct {
ReferenceLinkOrImage
}
type ReferenceImage struct {
ReferenceLinkOrImage
}
type Autolink struct {
inlineBase
Children []Inline
Link string
}
type delimiterType int
const (
linkOpeningDelimiter delimiterType = iota
imageOpeningDelimiter
)
type delimiter struct {
Type delimiterType
IsInactive bool
TextNode int
Range Range
}
type inlineParser struct {
markdown string
ranges []Range
referenceDefinitions []*ReferenceDefinition
raw string
position int
inlines []Inline
delimiterStack *list.List
}
func newInlineParser(markdown string, ranges []Range, referenceDefinitions []*ReferenceDefinition) *inlineParser {
return &inlineParser{
markdown: markdown,
ranges: ranges,
referenceDefinitions: referenceDefinitions,
delimiterStack: list.New(),
}
}
func (p *inlineParser) parseBackticks() {
count := 1
for i := p.position + 1; i < len(p.raw) && p.raw[i] == '`'; i++ {
count++
}
opening := p.raw[p.position : p.position+count]
search := p.position + count
for search < len(p.raw) {
end := strings.Index(p.raw[search:], opening)
if end == -1 {
break
}
if search+end+count < len(p.raw) && p.raw[search+end+count] == '`' {
search += end + count
for search < len(p.raw) && p.raw[search] == '`' {
search++
}
continue
}
code := strings.Join(strings.Fields(p.raw[p.position+count:search+end]), " ")
p.position = search + end + count
p.inlines = append(p.inlines, &CodeSpan{
Code: code,
})
return
}
p.position += len(opening)
absPos := relativeToAbsolutePosition(p.ranges, p.position-len(opening))
p.inlines = append(p.inlines, &Text{
Text: opening,
Range: Range{absPos, absPos + len(opening)},
})
}
func (p *inlineParser) parseLineEnding() {
if p.position >= 1 && p.raw[p.position-1] == '\t' {
p.inlines = append(p.inlines, &HardLineBreak{})
} else if p.position >= 2 && p.raw[p.position-1] == ' ' && (p.raw[p.position-2] == '\t' || p.raw[p.position-1] == ' ') {
p.inlines = append(p.inlines, &HardLineBreak{})
} else {
p.inlines = append(p.inlines, &SoftLineBreak{})
}
p.position++
if p.position < len(p.raw) && p.raw[p.position] == '\n' {
p.position++
}
}
func (p *inlineParser) parseEscapeCharacter() {
if p.position+1 < len(p.raw) && isEscapableByte(p.raw[p.position+1]) {
absPos := relativeToAbsolutePosition(p.ranges, p.position+1)
p.inlines = append(p.inlines, &Text{
Text: string(p.raw[p.position+1]),
Range: Range{absPos, absPos + len(string(p.raw[p.position+1]))},
})
p.position += 2
} else {
absPos := relativeToAbsolutePosition(p.ranges, p.position)
p.inlines = append(p.inlines, &Text{
Text: `\`,
Range: Range{absPos, absPos + 1},
})
p.position++
}
}
func (p *inlineParser) parseText() {
if next := strings.IndexAny(p.raw[p.position:], "\r\n\\`&![]wW:"); next == -1 {
absPos := relativeToAbsolutePosition(p.ranges, p.position)
p.inlines = append(p.inlines, &Text{
Text: strings.TrimRightFunc(p.raw[p.position:], isWhitespace),
Range: Range{absPos, absPos + len(p.raw[p.position:])},
})
p.position = len(p.raw)
} else {
absPos := relativeToAbsolutePosition(p.ranges, p.position)
if p.raw[p.position+next] == '\r' || p.raw[p.position+next] == '\n' {
s := strings.TrimRightFunc(p.raw[p.position:p.position+next], isWhitespace)
p.inlines = append(p.inlines, &Text{
Text: s,
Range: Range{absPos, absPos + len(s)},
})
} else {
if next == 0 {
// Always read at least one character since 'w', 'W', and ':' may not actually match another
// type of node
next = 1
}
p.inlines = append(p.inlines, &Text{
Text: p.raw[p.position : p.position+next],
Range: Range{absPos, absPos + next},
})
}
p.position += next
}
}
func (p *inlineParser) parseLinkOrImageDelimiter() {
absPos := relativeToAbsolutePosition(p.ranges, p.position)
if p.raw[p.position] == '[' {
p.inlines = append(p.inlines, &Text{
Text: "[",
Range: Range{absPos, absPos + 1},
})
p.delimiterStack.PushBack(&delimiter{
Type: linkOpeningDelimiter,
TextNode: len(p.inlines) - 1,
Range: Range{p.position, p.position + 1},
})
p.position++
} else if p.raw[p.position] == '!' && p.position+1 < len(p.raw) && p.raw[p.position+1] == '[' {
p.inlines = append(p.inlines, &Text{
Text: "![",
Range: Range{absPos, absPos + 2},
})
p.delimiterStack.PushBack(&delimiter{
Type: imageOpeningDelimiter,
TextNode: len(p.inlines) - 1,
Range: Range{p.position, p.position + 2},
})
p.position += 2
} else {
p.inlines = append(p.inlines, &Text{
Text: "!",
Range: Range{absPos, absPos + 1},
})
p.position++
}
}
func (p *inlineParser) peekAtInlineLinkDestinationAndTitle(position int) (destination, title Range, end int, ok bool) {
if position >= len(p.raw) || p.raw[position] != '(' {
return
}
position++
destinationStart := nextNonWhitespace(p.raw, position)
if destinationStart >= len(p.raw) {
return
} else if p.raw[destinationStart] == ')' {
return Range{destinationStart, destinationStart}, Range{destinationStart, destinationStart}, destinationStart + 1, true
}
destination, end, ok = parseLinkDestination(p.raw, destinationStart)
if !ok {
return
}
position = end
if position < len(p.raw) && isWhitespaceByte(p.raw[position]) {
titleStart := nextNonWhitespace(p.raw, position)
if titleStart >= len(p.raw) {
return
} else if p.raw[titleStart] == ')' {
return destination, Range{titleStart, titleStart}, titleStart + 1, true
}
title, end, ok = parseLinkTitle(p.raw, titleStart)
if !ok {
return
}
position = end
}
closingPosition := nextNonWhitespace(p.raw, position)
if closingPosition >= len(p.raw) || p.raw[closingPosition] != ')' {
return Range{}, Range{}, 0, false
}
return destination, title, closingPosition + 1, true
}
func (p *inlineParser) referenceDefinition(label string) *ReferenceDefinition {
clean := strings.Join(strings.Fields(label), " ")
for _, d := range p.referenceDefinitions {
if strings.EqualFold(clean, strings.Join(strings.Fields(d.Label()), " ")) {
return d
}
}
return nil
}
func (p *inlineParser) lookForLinkOrImage() {
for element := p.delimiterStack.Back(); element != nil; element = element.Prev() {
d := element.Value.(*delimiter)
if d.Type != imageOpeningDelimiter && d.Type != linkOpeningDelimiter {
continue
}
if d.IsInactive {
p.delimiterStack.Remove(element)
break
}
var inline Inline
if destination, title, next, ok := p.peekAtInlineLinkDestinationAndTitle(p.position + 1); ok {
destinationMarkdownPosition := relativeToAbsolutePosition(p.ranges, destination.Position)
linkOrImage := InlineLinkOrImage{
Children: append([]Inline(nil), p.inlines[d.TextNode+1:]...),
RawDestination: Range{destinationMarkdownPosition, destinationMarkdownPosition + destination.End - destination.Position},
markdown: p.markdown,
rawTitle: p.raw[title.Position:title.End],
}
if d.Type == imageOpeningDelimiter {
inline = &InlineImage{linkOrImage}
} else {
inline = &InlineLink{linkOrImage}
}
p.position = next
} else {
referenceLabel := ""
label, next, hasLinkLabel := parseLinkLabel(p.raw, p.position+1)
if hasLinkLabel && label.End > label.Position {
referenceLabel = p.raw[label.Position:label.End]
} else {
referenceLabel = p.raw[d.Range.End:p.position]
if !hasLinkLabel {
next = p.position + 1
}
}
if referenceLabel != "" {
if reference := p.referenceDefinition(referenceLabel); reference != nil {
linkOrImage := ReferenceLinkOrImage{
ReferenceDefinition: reference,
Children: append([]Inline(nil), p.inlines[d.TextNode+1:]...),
}
if d.Type == imageOpeningDelimiter {
inline = &ReferenceImage{linkOrImage}
} else {
inline = &ReferenceLink{linkOrImage}
}
p.position = next
}
}
}
if inline != nil {
if d.Type == imageOpeningDelimiter {
p.inlines = append(p.inlines[:d.TextNode], inline)
} else {
p.inlines = append(p.inlines[:d.TextNode], inline)
for element := element.Prev(); element != nil; element = element.Prev() {
if d := element.Value.(*delimiter); d.Type == linkOpeningDelimiter {
d.IsInactive = true
}
}
}
p.delimiterStack.Remove(element)
return
} else {
p.delimiterStack.Remove(element)
break
}
}
absPos := relativeToAbsolutePosition(p.ranges, p.position)
p.inlines = append(p.inlines, &Text{
Text: "]",
Range: Range{absPos, absPos + 1},
})
p.position++
}
func CharacterReference(ref string) string {
if ref == "" {
return ""
}
if ref[0] == '#' {
if len(ref) < 2 {
return ""
}
n := 0
if ref[1] == 'X' || ref[1] == 'x' {
if len(ref) < 3 {
return ""
}
for i := 2; i < len(ref); i++ {
if i > 9 {
return ""
}
d := ref[i]
switch {
case d >= '0' && d <= '9':
n = n*16 + int(d-'0')
case d >= 'a' && d <= 'f':
n = n*16 + 10 + int(d-'a')
case d >= 'A' && d <= 'F':
n = n*16 + 10 + int(d-'A')
default:
return ""
}
}
} else {
for i := 1; i < len(ref); i++ {
if i > 8 || ref[i] < '0' || ref[i] > '9' {
return ""
}
n = n*10 + int(ref[i]-'0')
}
}
c := rune(n)
if c == '\u0000' || !utf8.ValidRune(c) {
return string(unicode.ReplacementChar)
}
return string(c)
}
if entity, ok := htmlEntities[ref]; ok {
return entity
}
return ""
}
func (p *inlineParser) parseCharacterReference() {
absPos := relativeToAbsolutePosition(p.ranges, p.position)
p.position++
if semicolon := strings.IndexByte(p.raw[p.position:], ';'); semicolon == -1 {
p.inlines = append(p.inlines, &Text{
Text: "&",
Range: Range{absPos, absPos + 1},
})
} else if s := CharacterReference(p.raw[p.position : p.position+semicolon]); s != "" {
p.position += semicolon + 1
p.inlines = append(p.inlines, &Text{
Text: s,
Range: Range{absPos, absPos + len(s)},
})
} else {
p.inlines = append(p.inlines, &Text{
Text: "&",
Range: Range{absPos, absPos + 1},
})
}
}
func (p *inlineParser) parseAutolink(c rune) bool {
for element := p.delimiterStack.Back(); element != nil; element = element.Prev() {
d := element.Value.(*delimiter)
if !d.IsInactive {
return false
}
}
link := ""
text := ""
if c == ':' {
text = parseURLAutolink(p.raw, p.position)
link = text
// Since the current position is at the colon, we have to rewind the parsing slightly so that
// we don't duplicate the URL scheme
rewind := strings.Index(text, ":")
if rewind != -1 {
lastInline := p.inlines[len(p.inlines)-1]
lastText, ok := lastInline.(*Text)
if !ok {
// This should never occur since parseURLAutolink will only return a non-empty value
// when the previous text ends in a valid URL protocol which would mean that the previous
// node is a Text node
return false
}
p.inlines = p.inlines[0 : len(p.inlines)-1]
p.inlines = append(p.inlines, &Text{
Text: lastText.Text[:len(lastText.Text)-rewind],
Range: Range{lastText.Range.Position, lastText.Range.End - rewind},
})
p.position -= rewind
}
} else if c == 'w' {
text = parseWWWAutolink(p.raw, p.position)
link = "http://" + text
}
if text == "" {
return false
}
p.inlines = append(p.inlines, &Autolink{
Link: link,
Children: []Inline{&Text{Text: text}},
})
p.position += len(text)
return true
}
func (p *inlineParser) Parse() []Inline {
for _, r := range p.ranges {
p.raw += p.markdown[r.Position:r.End]
}
for p.position < len(p.raw) {
c, _ := utf8.DecodeRuneInString(p.raw[p.position:])
switch c {
case '\r', '\n':
p.parseLineEnding()
case '\\':
p.parseEscapeCharacter()
case '`':
p.parseBackticks()
case '&':
p.parseCharacterReference()
case '!', '[':
p.parseLinkOrImageDelimiter()
case ']':
p.lookForLinkOrImage()
case 'w', 'W', ':':
matched := p.parseAutolink(c)
if !matched {
p.parseText()
}
default:
p.parseText()
}
}
return p.inlines
}
func ParseInlines(markdown string, ranges []Range, referenceDefinitions []*ReferenceDefinition) (inlines []Inline) {
return newInlineParser(markdown, ranges, referenceDefinitions).Parse()
}
func MergeInlineText(inlines []Inline) []Inline {
var ret []Inline
for i, v := range inlines {
// always add first node
if i == 0 {
ret = append(ret, v)
continue
}
// not a text node? nothing to merge
text, ok := v.(*Text)
if !ok {
ret = append(ret, v)
continue
}
// previous node is not a text node? nothing to merge
prevText, ok := ret[len(ret)-1].(*Text)
if !ok {
ret = append(ret, v)
continue
}
// previous node is not right before this one
if prevText.Range.End != text.Range.Position {
ret = append(ret, v)
continue
}
// we have two consecutive text nodes
ret[len(ret)-1] = &Text{
Text: prevText.Text + text.Text,
Range: Range{prevText.Range.Position, text.Range.End},
}
}
return ret
}
func Unescape(markdown string) string {
ret := ""
position := 0
for position < len(markdown) {
c, cSize := utf8.DecodeRuneInString(markdown[position:])
switch c {
case '\\':
if position+1 < len(markdown) && isEscapableByte(markdown[position+1]) {
ret += string(markdown[position+1])
position += 2
} else {
ret += `\`
position++
}
case '&':
position++
if semicolon := strings.IndexByte(markdown[position:], ';'); semicolon == -1 {
ret += "&"
} else if s := CharacterReference(markdown[position : position+semicolon]); s != "" {
position += semicolon + 1
ret += s
} else {
ret += "&"
}
default:
ret += string(c)
position += cSize
}
}
return ret
}

View file

@ -0,0 +1,78 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
// Inspect traverses the markdown tree in depth-first order. If f returns true, Inspect invokes f
// recursively for each child of the block or inline, followed by a call of f(nil).
func Inspect(markdown string, f func(interface{}) bool) {
document, referenceDefinitions := Parse(markdown)
InspectBlock(document, func(block Block) bool {
if !f(block) {
return false
}
switch v := block.(type) {
case *Paragraph:
for _, inline := range MergeInlineText(v.ParseInlines(referenceDefinitions)) {
InspectInline(inline, func(inline Inline) bool {
return f(inline)
})
}
}
return true
})
}
// InspectBlock traverses the blocks in depth-first order, starting with block. If f returns true,
// InspectBlock invokes f recursively for each child of the block, followed by a call of f(nil).
func InspectBlock(block Block, f func(Block) bool) {
if !f(block) {
return
}
switch v := block.(type) {
case *Document:
for _, child := range v.Children {
InspectBlock(child, f)
}
case *List:
for _, child := range v.Children {
InspectBlock(child, f)
}
case *ListItem:
for _, child := range v.Children {
InspectBlock(child, f)
}
case *BlockQuote:
for _, child := range v.Children {
InspectBlock(child, f)
}
}
f(nil)
}
// InspectInline traverses the blocks in depth-first order, starting with block. If f returns true,
// InspectInline invokes f recursively for each child of the block, followed by a call of f(nil).
func InspectInline(inline Inline, f func(Inline) bool) {
if !f(inline) {
return
}
switch v := inline.(type) {
case *InlineImage:
for _, child := range v.Children {
InspectInline(child, f)
}
case *InlineLink:
for _, child := range v.Children {
InspectInline(child, f)
}
case *ReferenceImage:
for _, child := range v.Children {
InspectInline(child, f)
}
case *ReferenceLink:
for _, child := range v.Children {
InspectInline(child, f)
}
}
f(nil)
}

View file

@ -0,0 +1,27 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
type Line struct {
Range
}
func ParseLines(markdown string) (lines []Line) {
lineStartPosition := 0
isAfterCarriageReturn := false
for position, r := range markdown {
if r == '\n' {
lines = append(lines, Line{Range{lineStartPosition, position + 1}})
lineStartPosition = position + 1
} else if isAfterCarriageReturn {
lines = append(lines, Line{Range{lineStartPosition, position}})
lineStartPosition = position
}
isAfterCarriageReturn = r == '\r'
}
if lineStartPosition < len(markdown) {
lines = append(lines, Line{Range{lineStartPosition, len(markdown)}})
}
return
}

View file

@ -0,0 +1,130 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
import (
"unicode/utf8"
)
func parseLinkDestination(markdown string, position int) (raw Range, next int, ok bool) {
if position >= len(markdown) {
return
}
if markdown[position] == '<' {
isEscaped := false
for offset, c := range []byte(markdown[position+1:]) {
if isEscaped {
isEscaped = false
if isEscapableByte(c) {
continue
}
}
if c == '\\' {
isEscaped = true
} else if c == '<' {
break
} else if c == '>' {
return Range{position + 1, position + 1 + offset}, position + 1 + offset + 1, true
} else if isWhitespaceByte(c) {
break
}
}
}
openCount := 0
isEscaped := false
for offset, c := range []byte(markdown[position:]) {
if isEscaped {
isEscaped = false
if isEscapableByte(c) {
continue
}
}
switch c {
case '\\':
isEscaped = true
case '(':
openCount++
case ')':
if openCount < 1 {
return Range{position, position + offset}, position + offset, true
}
openCount--
default:
if isWhitespaceByte(c) {
return Range{position, position + offset}, position + offset, true
}
}
}
return Range{position, len(markdown)}, len(markdown), true
}
func parseLinkTitle(markdown string, position int) (raw Range, next int, ok bool) {
if position >= len(markdown) {
return
}
originalPosition := position
var closer byte
switch markdown[position] {
case '"', '\'':
closer = markdown[position]
case '(':
closer = ')'
default:
return
}
position++
for position < len(markdown) {
switch markdown[position] {
case '\\':
position++
if position < len(markdown) && isEscapableByte(markdown[position]) {
position++
}
case closer:
return Range{originalPosition + 1, position}, position + 1, true
default:
position++
}
}
return
}
func parseLinkLabel(markdown string, position int) (raw Range, next int, ok bool) {
if position >= len(markdown) || markdown[position] != '[' {
return
}
originalPosition := position
position++
for position < len(markdown) {
switch markdown[position] {
case '\\':
position++
if position < len(markdown) && isEscapableByte(markdown[position]) {
position++
}
case '[':
return
case ']':
if position-originalPosition >= 1000 && utf8.RuneCountInString(markdown[originalPosition:position]) >= 1000 {
return
}
return Range{originalPosition + 1, position}, position + 1, true
default:
position++
}
}
return
}

View file

@ -0,0 +1,220 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
import (
"strings"
)
type ListItem struct {
blockBase
markdown string
hasTrailingBlankLine bool
hasBlankLineBetweenChildren bool
Indentation int
Children []Block
}
func (b *ListItem) Continuation(indentation int, r Range) *continuation {
s := b.markdown[r.Position:r.End]
if strings.TrimSpace(s) == "" {
if b.Children == nil {
return nil
}
return &continuation{
Remaining: r,
}
}
if indentation < b.Indentation {
return nil
}
return &continuation{
Indentation: indentation - b.Indentation,
Remaining: r,
}
}
func (b *ListItem) AddChild(openBlocks []Block) []Block {
b.Children = append(b.Children, openBlocks[0])
if b.hasTrailingBlankLine {
b.hasBlankLineBetweenChildren = true
}
b.hasTrailingBlankLine = false
return openBlocks
}
func (b *ListItem) AddLine(indentation int, r Range) bool {
isBlank := strings.TrimSpace(b.markdown[r.Position:r.End]) == ""
if isBlank {
b.hasTrailingBlankLine = true
}
return false
}
func (b *ListItem) HasTrailingBlankLine() bool {
return b.hasTrailingBlankLine || (len(b.Children) > 0 && b.Children[len(b.Children)-1].HasTrailingBlankLine())
}
func (b *ListItem) isLoose() bool {
if b.hasBlankLineBetweenChildren {
return true
}
for i, child := range b.Children {
if i < len(b.Children)-1 && child.HasTrailingBlankLine() {
return true
}
}
return false
}
type List struct {
blockBase
markdown string
hasTrailingBlankLine bool
hasBlankLineBetweenChildren bool
IsLoose bool
IsOrdered bool
OrderedStart int
BulletOrDelimiter byte
Children []*ListItem
}
func (b *List) Continuation(indentation int, r Range) *continuation {
s := b.markdown[r.Position:r.End]
if strings.TrimSpace(s) == "" {
return &continuation{
Remaining: r,
}
}
return &continuation{
Indentation: indentation,
Remaining: r,
}
}
func (b *List) AddChild(openBlocks []Block) []Block {
if item, ok := openBlocks[0].(*ListItem); ok {
b.Children = append(b.Children, item)
if b.hasTrailingBlankLine {
b.hasBlankLineBetweenChildren = true
}
b.hasTrailingBlankLine = false
return openBlocks
} else if list, ok := openBlocks[0].(*List); ok {
if len(list.Children) == 1 && list.IsOrdered == b.IsOrdered && list.BulletOrDelimiter == b.BulletOrDelimiter {
return b.AddChild(openBlocks[1:])
}
}
return nil
}
func (b *List) AddLine(indentation int, r Range) bool {
isBlank := strings.TrimSpace(b.markdown[r.Position:r.End]) == ""
if isBlank {
b.hasTrailingBlankLine = true
}
return false
}
func (b *List) HasTrailingBlankLine() bool {
return b.hasTrailingBlankLine || (len(b.Children) > 0 && b.Children[len(b.Children)-1].HasTrailingBlankLine())
}
func (b *List) isLoose() bool {
if b.hasBlankLineBetweenChildren {
return true
}
for i, child := range b.Children {
if child.isLoose() || (i < len(b.Children)-1 && child.HasTrailingBlankLine()) {
return true
}
}
return false
}
func (b *List) Close() {
b.IsLoose = b.isLoose()
}
func parseListMarker(markdown string, r Range) (success, isOrdered bool, orderedStart int, bulletOrDelimiter byte, markerWidth int, remaining Range) {
digits := 0
n := 0
for i := r.Position; i < r.End && markdown[i] >= '0' && markdown[i] <= '9'; i++ {
digits++
n = n*10 + int(markdown[i]-'0')
}
if digits > 0 {
if digits > 9 || r.Position+digits >= r.End {
return
}
next := markdown[r.Position+digits]
if next != '.' && next != ')' {
return
}
return true, true, n, next, digits + 1, Range{r.Position + digits + 1, r.End}
}
if r.Position >= r.End {
return
}
next := markdown[r.Position]
if next != '-' && next != '+' && next != '*' {
return
}
return true, false, 0, next, 1, Range{r.Position + 1, r.End}
}
func listStart(markdown string, indent int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
afterList := false
if len(matchedBlocks) > 0 {
_, afterList = matchedBlocks[len(matchedBlocks)-1].(*List)
}
if !afterList && indent > 3 {
return nil
}
success, isOrdered, orderedStart, bulletOrDelimiter, markerWidth, remaining := parseListMarker(markdown, r)
if !success {
return nil
}
isBlank := strings.TrimSpace(markdown[remaining.Position:remaining.End]) == ""
if len(matchedBlocks) > 0 && len(unmatchedBlocks) == 0 {
if _, ok := matchedBlocks[len(matchedBlocks)-1].(*Paragraph); ok {
if isBlank || (isOrdered && orderedStart != 1) {
return nil
}
}
}
indentAfterMarker, indentBytesAfterMarker := countIndentation(markdown, remaining)
if !isBlank && indentAfterMarker < 1 {
return nil
}
remaining = Range{remaining.Position + indentBytesAfterMarker, remaining.End}
consumedIndentAfterMarker := indentAfterMarker
if isBlank || indentAfterMarker >= 5 {
consumedIndentAfterMarker = 1
}
listItem := &ListItem{
markdown: markdown,
Indentation: indent + markerWidth + consumedIndentAfterMarker,
}
list := &List{
markdown: markdown,
IsOrdered: isOrdered,
OrderedStart: orderedStart,
BulletOrDelimiter: bulletOrDelimiter,
Children: []*ListItem{listItem},
}
ret := []Block{list, listItem}
if descendants := blockStartOrParagraph(markdown, indentAfterMarker-consumedIndentAfterMarker, remaining, nil, nil); descendants != nil {
listItem.Children = append(listItem.Children, descendants[0])
ret = append(ret, descendants...)
}
return ret
}

View file

@ -0,0 +1,140 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
// This package implements a parser for the subset of the CommonMark spec necessary for us to do
// server-side processing. It is not a full implementation and lacks many features. But it is
// complete enough to efficiently and accurately allow us to do what we need to like rewrite image
// URLs for proxying.
package markdown
import (
"strings"
)
func isEscapable(c rune) bool {
return c > ' ' && (c < '0' || (c > '9' && (c < 'A' || (c > 'Z' && (c < 'a' || (c > 'z' && c <= '~'))))))
}
func isEscapableByte(c byte) bool {
return isEscapable(rune(c))
}
func isWhitespace(c rune) bool {
switch c {
case ' ', '\t', '\n', '\u000b', '\u000c', '\r':
return true
default:
return false
}
}
func isWhitespaceByte(c byte) bool {
return isWhitespace(rune(c))
}
func isHex(c rune) bool {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
}
func isHexByte(c byte) bool {
return isHex(rune(c))
}
func isAlphanumeric(c rune) bool {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}
func isAlphanumericByte(c byte) bool {
return isAlphanumeric(rune(c))
}
func nextNonWhitespace(markdown string, position int) int {
for offset, c := range []byte(markdown[position:]) {
if !isWhitespaceByte(c) {
return position + offset
}
}
return len(markdown)
}
func nextLine(markdown string, position int) (linePosition int, skippedNonWhitespace bool) {
for i := position; i < len(markdown); i++ {
c := markdown[i]
if c == '\r' {
if i+1 < len(markdown) && markdown[i+1] == '\n' {
return i + 2, skippedNonWhitespace
}
return i + 1, skippedNonWhitespace
} else if c == '\n' {
return i + 1, skippedNonWhitespace
} else if !isWhitespaceByte(c) {
skippedNonWhitespace = true
}
}
return len(markdown), skippedNonWhitespace
}
func countIndentation(markdown string, r Range) (spaces, bytes int) {
for i := r.Position; i < r.End; i++ {
if markdown[i] == ' ' {
spaces++
bytes++
} else if markdown[i] == '\t' {
spaces += 4
bytes++
} else {
break
}
}
return
}
func trimLeftSpace(markdown string, r Range) Range {
s := markdown[r.Position:r.End]
trimmed := strings.TrimLeftFunc(s, isWhitespace)
return Range{r.Position, r.End - (len(s) - len(trimmed))}
}
func trimRightSpace(markdown string, r Range) Range {
s := markdown[r.Position:r.End]
trimmed := strings.TrimRightFunc(s, isWhitespace)
return Range{r.Position, r.End - (len(s) - len(trimmed))}
}
func relativeToAbsolutePosition(ranges []Range, position int) int {
rem := position
for _, r := range ranges {
l := r.End - r.Position
if rem < l {
return r.Position + rem
}
rem -= l
}
if len(ranges) == 0 {
return 0
}
return ranges[len(ranges)-1].End
}
func trimBytesFromRanges(ranges []Range, bytes int) (result []Range) {
rem := bytes
for _, r := range ranges {
if rem == 0 {
result = append(result, r)
continue
}
l := r.End - r.Position
if rem < l {
result = append(result, Range{r.Position + rem, r.End})
rem = 0
continue
}
rem -= l
}
return
}
func Parse(markdown string) (*Document, []*ReferenceDefinition) {
lines := ParseLines(markdown)
return ParseBlocks(markdown, lines)
}

View file

@ -0,0 +1,71 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
import (
"strings"
)
type Paragraph struct {
blockBase
markdown string
Text []Range
ReferenceDefinitions []*ReferenceDefinition
}
func (b *Paragraph) ParseInlines(referenceDefinitions []*ReferenceDefinition) []Inline {
return ParseInlines(b.markdown, b.Text, referenceDefinitions)
}
func (b *Paragraph) Continuation(indentation int, r Range) *continuation {
s := b.markdown[r.Position:r.End]
if strings.TrimSpace(s) == "" {
return nil
}
return &continuation{
Indentation: indentation,
Remaining: r,
}
}
func (b *Paragraph) Close() {
for {
for i := 0; i < len(b.Text); i++ {
b.Text[i] = trimLeftSpace(b.markdown, b.Text[i])
if b.Text[i].Position < b.Text[i].End {
break
}
}
if len(b.Text) == 0 || b.Text[0].Position < b.Text[0].End && b.markdown[b.Text[0].Position] != '[' {
break
}
definition, remaining := parseReferenceDefinition(b.markdown, b.Text)
if definition == nil {
break
}
b.ReferenceDefinitions = append(b.ReferenceDefinitions, definition)
b.Text = remaining
}
for i := len(b.Text) - 1; i >= 0; i-- {
b.Text[i] = trimRightSpace(b.markdown, b.Text[i])
if b.Text[i].Position < b.Text[i].End {
break
}
}
}
func newParagraph(markdown string, r Range) *Paragraph {
s := markdown[r.Position:r.End]
if strings.TrimSpace(s) == "" {
return nil
}
return &Paragraph{
markdown: markdown,
Text: []Range{r},
}
}

View file

@ -0,0 +1,75 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package markdown
type ReferenceDefinition struct {
RawDestination Range
markdown string
rawLabel string
rawTitle string
}
func (d *ReferenceDefinition) Destination() string {
return Unescape(d.markdown[d.RawDestination.Position:d.RawDestination.End])
}
func (d *ReferenceDefinition) Label() string {
return d.rawLabel
}
func (d *ReferenceDefinition) Title() string {
return Unescape(d.rawTitle)
}
func parseReferenceDefinition(markdown string, ranges []Range) (*ReferenceDefinition, []Range) {
raw := ""
for _, r := range ranges {
raw += markdown[r.Position:r.End]
}
label, next, ok := parseLinkLabel(raw, 0)
if !ok {
return nil, nil
}
position := next
if position >= len(raw) || raw[position] != ':' {
return nil, nil
}
position++
destination, next, ok := parseLinkDestination(raw, nextNonWhitespace(raw, position))
if !ok {
return nil, nil
}
position = next
absoluteDestination := relativeToAbsolutePosition(ranges, destination.Position)
ret := &ReferenceDefinition{
RawDestination: Range{absoluteDestination, absoluteDestination + destination.End - destination.Position},
markdown: markdown,
rawLabel: raw[label.Position:label.End],
}
if position < len(raw) && isWhitespaceByte(raw[position]) {
title, next, ok := parseLinkTitle(raw, nextNonWhitespace(raw, position))
if !ok {
if nextLine, skippedNonWhitespace := nextLine(raw, position); !skippedNonWhitespace {
return ret, trimBytesFromRanges(ranges, nextLine)
}
return nil, nil
}
if nextLine, skippedNonWhitespace := nextLine(raw, next); !skippedNonWhitespace {
ret.rawTitle = raw[title.Position:title.End]
return ret, trimBytesFromRanges(ranges, nextLine)
}
}
if nextLine, skippedNonWhitespace := nextLine(raw, position); !skippedNonWhitespace {
return ret, trimBytesFromRanges(ranges, nextLine)
}
return nil, nil
}