smtp2shoutrrr/html.go

package smtp2shoutrrr

import (
	"fmt"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/strikethrough"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

const (
	// maxNestingDepth bounds the elements that give every line they contain a
	// prefix. The converter re-emits that prefix per line and per level, so
	// depth multiplies against line count: a message at the server's own 1 MB
	// limit, nested 250 quotes deep, renders to 131 MB and takes minutes.
	// Real mail quotes a handful of levels and indents fewer.
	maxNestingDepth = 6

	// maxRenderedBytes caps what is forwarded. No chat target displays this
	// much, and a notification is not the place to find out.
	maxRenderedBytes = 64 << 10

	// truncationMarker tells the reader the message goes on, rather than
	// letting the cap end it mid-sentence and look like the whole of it.
	truncationMarker = "…"
)

// renderMarkdown rewrites an HTML body as Markdown.
//
// The conversion itself belongs to html-to-markdown, which knows CommonMark —
// escaping, delimiter runs, fencing a code block past the backticks inside it.
// What it has no opinion about is mail, since it is written for documents: a
// document has no preheader written for an inbox list, no tracking pixel, no
// cid: attachment, and lays nothing out in tables. That is what prepare does
// to the tree first, so the converter only ever sees what a notification
// should carry.
func renderMarkdown(source string) (string, error) {
	document, err := html.Parse(strings.NewReader(source))
	if err != nil {
		// The parser recovers from any markup it can hold, so this is a
		// document too deeply nested for it rather than an invalid one.
		return "", fmt.Errorf("parsing HTML body: %w", err)
	}

	prepare(document, 0)

	rendered, err := newConverter().ConvertNode(document)
	if err != nil {
		return "", fmt.Errorf("converting HTML body: %w", err)
	}

	return truncate(strings.TrimSpace(string(rendered))), nil
}

// newConverter builds a converter per message, which is what the library's own
// entry points do: a Converter carries the error of the conversion it is
// running, and go-smtp serves every connection on its own goroutine.
func newConverter() *converter.Converter {
	return converter.NewConverter(
		converter.WithPlugins(
			base.NewBasePlugin(),
			commonmark.NewCommonmarkPlugin(
				// The default is "* * *", which reads as a stray line of
				// asterisks anywhere the Markdown is not rendered.
				commonmark.WithHorizontalRule("---"),
			),
			strikethrough.NewStrikethroughPlugin(),
		),
	)
}

func prepare(node *html.Node, depth int) {
	child := node.FirstChild

	for child != nil {
		next := child.NextSibling

		switch child.Type {
		case html.CommentNode:
			node.RemoveChild(child)

		case html.TextNode:
			child.Data = sanitizeText(child.Data)

		case html.ElementNode:
			if unwrapped := prepareElement(node, child, depth); unwrapped != nil {
				// The element was replaced by its own children, which have
				// not been looked at yet.
				next = unwrapped
			}
		}

		child = next
	}
}

func prepareElement(parent, node *html.Node, depth int) *html.Node {
	if isHidden(node) {
		// A template opens with a hidden block holding the preview line the
		// inbox shows, which is written for the list view and reads as noise
		// anywhere else.
		parent.RemoveChild(node)

		return nil
	}

	switch node.DataAtom {
	case atom.Img:
		return prepareImage(parent, node)

	case atom.A:
		href := sanitizeURL(attrValue(node, "href"))
		if !isUsableURL(href) {
			// In-page anchors, script handlers and the message's own inline
			// attachments give a notification's reader nothing to open, but
			// the text of the link is still part of the message.
			return unwrap(parent, node)
		}

		setAttr(node, "href", href)
		prepare(node, depth)

		if !hasContent(node) {
			// A link wrapped around a tracking pixel, or around the spacer
			// that mail templates use for one, would render as "[](url)" —
			// invisible to the reader. Its destination is the only thing left
			// worth forwarding.
			node.AppendChild(&html.Node{Type: html.TextNode, Data: href})
		}

		return nil

	case atom.Td, atom.Th:
		// Mail is laid out in tables far more often than it tabulates
		// anything, and a converter with no table rules runs the cells of a
		// row together: "Total4Failed0".
		parent.InsertBefore(&html.Node{Type: html.TextNode, Data: " "}, node)

	case atom.Tr:
		if hasElementSibling(node) {
			parent.InsertBefore(&html.Node{
				Type: html.ElementNode, DataAtom: atom.Br, Data: "br",
			}, node)
		}

	case atom.Blockquote, atom.Ul, atom.Ol:
		depth++
		if depth > maxNestingDepth {
			return unwrap(parent, node)
		}
	}

	prepare(node, depth)

	return nil
}

// prepareImage keeps of an image only what a notification can use. Tracking
// pixels, spacers and sliced-up banners carry no alt text and go entirely; an
// inline attachment has no address the reader can fetch, so its alt text
// stays behind as ordinary words.
func prepareImage(parent, node *html.Node) *html.Node {
	alt := collapseSpace(sanitizeText(attrValue(node, "alt")))
	source := sanitizeURL(attrValue(node, "src"))

	switch {
	case alt == "":
		parent.RemoveChild(node)
	case !isUsableURL(source):
		parent.InsertBefore(&html.Node{Type: html.TextNode, Data: alt}, node)
		parent.RemoveChild(node)
	default:
		setAttr(node, "alt", alt)
		setAttr(node, "src", source)
	}

	return nil
}

// unwrap replaces a node with its own children and returns the first of them.
func unwrap(parent, node *html.Node) *html.Node {
	first := node.FirstChild

	for child := node.FirstChild; child != nil; child = node.FirstChild {
		node.RemoveChild(child)
		parent.InsertBefore(child, node)
	}

	parent.RemoveChild(node)

	return first
}

// hasContent reports whether anything left under a node would show. It runs
// after the subtree has been prepared, so an element that survived that is one
// the reader will see.
func hasContent(node *html.Node) bool {
	for child := node.FirstChild; child != nil; child = child.NextSibling {
		switch child.Type {
		case html.ElementNode:
			if child.DataAtom == atom.Img || hasContent(child) {
				return true
			}
		case html.TextNode:
			if strings.TrimSpace(child.Data) != "" {
				return true
			}
		}
	}

	return false
}

func hasElementSibling(node *html.Node) bool {
	for sibling := node.PrevSibling; sibling != nil; sibling = sibling.PrevSibling {
		if sibling.Type == html.ElementNode {
			return true
		}
	}

	return false
}

func truncate(rendered string) string {
	if len(rendered) <= maxRenderedBytes {
		return rendered
	}

	cut := maxRenderedBytes
	for cut > 0 && !utf8.RuneStart(rendered[cut]) {
		cut--
	}

	return strings.TrimRight(rendered[:cut], " \t\n") + "\n" + truncationMarker
}

// isUsableURL rejects the destinations a notification's reader cannot act on:
// in-page anchors, inline data, script handlers and the message's own inline
// attachments.
func isUsableURL(raw string) bool {
	if raw == "" || strings.HasPrefix(raw, "#") {
		return false
	}

	scheme, _, found := strings.Cut(raw, ":")
	if !found || !isScheme(scheme) {
		return true
	}

	switch strings.ToLower(scheme) {
	case "javascript", "data", "cid", "about", "blob":
		return false
	}

	return true
}

func isScheme(s string) bool {
	if s == "" || !unicode.IsLetter(rune(s[0])) {
		return false
	}

	return strings.IndexFunc(s, func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r) &&
			r != '+' && r != '-' && r != '.'
	}) < 0
}

// sanitizeURL applies to a URL attribute what a browser applies before it
// fetches one: the tabs and line breaks that may sit inside the value are
// removed. Left in, they end the line the notification is on, which is a place
// to write a sentence the reader will take for the sender's.
func sanitizeURL(raw string) string {
	return strings.Map(func(r rune) rune {
		if r == ' ' {
			return ' '
		}

		if unicode.IsSpace(r) || unicode.IsControl(r) {
			return -1
		}

		return r
	}, strings.TrimSpace(raw))
}

// collapseSpace squeezes the runs of whitespace an attribute value may hold
// into single spaces, for the same reason: alt text spans lines in the source
// and must not span them in the message.
func collapseSpace(s string) string {
	return strings.Join(strings.Fields(s), " ")
}

// sanitizeText turns the characters mail uses for layout into ones a
// notification can show: the non-breaking spaces that hold table cells apart
// become ordinary spaces, and the zero-width padding that hides a preheader
// from the inbox preview is dropped rather than forwarded invisibly.
func sanitizeText(s string) string {
	return strings.Map(func(r rune) rune {
		switch r {
		case zeroWidthSpace, zeroWidthNonJoiner, zeroWidthJoiner,
			wordJoiner, byteOrderMark, softHyphen:
			return -1
		case noBreakSpace, narrowNoBreakSpace, figureSpace:
			return ' '
		}

		if unicode.IsSpace(r) {
			return r
		}

		// Control characters have no rendering of their own and only muddle
		// the payloads the targets are sent as.
		if unicode.IsControl(r) {
			return -1
		}

		return r
	}, s)
}

// The characters mail uses for layout rather than for words: padding that
// hides a preheader from the inbox preview, and the spaces that hold table
// cells apart without letting them wrap.
const (
	softHyphen         = '\u00ad'
	noBreakSpace       = '\u00a0'
	zeroWidthSpace     = '\u200b'
	zeroWidthNonJoiner = '\u200c'
	zeroWidthJoiner    = '\u200d'
	narrowNoBreakSpace = '\u202f'
	wordJoiner         = '\u2060'
	figureSpace        = '\u2007'
	byteOrderMark      = '\ufeff'
)

// isHidden reports whether a browser would leave the element out of the page.
func isHidden(node *html.Node) bool {
	for _, attr := range node.Attr {
		switch attr.Key {
		case "hidden":
			return true
		case "style":
			if isHidingStyle(spaceStripper.Replace(strings.ToLower(attr.Val))) {
				return true
			}
		}
	}

	return false
}

func isHidingStyle(style string) bool {
	for _, declaration := range []string{"display:none", "visibility:hidden", "mso-hide:all"} {
		if strings.Contains(style, declaration) {
			return true
		}
	}

	// font-size:0 hides a preheader too, but only when the zero is the whole
	// value: font-size:0.9em is an ordinary line of text.
	if index := strings.Index(style, "font-size:0"); index >= 0 {
		rest := strings.TrimPrefix(style[index+len("font-size:0"):], "px")
		rest = strings.TrimPrefix(rest, "pt")

		return rest == "" || strings.HasPrefix(rest, ";")
	}

	return false
}

var spaceStripper = strings.NewReplacer(" ", "", "\t", "", "\n", "", "\r", "")

func attrValue(node *html.Node, key string) string {
	for _, attr := range node.Attr {
		if attr.Key == key {
			return attr.Val
		}
	}

	return ""
}

func setAttr(node *html.Node, key, value string) {
	for i := range node.Attr {
		if node.Attr[i].Key == key {
			node.Attr[i].Val = value

			return
		}
	}

	node.Attr = append(node.Attr, html.Attribute{Key: key, Val: value})
}