diff --git a/README.md b/README.md index 057a7a8..538800e 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,11 @@ Targets = [ "discord://token@id", "slack://token@channel" ] +# Optional: how an HTML message body should reach these targets. +# "raw" forward the message unchanged (default) +# "markdown" render an HTML body as Markdown +# See "Converting HTML messages" below. +# Format = "markdown" # Single Target (Deprecated) # The Target field is still supported for backward compatibility @@ -42,14 +47,17 @@ Target = "ntfy://ntfy.sh/legacy-topic?tags=thing" # Will show deprecation warni [CatchAll] # Shoutrrr services to forward unmatched emails to Targets = ["ntfy://ntfy.sh/catch-all-topic?tags=unmatched"] +# Format applies to the catch-all as well +# Format = "markdown" ``` The server refuses to start if the configuration cannot forward anything — no recipients and no catch-all, a recipient without addresses or usable targets, or a mistyped table name such as `[[Recipient]]`, which TOML would otherwise accept -silently. It also refuses to start without a `Username` and a `Password`: these -used to fall back to `username`/`password`, which left the authentication gate -open to the most obvious guess there is. +silently, or a `Format` that names none of the supported conversions. It also +refuses to start without a `Username` and a `Password`: these used to fall back +to `username`/`password`, which left the authentication gate open to the most +obvious guess there is. Clients must authenticate with the configured `Username` and `Password` before starting a mail transaction; an unauthenticated client is refused rather than @@ -86,6 +94,67 @@ docker run -v /path/to/config.toml:/config.toml \ git.nakama.town/fmartingr/smtp2shoutrrr:latest ``` +## Converting HTML messages + +Plenty of mail carries nothing but HTML, and plenty of notification services +render none of it — a Mattermost direct message shows the markup verbatim. +Setting `Format` on a recipient (or on `[CatchAll]`) rewrites an HTML body +before it is forwarded: + +| `Format` | Effect | +| ---------- | --------------------------------------------------- | +| `raw` | Forward the body as the message wrote it. (default) | +| `markdown` | Render an HTML body as Markdown. | + +There is deliberately no plain-text format beside it. Markdown reads as plain +text wherever nothing renders it — that is rather the point of Markdown — so a +second conversion would only be a worse copy of this one. + +The value is not case sensitive, and an unrecognised one is refused when the +server starts rather than silently forwarding raw HTML. + +Only an HTML body is ever rewritten. A message that arrives as plain text is +what the sender chose to write and is forwarded untouched whatever `Format` +says. Which part of a multipart message that is follows RFC 2046: the parts of +a `multipart/alternative` are the same content in several forms, so the +`text/plain` one is preferred where it is not empty; the parts of any other +multipart are cumulative, so the first one carrying a body is the message and +the footers, signatures and attachments after it are not. + +The Markdown itself is produced by +[html-to-markdown](https://github.com/JohannesKaufmann/html-to-markdown), which +knows CommonMark — escaping, delimiter runs, fencing a code block past the +backticks inside it. What it does not know is mail, because it is written for +documents. Before the conversion runs, the parsed message is stripped down to +what a notification should carry: + +- Hidden elements go. A template opens with a preheader written for the inbox + list, which reads as noise anywhere else. +- Images without alt text go, which removes tracking pixels, spacers and + sliced-up banners. An inline `cid:` attachment leaves its alt text behind as + ordinary words. +- Destinations a reader cannot open — `#anchors`, `javascript:`, `cid:` — are + dropped and the text of the link is kept. Tabs and line breaks are removed + from the rest, since a line break inside an `href` is invisible in the + document and a fabricated line in the message. +- Table rows become lines and their cells stay apart, since mail lays itself + out in tables far more often than it tabulates anything. +- Quoting and list nesting is flattened past six levels. A line prefix is + re-emitted on every line of every level it nests, so depth multiplies against + line count: a message at the 1 MB limit nested 250 quotes deep otherwise + renders to 131 MB and takes minutes of CPU. + +If a body is too deeply nested for the HTML parser, or renders to nothing at +all because it was images and tracking pixels, it is forwarded unchanged and a +warning is logged: reformatting is a courtesy to the target, not a condition of +delivery. Output past 64 KiB is cut short with a `…`, which no chat target +would have displayed anyway. + +Bodies are decoded before they are rendered, so `quoted-printable` and `base64` +transfer encodings and non-UTF-8 character sets (`iso-8859-1`, `windows-1252`, +…) reach the target as readable text in both formats, `raw` included — which is +the one way `raw` is not quite the bytes that arrived. + ## Development Run the server with: diff --git a/backend.go b/backend.go index fd05096..36f11a6 100644 --- a/backend.go +++ b/backend.go @@ -56,7 +56,7 @@ type Backend struct { config *Config } -func (bkd *Backend) sendNotifications(recipient ConfigRecipient, email ReceivedEmail) error { +func (bkd *Backend) sendNotifications(recipient ConfigRecipient, email *ReceivedEmail) error { // Get all target URLs (handles merging and caching) targetURLs := recipient.GetTargetURLs() @@ -66,7 +66,7 @@ func (bkd *Backend) sendNotifications(recipient ConfigRecipient, email ReceivedE } // Prepare email body once (reuse for all targets) - body, err := email.Body() + body, err := email.FormattedBody(recipient.Format) if err != nil { slog.Error("Error getting email body", slog.String("err", err.Error())) return fmt.Errorf("failed to get email body: %w", err) @@ -74,6 +74,7 @@ func (bkd *Backend) sendNotifications(recipient ConfigRecipient, email ReceivedE slog.Debug("Prepared email body", slog.Int("body_length", len(body)), + slog.String("format", string(recipient.Format)), slog.String("subject", email.Msg.Header.Get("Subject")), slog.String("content_type", email.Msg.Header.Get("Content-Type"))) @@ -134,7 +135,7 @@ func (bkd *Backend) NewSession(c *smtp.Conn) (smtp.Session, error) { }, nil } -func (bkd *Backend) forwardEmail(email ReceivedEmail) error { +func (bkd *Backend) forwardEmail(email *ReceivedEmail) error { slog.Info("forwading message", slog.String("to", strings.Join(email.Recipients, ","))) // Try to match configured recipients first @@ -181,7 +182,10 @@ type Session struct { config *Config - forwarderFunc func(ReceivedEmail) error + // forwarderFunc takes the message by pointer: the body is read from a + // single-use stream and cached on it, so a copy would forward an empty + // notification to every recipient after the first. + forwarderFunc func(*ReceivedEmail) error } func (s *Session) isAuthenticated() bool { @@ -292,7 +296,7 @@ func (s *Session) Data(r io.Reader) error { slog.Info("Received email", slog.String("destination", strings.Join(recipients, ","))) - if err := s.forwarderFunc(ReceivedEmail{ + if err := s.forwarderFunc(&ReceivedEmail{ Recipients: recipients, Msg: msg, }); err != nil { diff --git a/backend_test.go b/backend_test.go index e4c5eb4..b3fe864 100644 --- a/backend_test.go +++ b/backend_test.go @@ -699,3 +699,105 @@ func TestRecipientWithoutTargetsIsRejectedPermanently(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "550", "a retry cannot add targets to a configuration") } + +// The case the option exists for: a target that renders no HTML, receiving +// mail that carries nothing else. +func TestHTMLBodyIsConvertedForTheRecipientsFormat(t *testing.T) { + notifications := make(chan string, 4) + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + + return + } + + notifications <- string(body) + w.WriteHeader(http.StatusOK) + })) + defer mockServer.Close() + + config := &Config{ + Port: 0, + Username: "testuser", + Password: "testpass", + Recipients: []ConfigRecipient{ + { + Addresses: []string{"markdown@example.com"}, + Targets: []string{"generic+" + mockServer.URL + "/?template=json"}, + Format: FormatMarkdown, + }, + { + Addresses: []string{"raw@example.com"}, + Targets: []string{"generic+" + mockServer.URL + "/?template=json"}, + }, + }, + } + + addr, _, _ := startServer(t, NewSMTPServer(config)) + auth := smtp.PlainAuth("", config.Username, config.Password, "localhost") + + message := []byte(strings.Join([]string{ + "Subject: Deploy failed", + `Content-Type: multipart/mixed; boundary="b"`, + "", + "--b", + "Content-Type: text/html; charset=utf-8", + "Content-Transfer-Encoding: quoted-printable", + "", + "

The staging deploy =E2=80=94 run 4=", + "2 =E2=80=94 failed.

", + "--b--", + "", + }, "\r\n")) + + for _, recipient := range []string{"markdown@example.com", "raw@example.com"} { + require.NoError(t, smtp.SendMail(addr, auth, "sender@example.com", []string{recipient}, message)) + } + + converted := <-notifications + require.Contains(t, converted, + `The **staging** deploy — [run 42](https://ci.example.com/42) — failed.`) + require.NotContains(t, converted, "") + require.NotContains(t, converted, "=E2=80=94", "the transfer encoding is undone before the markup is") + + unconverted := <-notifications + require.Contains(t, unconverted, `staging`, + "a recipient that asked for nothing still receives the message as it arrived") +} + +// The message body is read from a single-use stream and cached on the value +// that holds it, so passing it by value hands every recipient after the first +// an empty notification with a 250. +func TestMessageBodySurvivesASecondRecipient(t *testing.T) { + notifications := make(chan string, 4) + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + + return + } + + notifications <- string(body) + w.WriteHeader(http.StatusOK) + })) + defer mockServer.Close() + + recipient := ConfigRecipient{ + Addresses: []string{"test@example.com"}, + Targets: []string{"generic+" + mockServer.URL + "/?template=json"}, + } + + backend := &Backend{config: &Config{Username: "u", Password: "p"}} + email := &ReceivedEmail{ + Recipients: []string{"test@example.com"}, + Msg: readMessage(t, "Subject: Test\r\n\r\nthe body"), + } + + require.NoError(t, backend.sendNotifications(recipient, email)) + require.NoError(t, backend.sendNotifications(recipient, email)) + + require.Contains(t, <-notifications, "the body") + require.Contains(t, <-notifications, "the body") +} diff --git a/config.go b/config.go index 61411e8..d6a8c63 100644 --- a/config.go +++ b/config.go @@ -67,6 +67,14 @@ func (c *Config) SetDefaults() { if c.Port == 0 { c.Port = 11125 } + + for i := range c.Recipients { + c.Recipients[i].Format = c.Recipients[i].Format.normalize() + } + + if c.CatchAll != nil { + c.CatchAll.Format = c.CatchAll.Format.normalize() + } } // Validate rejects configuration that would leave the server relaying for @@ -95,10 +103,20 @@ func (c *Config) Validate() error { return fmt.Errorf("recipient %d (%s) has no usable Targets", i, strings.Join(r.Addresses, ",")) } + if !r.Format.valid() { + return fmt.Errorf("recipient %d (%s) has an unknown Format %q, expected one of: %s", + i, strings.Join(r.Addresses, ","), r.Format, formatNames()) + } } - if c.CatchAll != nil && len(c.CatchAll.GetTargetURLs()) == 0 { - return errors.New("CatchAll has no usable Targets") + if c.CatchAll != nil { + if len(c.CatchAll.GetTargetURLs()) == 0 { + return errors.New("CatchAll has no usable Targets") + } + if !c.CatchAll.Format.valid() { + return fmt.Errorf("CatchAll has an unknown Format %q, expected one of: %s", + c.CatchAll.Format, formatNames()) + } } if len(c.Recipients) == 0 && c.CatchAll == nil { @@ -109,9 +127,16 @@ func (c *Config) Validate() error { } type ConfigRecipient struct { - Addresses []string // email addresses - Target string // deprecated: use Targets instead - Targets []string // shoutrrr addresses (supports multiple) + Addresses []string // email addresses + Target string // deprecated: use Targets instead + Targets []string // shoutrrr addresses (supports multiple) + + // Format is the representation these targets want an HTML message body + // in: raw (the default, forwarding the message unchanged), text or + // markdown. Targets that render neither HTML nor Markdown — a Mattermost + // direct message, for one — otherwise receive the markup verbatim. + Format BodyFormat + targetURLs []*url.URL // cached parsed URLs } diff --git a/config_test.go b/config_test.go index 81d898c..296856b 100644 --- a/config_test.go +++ b/config_test.go @@ -191,3 +191,61 @@ Targets = ["ntfy://ntfy.sh/catch-all"] require.Contains(t, logged.String(), "ignoring unknown key in config file") require.Contains(t, logged.String(), "Recipient") } + +func TestLoadConfigReadsFormat(t *testing.T) { + config, err := LoadConfig(writeConfig(t, credentials+` +[[Recipients]] +Addresses = ["user@example.com"] +Targets = ["ntfy://ntfy.sh/topic"] +Format = "Markdown" + +[[Recipients]] +Addresses = ["asis@example.com"] +Targets = ["ntfy://ntfy.sh/asis"] + +[CatchAll] +Targets = ["ntfy://ntfy.sh/catch-all"] +Format = "markdown" +`)) + require.NoError(t, err) + + require.Equal(t, FormatMarkdown, config.Recipients[0].Format, "the option is not case sensitive") + require.Equal(t, FormatRaw, config.Recipients[1].Format, "an unset Format forwards the message unchanged") + require.Equal(t, FormatMarkdown, config.CatchAll.Format) +} + +// A typo here is silent otherwise: the server starts and forwards raw HTML to +// a target that cannot render it. +func TestLoadConfigRejectsUnknownFormat(t *testing.T) { + for name, contents := range map[string]string{ + "on a recipient": ` +[[Recipients]] +Addresses = ["user@example.com"] +Targets = ["ntfy://ntfy.sh/topic"] +Format = "text" +`, + "on the catch-all": ` +[CatchAll] +Targets = ["ntfy://ntfy.sh/catch-all"] +Format = "md" +`, + } { + t.Run(name, func(t *testing.T) { + _, err := LoadConfig(writeConfig(t, credentials+contents)) + require.Error(t, err) + require.Contains(t, err.Error(), "raw, markdown") + }) + } +} + +// Validate used to depend on SetDefaults having run, so a Config assembled in +// Go failed on a Format nobody had set. +func TestValidateAcceptsAConfigBuiltWithoutDefaults(t *testing.T) { + config := &Config{ + Username: "user", + Password: "secret", + Recipients: []ConfigRecipient{{Addresses: []string{"user@example.com"}, Targets: []string{"ntfy://ntfy.sh/topic"}}}, + } + + require.NoError(t, config.Validate()) +} diff --git a/email.go b/email.go index 3612bb5..7d0e513 100644 --- a/email.go +++ b/email.go @@ -2,14 +2,20 @@ package smtp2shoutrrr import ( "bytes" + "encoding/base64" "errors" "fmt" "io" "log/slog" "mime" "mime/multipart" + "mime/quotedprintable" "net/mail" + "net/textproto" "strings" + "unicode" + + "golang.org/x/net/html/charset" ) // errMalformedMessage marks a message this server can never turn into a @@ -17,81 +23,383 @@ import ( // sender is told to give up rather than retry forever. var errMalformedMessage = errors.New("malformed message") +// maxMultipartDepth bounds how deep a message can make this server walk. Real +// mail nests three levels at most (mixed → related → alternative); anything +// past that is a generated or hostile message rather than one with a body +// worth finding. +const maxMultipartDepth = 8 + type ReceivedEmail struct { Recipients []string Msg *mail.Message + + bodyRead bool body string + bodyIsHTML bool } +// Body returns the message body as the message carried it: its text/plain part +// when there is one, its text/html part otherwise. func (re *ReceivedEmail) Body() (string, error) { - if re.body == "" { - contentType := re.Msg.Header.Get("Content-Type") - - if contentType == "" { - body, err := io.ReadAll(re.Msg.Body) - if err != nil { - return "", fmt.Errorf("failed to read email body: %w", err) - } - re.body = string(body) - } else { - mediaType, params, err := mime.ParseMediaType(contentType) - if err != nil { - return "", fmt.Errorf("%w: parsing Content-Type %q: %w", errMalformedMessage, contentType, err) - } - - if strings.HasPrefix(mediaType, "multipart/") { - // Handle any multipart type (alternative, mixed, related, etc.) - mr := multipart.NewReader(re.Msg.Body, params["boundary"]) - var htmlBody string - - for { - part, err := mr.NextPart() - if err != nil { - break - } - - partContentType := part.Header.Get("Content-Type") - slog.Debug("Processing email part", slog.String("content_type", partContentType)) - - // Prefer text/plain, but keep HTML as fallback - if strings.HasPrefix(partContentType, "text/plain") { - body := new(bytes.Buffer) - _, err = body.ReadFrom(part) - _ = part.Close() - if err != nil { - slog.Error("Failed to read part body", slog.String("err", err.Error())) - return "", fmt.Errorf("failed to read part body: %w", err) - } - - re.body = body.String() - break // Found text/plain, use it - } else if strings.HasPrefix(partContentType, "text/html") && htmlBody == "" { - // Store HTML as fallback if no text/plain found - body := new(bytes.Buffer) - _, err = body.ReadFrom(part) - _ = part.Close() - if err == nil { - htmlBody = body.String() - } - } else { - _ = part.Close() - } - } - - // If no text/plain was found, use HTML - if re.body == "" && htmlBody != "" { - re.body = htmlBody - } - } else if strings.HasPrefix(mediaType, "text/") { - // Handle non-multipart text content (text/plain, text/html, etc.) - body, err := io.ReadAll(re.Msg.Body) - if err != nil { - return "", fmt.Errorf("failed to read email body: %w", err) - } - re.body = string(body) - } - } + if err := re.readBody(); err != nil { + return "", err } return re.body, nil } + +// FormattedBody returns the body in the requested format. Only an HTML body is +// ever rewritten: a message that already arrived as plain text is what the +// sender chose to write, and no target is better served by a round trip +// through a renderer. +func (re *ReceivedEmail) FormattedBody(format BodyFormat) (string, error) { + if err := re.readBody(); err != nil { + return "", err + } + + if format.normalize() == FormatRaw || !re.bodyIsHTML { + return re.body, nil + } + + // Reformatting is a courtesy to the target, not a condition of delivery. + // A body the renderer cannot take apart, or one that renders to nothing + // because it was all images and tracking pixels, is forwarded as it + // arrived: an empty notification is refused by most targets, which puts + // the message into a retry loop it can never leave. + rendered, err := renderMarkdown(re.body) + if err != nil { + slog.Warn("failed to render HTML body, forwarding it unchanged", + slog.String("format", string(format)), + slog.String("err", err.Error())) + + return re.body, nil + } + + if rendered == "" && strings.TrimSpace(re.body) != "" { + slog.Warn("HTML body rendered to nothing, forwarding it unchanged", + slog.String("format", string(format)), + slog.Int("html_length", len(re.body))) + + return re.body, nil + } + + return rendered, nil +} + +func (re *ReceivedEmail) readBody() error { + if re.bodyRead { + return nil + } + + body, isHTML, err := readEntity(textproto.MIMEHeader(re.Msg.Header), re.Msg.Body, 0) + if err != nil { + return err + } + + re.body, re.bodyIsHTML, re.bodyRead = body, isHTML, true + + return nil +} + +// readEntity reads one MIME entity — the message itself or a part of it — and +// reports the body it contributes and whether that body is HTML. +func readEntity(header textproto.MIMEHeader, body io.Reader, depth int) (string, bool, error) { + contentType := header.Get("Content-Type") + if contentType == "" { + content, err := decodeContent(header, body, "") + + return content, false, err + } + + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + return "", false, fmt.Errorf("%w: parsing Content-Type %q: %w", errMalformedMessage, contentType, err) + } + + return readMediaType(header, body, mediaType, params, depth) +} + +func readMediaType( + header textproto.MIMEHeader, + body io.Reader, + mediaType string, + params map[string]string, + depth int, +) (string, bool, error) { + switch { + case strings.HasPrefix(mediaType, "multipart/"): + return readMultipart(body, mediaType, params["boundary"], depth) + + case strings.HasPrefix(mediaType, "text/"): + content, err := decodeContent(header, body, params["charset"]) + + return content, mediaType == "text/html", err + + default: + // An image or an application/* payload on its own carries no text to + // notify with; the subject still reaches the target as the title. + return "", false, nil + } +} + +func readMultipart(body io.Reader, mediaType, boundary string, depth int) (string, bool, error) { + if boundary == "" { + return "", false, fmt.Errorf("%w: multipart message without a boundary", errMalformedMessage) + } + + if depth >= maxMultipartDepth { + slog.Warn("stopped descending into a deeply nested message", slog.Int("depth", depth)) + + return "", false, nil + } + + // RFC 2046 §5.1.4: the parts of a multipart/alternative are one content in + // several forms, so exactly one of them is chosen. The parts of any other + // multipart are cumulative (§5.1.3): the first one carrying a body is the + // message, and what follows it is footers, signatures and enclosures. + alternatives := mediaType == "multipart/alternative" + + var chosen bodySelector + + reader := multipart.NewReader(body, boundary) + + for { + part, err := reader.NextPart() + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + // A truncated container still hands over the parts that were + // readable. Only a message that yielded nothing at all is worth + // refusing, since there is nothing left to notify with. + slog.Warn("stopped reading a multipart message early", slog.String("err", err.Error())) + + if chosen.empty() { + return "", false, fmt.Errorf("%w: reading multipart body: %w", errMalformedMessage, err) + } + + break + } + + content, isHTML, err := readPart(part, depth) + _ = part.Close() + + if err != nil { + // One unreadable part is not the whole message; the rest of the + // tree may still hold a body. + slog.Warn("ignoring an unreadable email part", slog.String("err", err.Error())) + + continue + } + + if !alternatives { + if strings.TrimSpace(content) != "" { + return content, isHTML, nil + } + + continue + } + + chosen.offer(content, isHTML) + + if chosen.done() { + break + } + } + + content, isHTML := chosen.result() + + return content, isHTML, nil +} + +func readPart(part *multipart.Part, depth int) (string, bool, error) { + contentType := part.Header.Get("Content-Type") + slog.Debug("Processing email part", slog.String("content_type", contentType)) + + // mime.ParseMediaType hands back a usable media type alongside the error + // it reports for a malformed parameter — an empty charset, an unquoted + // file name — which older mailers emit often enough that dropping those + // parts would lose the message body itself. + if disposition, _, err := mime.ParseMediaType(part.Header.Get("Content-Disposition")); err == nil || + errors.Is(err, mime.ErrInvalidMediaParameter) { + if disposition == "attachment" { + // An attached .txt or .html file is something the sender enclosed, + // not what they wrote. + return "", false, nil + } + } + + // RFC 2045 §5.2 makes a part without a Content-Type plain US-ASCII text. + mediaType, params := "text/plain", map[string]string{} + + if contentType != "" { + var err error + + mediaType, params, err = mime.ParseMediaType(contentType) + if err != nil && !errors.Is(err, mime.ErrInvalidMediaParameter) { + slog.Warn("ignoring email part with an unparseable Content-Type", + slog.String("content_type", contentType), + slog.String("err", err.Error())) + + return "", false, nil + } + + if params == nil { + params = map[string]string{} + } + } + + if !strings.HasPrefix(mediaType, "multipart/") && + mediaType != "text/plain" && mediaType != "text/html" { + // text/calendar, an inline image, a signature: nothing to notify with. + return "", false, nil + } + + return readMediaType(part.Header, part, mediaType, params, depth+1) +} + +// decodeContent turns one entity's bytes into a UTF-8 string, undoing the +// transfer encoding and the character set the message declared. Without this a +// quoted-printable body reaches the target as "=E2=80=99" and a Latin-1 one as +// mojibake — and neither survives a trip through an HTML renderer. +func decodeContent(header textproto.MIMEHeader, body io.Reader, charsetLabel string) (string, error) { + raw, err := io.ReadAll(body) + if err != nil { + return "", fmt.Errorf("reading message body: %w", err) + } + + return decodeCharset(decodeTransferEncoding(raw, header.Get("Content-Transfer-Encoding")), charsetLabel), nil +} + +// decodeTransferEncoding is deliberately forgiving. A body that will not decode +// is still a body: forwarding the bytes as they arrived shows the reader +// something, where refusing the message tells the sender to stop retrying and +// loses the notification for good. +func decodeTransferEncoding(raw []byte, encoding string) []byte { + switch strings.ToLower(strings.TrimSpace(encoding)) { + case "base64": + return decodeBase64(raw) + + case "quoted-printable": + // mime/multipart already unwraps this for the parts it hands out and + // drops the header with it, so only a single-part message arrives + // here still encoded. + decoded, err := io.ReadAll(quotedprintable.NewReader(bytes.NewReader(raw))) + if err != nil { + slog.Warn("failed to decode a quoted-printable body, forwarding what could be decoded", + slog.String("err", err.Error())) + } + + if len(decoded) == 0 { + return raw + } + + return decoded + + default: + // 7bit, 8bit and binary are the identity encoding; anything else is + // something this server has no way to undo. + return raw + } +} + +// decodeBase64 tolerates the two liberties real mailers take with a base64 +// body: they wrap it across lines, and they leave the padding off the last +// group. +func decodeBase64(raw []byte) []byte { + compact := bytes.Map(func(r rune) rune { + if unicode.IsSpace(r) { + return -1 + } + + return r + }, raw) + + encoding := base64.StdEncoding + if len(compact)%4 != 0 { + encoding = base64.RawStdEncoding + } + + decoded, err := encoding.DecodeString(string(compact)) + if err != nil { + slog.Warn("failed to decode a base64 body, forwarding what could be decoded", + slog.String("err", err.Error())) + } + + if len(decoded) == 0 { + return raw + } + + return decoded +} + +func decodeCharset(raw []byte, label string) string { + switch strings.ToLower(strings.TrimSpace(label)) { + case "", "utf-8", "utf8", "us-ascii", "ascii": + return string(raw) + } + + encoding, _ := charset.Lookup(label) + if encoding == nil { + slog.Warn("unknown message charset, forwarding the body undecoded", + slog.String("charset", label)) + + return string(raw) + } + + decoded, err := encoding.NewDecoder().Bytes(raw) + if err != nil { + slog.Warn("failed to decode message charset, forwarding the body undecoded", + slog.String("charset", label), + slog.String("err", err.Error())) + + return string(raw) + } + + return string(decoded) +} + +// bodySelector picks between the forms of a multipart/alternative. Plain text +// beats HTML, and the first of each kind beats the rest, which is the order +// the message puts its own preference in. A blank part is held rather than +// chosen: senders that build the HTML from a template routinely emit an empty +// alternative beside it. +type bodySelector struct { + plain string + html string + hasPlain bool + hasHTML bool +} + +func (s *bodySelector) offer(content string, isHTML bool) { + if isHTML { + if !s.hasHTML || strings.TrimSpace(s.html) == "" { + s.html, s.hasHTML = content, true + } + + return + } + + if !s.hasPlain || strings.TrimSpace(s.plain) == "" { + s.plain, s.hasPlain = content, true + } +} + +func (s *bodySelector) empty() bool { + return !s.hasPlain && !s.hasHTML +} + +// done reports that nothing later in the container can improve on what has +// been found. +func (s *bodySelector) done() bool { + return s.hasPlain && strings.TrimSpace(s.plain) != "" +} + +func (s *bodySelector) result() (string, bool) { + if s.done() || !s.hasHTML { + return s.plain, false + } + + return s.html, true +} diff --git a/email_test.go b/email_test.go index 9a4c9db..805541d 100644 --- a/email_test.go +++ b/email_test.go @@ -1,9 +1,11 @@ package smtp2shoutrrr import ( + "encoding/base64" "net/mail" "strings" "testing" + "unicode/utf8" "github.com/stretchr/testify/require" ) @@ -88,3 +90,419 @@ func TestBodyFallsBackToHTMLPart(t *testing.T) { require.NoError(t, err) require.Equal(t, "

html body

", body) } + +func TestBodyDecodesQuotedPrintable(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + "Content-Type: text/plain; charset=utf-8", + "Content-Transfer-Encoding: quoted-printable", + "", + "caf=C3=A9 =E2=80=94 a very long line that the sender wrapped =", + "here", + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "café — a very long line that the sender wrapped here\r\n", body) +} + +func TestBodyDecodesBase64(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + "Content-Type: text/plain; charset=utf-8", + "Content-Transfer-Encoding: base64", + "", + base64.StdEncoding.EncodeToString([]byte("café — encoded")), + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "café — encoded", body) +} + +// Windows-1252 and Latin-1 are still what a good deal of automated mail is +// written in, and their bytes are not valid UTF-8. +func TestBodyDecodesNonUTF8Charset(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, + "Subject: Test\r\nContent-Type: text/plain; charset=iso-8859-1\r\n\r\ncaf\xe9\r\n")} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "café\r\n", body) + require.True(t, utf8.ValidString(body)) +} + +func TestBodyUnknownCharsetIsForwardedUndecoded(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, + "Subject: Test\r\nContent-Type: text/plain; charset=not-a-charset\r\n\r\nbody\r\n")} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "body\r\n", body) +} + +// A forwarded message is a multipart/mixed wrapping the multipart/alternative +// that holds what the sender actually wrote. Stopping at the outer container +// left the notification empty. +func TestBodyDescendsIntoNestedMultipart(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/mixed; boundary="outer"`, + "", + "--outer", + `Content-Type: multipart/alternative; boundary="inner"`, + "", + "--inner", + "Content-Type: text/html; charset=utf-8", + "", + "

html body

", + "--inner", + "Content-Type: text/plain; charset=utf-8", + "", + "plain body", + "--inner--", + "--outer--", + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "plain body", body) +} + +func TestBodySkipsAttachments(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/mixed; boundary="b"`, + "", + "--b", + "Content-Type: text/plain; charset=utf-8", + `Content-Disposition: attachment; filename="notes.txt"`, + "", + "an enclosed file, not the message", + "--b", + "Content-Type: text/plain; charset=utf-8", + "", + "the message itself", + "--b--", + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "the message itself", body) +} + +// Senders that build the HTML alternative from a template routinely emit an +// empty text/plain beside it. +func TestBodyPrefersHTMLOverAnEmptyPlainTextPart(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/alternative; boundary="b"`, + "", + "--b", + "Content-Type: text/plain; charset=utf-8", + "", + " ", + "--b", + "Content-Type: text/html; charset=utf-8", + "", + "

html body

", + "--b--", + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "

html body

", body) +} + +func TestBodyRejectsMultipartWithoutBoundary(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, + "Subject: Test\r\nContent-Type: multipart/alternative\r\n\r\nbody\r\n")} + + _, err := email.Body() + require.ErrorIs(t, err, errMalformedMessage) +} + +func TestFormattedBodyConvertsHTML(t *testing.T) { + raw := strings.Join([]string{ + "Subject: Test", + "Content-Type: text/html; charset=utf-8", + "Content-Transfer-Encoding: quoted-printable", + "", + "

Build failed: run 42

", + "", + }, "\r\n") + + for format, want := range map[BodyFormat]string{ + FormatMarkdown: "Build **failed**: [run 42](https://ci.example.com/42)", + FormatRaw: "

Build failed: run 42

\r\n", + } { + t.Run(string(format), func(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, raw)} + + body, err := email.FormattedBody(format) + require.NoError(t, err) + require.Equal(t, want, body) + }) + } +} + +// A recipient asking for Markdown is asking for HTML to stop reaching it, not +// for the plain text a sender wrote to be run through a renderer. +func TestFormattedBodyLeavesPlainTextAlone(t *testing.T) { + raw := "Subject: Test\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n2 * 3 = 6 \r\n" + + for _, format := range []BodyFormat{FormatRaw, FormatMarkdown} { + t.Run(string(format), func(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, raw)} + + body, err := email.FormattedBody(format) + require.NoError(t, err) + require.Equal(t, "2 * 3 = 6 \r\n", body) + }) + } +} + +// The zero value of the option means the same as raw, so a recipient built in +// code rather than loaded from a file still forwards what it received. +func TestFormattedBodyTreatsUnsetFormatAsRaw(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, + "Subject: Test\r\nContent-Type: text/html\r\n\r\n

html body

\r\n")} + + body, err := email.FormattedBody("") + require.NoError(t, err) + require.Equal(t, "

html body

\r\n", body) +} + +// Reformatting is a courtesy, so a body the renderer cannot use is still +// forwarded rather than costing the reader the notification. +func TestFormattedBodyFallsBackToTheRawHTML(t *testing.T) { + // The parser refuses a document with more than 512 elements open at once. + tooDeep := strings.Repeat("
", 600) + "deep" + strings.Repeat("
", 600) + + for name, source := range map[string]string{ + "a body the parser refuses": tooDeep, + // Otherwise shoutrrr is handed "", which Mattermost and Discord + // reject, so every target fails and the sender retries the same + // message forever. + "a body that renders to nothing": ``, + } { + t.Run(name, func(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, + "Subject: Test\r\nContent-Type: text/html\r\n\r\n"+source)} + + body, err := email.FormattedBody(FormatMarkdown) + require.NoError(t, err) + require.Equal(t, source, body) + }) + } +} + +func TestBodyIsReadOnlyOnce(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, + "Subject: Test\r\nContent-Type: text/html\r\n\r\n

body

\r\n")} + + first, err := email.FormattedBody(FormatMarkdown) + require.NoError(t, err) + + second, err := email.FormattedBody(FormatMarkdown) + require.NoError(t, err) + + require.Equal(t, "body", first) + require.Equal(t, first, second, "the message body is consumed as it is read") +} + +// Regression: an unreadable transfer encoding used to be reported as a +// permanent 550, which tells the sender to stop retrying and loses a +// notification that main delivered. Real mailers emit unpadded base64. +func TestBodyToleratesAnUndecodableTransferEncoding(t *testing.T) { + for name, tc := range map[string]struct{ raw, want string }{ + "unpadded base64": { + raw: base64.RawStdEncoding.EncodeToString([]byte("hello world")), + want: "hello world", + }, + "base64 that is not base64 at all": { + raw: "!!!! not base64 !!!!", + want: "!!!! not base64 !!!!", + }, + } { + t.Run(name, func(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, + "Subject: Test\r\nContent-Type: text/plain\r\n"+ + "Content-Transfer-Encoding: base64\r\n\r\n"+tc.raw+"\r\n")} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, tc.want, strings.TrimSpace(body)) + }) + } +} + +// Regression: one part that would not decode used to fail the whole message, +// even with a perfectly good alternative already in hand. +func TestBodyKeepsAGoodPartBesideAnUnreadableOne(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/alternative; boundary="b"`, + "", + "--b", + "Content-Type: text/html", + "Content-Transfer-Encoding: base64", + "", + "!!!!not base64!!!!", + "--b", + "Content-Type: text/plain", + "", + "the real body", + "--b--", + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "the real body", body) +} + +// Regression: mime.ParseMediaType returns a usable media type alongside the +// error it reports for a malformed parameter. Discarding both dropped the part +// that held the message and delivered an empty notification with a 250. +func TestBodyReadsPartsWithAMalformedContentType(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/alternative; boundary="b"`, + "", + "--b", + "Content-Type: text/plain; charset=", + "", + "still text", + "--b--", + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "still text", body) +} + +// The same fault the other way round: the attachment guard used to fail open +// on the unquoted file names older mailers emit, so the enclosure won. +func TestBodySkipsAttachmentsWithAMalformedDisposition(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/mixed; boundary="b"`, + "", + "--b", + "Content-Type: text/plain", + "Content-Disposition: attachment; filename=log file.txt", + "", + "an enclosed file, not the message", + "--b", + "Content-Type: text/plain", + "", + "the message itself", + "--b--", + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "the message itself", body) +} + +// RFC 2046 §5.1.3: the parts of a multipart/mixed are cumulative, so the first +// one carrying a body is the message and the rest are footers and enclosures. +// Applying "plain beats HTML" across them handed list mail its unsubscribe +// footer as the notification. +func TestBodyPrefersTheFirstPartOfACumulativeMultipart(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/mixed; boundary="b"`, + "", + "--b", + "Content-Type: text/html", + "", + "

the newsletter

", + "--b", + "Content-Type: text/plain", + "", + "You are receiving this because you subscribed.", + "--b--", + "", + }, "\r\n"))} + + body, err := email.FormattedBody(FormatMarkdown) + require.NoError(t, err) + require.Equal(t, "the newsletter", body, + "the HTML body is the message, and asking for Markdown must reach it") +} + +func TestBodyPassesOverAnEmptyHTMLAlternative(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/alternative; boundary="b"`, + "", + "--b", + "Content-Type: text/html", + "", + " ", + "--b", + "Content-Type: text/html", + "", + "

the real one

", + "--b--", + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "

the real one

", body) +} + +// A container this server cannot finish reading still hands over the parts it +// read. Only a message that yielded nothing at all is refused, since there is +// then nothing left to notify with and the same bytes fail the same way on +// every redelivery. +func TestBodyKeepsWhatItReadFromABrokenContainer(t *testing.T) { + t.Run("a good part before the break is delivered", func(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/mixed; boundary="b"`, + "", + "--b", + "Content-Type: text/plain", + "", + "good body", + "--b", + "this is not a header", + "", + "x", + "--b--", + "", + }, "\r\n"))} + + body, err := email.Body() + require.NoError(t, err) + require.Equal(t, "good body", body) + }) + + t.Run("a break before anything readable is permanent", func(t *testing.T) { + email := ReceivedEmail{Msg: readMessage(t, strings.Join([]string{ + "Subject: Test", + `Content-Type: multipart/alternative; boundary="b"`, + "", + "--b", + "this is not a header", + "", + "body", + "--b--", + "", + }, "\r\n"))} + + _, err := email.Body() + require.ErrorIs(t, err, errMalformedMessage) + }) +} diff --git a/format.go b/format.go new file mode 100644 index 0000000..f7ed018 --- /dev/null +++ b/format.go @@ -0,0 +1,49 @@ +package smtp2shoutrrr + +import ( + "slices" + "strings" +) + +// BodyFormat names the representation a recipient's targets want the message +// body in. Only an HTML body is ever rewritten: a message that already arrived +// as plain text is forwarded untouched whatever the recipient asked for. +type BodyFormat string + +const ( + // FormatRaw forwards the body exactly as the message carried it. + FormatRaw BodyFormat = "raw" + // FormatMarkdown renders an HTML body as Markdown. There is deliberately + // no plain-text format beside it: Markdown reads as plain text wherever + // nothing renders it, so a second conversion would only be a worse copy + // of this one. + FormatMarkdown BodyFormat = "markdown" +) + +var bodyFormats = []BodyFormat{FormatRaw, FormatMarkdown} + +// normalize maps an unset Format to the one that changes nothing, and accepts +// the casing a hand-written configuration file is likely to use. +func (f BodyFormat) normalize() BodyFormat { + normalized := BodyFormat(strings.ToLower(strings.TrimSpace(string(f)))) + if normalized == "" { + return FormatRaw + } + + return normalized +} + +// valid normalizes first, so a Config assembled in Go rather than loaded from +// a file does not fail validation on a Format nobody set. +func (f BodyFormat) valid() bool { + return slices.Contains(bodyFormats, f.normalize()) +} + +func formatNames() string { + names := make([]string, 0, len(bodyFormats)) + for _, format := range bodyFormats { + names = append(names, string(format)) + } + + return strings.Join(names, ", ") +} diff --git a/go.mod b/go.mod index 0892ec1..5ce3be2 100644 --- a/go.mod +++ b/go.mod @@ -5,15 +5,18 @@ go 1.27 toolchain go1.27.1 require ( + github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 github.com/containrrr/shoutrrr v0.8.0 github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 github.com/emersion/go-smtp v0.25.0 github.com/pelletier/go-toml/v2 v2.4.3 github.com/stretchr/testify v1.12.1 golang.org/x/crypto/x509roots/fallback v0.0.0-20260902180247-86efde54dc70 + golang.org/x/net v0.58.0 ) require ( + github.com/JohannesKaufmann/dom v0.3.1 // indirect github.com/fatih/color v1.19.0 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -21,7 +24,7 @@ require ( github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.24 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.58.0 // indirect golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/tools v0.49.0 // indirect ) diff --git a/go.sum b/go.sum index 5b3ff23..d052506 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/JohannesKaufmann/dom v0.3.1 h1:J16l9JAHWgkFPR3VIPbQ1gvS0cWab6laK1q7PFL3qh0= +github.com/JohannesKaufmann/dom v0.3.1/go.mod h1:BZPkf8ZeYrBgABjwJn9iiKt8aiCtkxpHkevms+Yp2DE= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 h1:XFJZFWESIWlUEHHjzBuv8RvrtCWnSGlimEX17ysSDb8= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2/go.mod h1:BHWO8lJzttJLqwuV8Rb1B3OG2OSzLbssZDI1FRg2eAA= github.com/containrrr/shoutrrr v0.8.0 h1:mfG2ATzIS7NR2Ec6XL+xyoHzN97H8WPjir8aYzJUSec= github.com/containrrr/shoutrrr v0.8.0/go.mod h1:ioyQAyu1LJY6sILuNyKaQaw+9Ttik5QePU8atnAdO2o= github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk= @@ -28,8 +32,16 @@ github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= +github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= +github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto/x509roots/fallback v0.0.0-20260902180247-86efde54dc70 h1:VwViOGcd7C8/Gs18efVlMxvUCS0un6KRKBPFEBvpUXg= diff --git a/html.go b/html.go new file mode 100644 index 0000000..0325b5c --- /dev/null +++ b/html.go @@ -0,0 +1,400 @@ +package smtp2shoutrrr + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "github.com/JohannesKaufmann/html-to-markdown/v2/converter" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/strikethrough" + "golang.org/x/net/html" + "golang.org/x/net/html/atom" +) + +const ( + // maxNestingDepth bounds the elements that give every line they contain a + // prefix. The converter re-emits that prefix per line and per level, so + // depth multiplies against line count: a message at the server's own 1 MB + // limit, nested 250 quotes deep, renders to 131 MB and takes minutes. + // Real mail quotes a handful of levels and indents fewer. + maxNestingDepth = 6 + + // maxRenderedBytes caps what is forwarded. No chat target displays this + // much, and a notification is not the place to find out. + maxRenderedBytes = 64 << 10 + + // truncationMarker tells the reader the message goes on, rather than + // letting the cap end it mid-sentence and look like the whole of it. + truncationMarker = "…" +) + +// renderMarkdown rewrites an HTML body as Markdown. +// +// The conversion itself belongs to html-to-markdown, which knows CommonMark — +// escaping, delimiter runs, fencing a code block past the backticks inside it. +// What it has no opinion about is mail, since it is written for documents: a +// document has no preheader written for an inbox list, no tracking pixel, no +// cid: attachment, and lays nothing out in tables. That is what prepare does +// to the tree first, so the converter only ever sees what a notification +// should carry. +func renderMarkdown(source string) (string, error) { + document, err := html.Parse(strings.NewReader(source)) + if err != nil { + // The parser recovers from any markup it can hold, so this is a + // document too deeply nested for it rather than an invalid one. + return "", fmt.Errorf("parsing HTML body: %w", err) + } + + prepare(document, 0) + + rendered, err := newConverter().ConvertNode(document) + if err != nil { + return "", fmt.Errorf("converting HTML body: %w", err) + } + + return truncate(strings.TrimSpace(string(rendered))), nil +} + +// newConverter builds a converter per message, which is what the library's own +// entry points do: a Converter carries the error of the conversion it is +// running, and go-smtp serves every connection on its own goroutine. +func newConverter() *converter.Converter { + return converter.NewConverter( + converter.WithPlugins( + base.NewBasePlugin(), + commonmark.NewCommonmarkPlugin( + // The default is "* * *", which reads as a stray line of + // asterisks anywhere the Markdown is not rendered. + commonmark.WithHorizontalRule("---"), + ), + strikethrough.NewStrikethroughPlugin(), + ), + ) +} + +func prepare(node *html.Node, depth int) { + child := node.FirstChild + + for child != nil { + next := child.NextSibling + + switch child.Type { + case html.CommentNode: + node.RemoveChild(child) + + case html.TextNode: + child.Data = sanitizeText(child.Data) + + case html.ElementNode: + if unwrapped := prepareElement(node, child, depth); unwrapped != nil { + // The element was replaced by its own children, which have + // not been looked at yet. + next = unwrapped + } + } + + child = next + } +} + +func prepareElement(parent, node *html.Node, depth int) *html.Node { + if isHidden(node) { + // A template opens with a hidden block holding the preview line the + // inbox shows, which is written for the list view and reads as noise + // anywhere else. + parent.RemoveChild(node) + + return nil + } + + switch node.DataAtom { + case atom.Img: + return prepareImage(parent, node) + + case atom.A: + href := sanitizeURL(attrValue(node, "href")) + if !isUsableURL(href) { + // In-page anchors, script handlers and the message's own inline + // attachments give a notification's reader nothing to open, but + // the text of the link is still part of the message. + return unwrap(parent, node) + } + + setAttr(node, "href", href) + prepare(node, depth) + + if !hasContent(node) { + // A link wrapped around a tracking pixel, or around the spacer + // that mail templates use for one, would render as "[](url)" — + // invisible to the reader. Its destination is the only thing left + // worth forwarding. + node.AppendChild(&html.Node{Type: html.TextNode, Data: href}) + } + + return nil + + case atom.Td, atom.Th: + // Mail is laid out in tables far more often than it tabulates + // anything, and a converter with no table rules runs the cells of a + // row together: "Total4Failed0". + parent.InsertBefore(&html.Node{Type: html.TextNode, Data: " "}, node) + + case atom.Tr: + if hasElementSibling(node) { + parent.InsertBefore(&html.Node{ + Type: html.ElementNode, DataAtom: atom.Br, Data: "br", + }, node) + } + + case atom.Blockquote, atom.Ul, atom.Ol: + depth++ + if depth > maxNestingDepth { + return unwrap(parent, node) + } + } + + prepare(node, depth) + + return nil +} + +// prepareImage keeps of an image only what a notification can use. Tracking +// pixels, spacers and sliced-up banners carry no alt text and go entirely; an +// inline attachment has no address the reader can fetch, so its alt text +// stays behind as ordinary words. +func prepareImage(parent, node *html.Node) *html.Node { + alt := collapseSpace(sanitizeText(attrValue(node, "alt"))) + source := sanitizeURL(attrValue(node, "src")) + + switch { + case alt == "": + parent.RemoveChild(node) + case !isUsableURL(source): + parent.InsertBefore(&html.Node{Type: html.TextNode, Data: alt}, node) + parent.RemoveChild(node) + default: + setAttr(node, "alt", alt) + setAttr(node, "src", source) + } + + return nil +} + +// unwrap replaces a node with its own children and returns the first of them. +func unwrap(parent, node *html.Node) *html.Node { + first := node.FirstChild + + for child := node.FirstChild; child != nil; child = node.FirstChild { + node.RemoveChild(child) + parent.InsertBefore(child, node) + } + + parent.RemoveChild(node) + + return first +} + +// hasContent reports whether anything left under a node would show. It runs +// after the subtree has been prepared, so an element that survived that is one +// the reader will see. +func hasContent(node *html.Node) bool { + for child := node.FirstChild; child != nil; child = child.NextSibling { + switch child.Type { + case html.ElementNode: + if child.DataAtom == atom.Img || hasContent(child) { + return true + } + case html.TextNode: + if strings.TrimSpace(child.Data) != "" { + return true + } + } + } + + return false +} + +func hasElementSibling(node *html.Node) bool { + for sibling := node.PrevSibling; sibling != nil; sibling = sibling.PrevSibling { + if sibling.Type == html.ElementNode { + return true + } + } + + return false +} + +func truncate(rendered string) string { + if len(rendered) <= maxRenderedBytes { + return rendered + } + + cut := maxRenderedBytes + for cut > 0 && !utf8.RuneStart(rendered[cut]) { + cut-- + } + + return strings.TrimRight(rendered[:cut], " \t\n") + "\n" + truncationMarker +} + +// isUsableURL rejects the destinations a notification's reader cannot act on: +// in-page anchors, inline data, script handlers and the message's own inline +// attachments. +func isUsableURL(raw string) bool { + if raw == "" || strings.HasPrefix(raw, "#") { + return false + } + + scheme, _, found := strings.Cut(raw, ":") + if !found || !isScheme(scheme) { + return true + } + + switch strings.ToLower(scheme) { + case "javascript", "data", "cid", "about", "blob": + return false + } + + return true +} + +func isScheme(s string) bool { + if s == "" || !unicode.IsLetter(rune(s[0])) { + return false + } + + return strings.IndexFunc(s, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) && + r != '+' && r != '-' && r != '.' + }) < 0 +} + +// sanitizeURL applies to a URL attribute what a browser applies before it +// fetches one: the tabs and line breaks that may sit inside the value are +// removed. Left in, they end the line the notification is on, which is a place +// to write a sentence the reader will take for the sender's. +func sanitizeURL(raw string) string { + return strings.Map(func(r rune) rune { + if r == ' ' { + return ' ' + } + + if unicode.IsSpace(r) || unicode.IsControl(r) { + return -1 + } + + return r + }, strings.TrimSpace(raw)) +} + +// collapseSpace squeezes the runs of whitespace an attribute value may hold +// into single spaces, for the same reason: alt text spans lines in the source +// and must not span them in the message. +func collapseSpace(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +// sanitizeText turns the characters mail uses for layout into ones a +// notification can show: the non-breaking spaces that hold table cells apart +// become ordinary spaces, and the zero-width padding that hides a preheader +// from the inbox preview is dropped rather than forwarded invisibly. +func sanitizeText(s string) string { + return strings.Map(func(r rune) rune { + switch r { + case zeroWidthSpace, zeroWidthNonJoiner, zeroWidthJoiner, + wordJoiner, byteOrderMark, softHyphen: + return -1 + case noBreakSpace, narrowNoBreakSpace, figureSpace: + return ' ' + } + + if unicode.IsSpace(r) { + return r + } + + // Control characters have no rendering of their own and only muddle + // the payloads the targets are sent as. + if unicode.IsControl(r) { + return -1 + } + + return r + }, s) +} + +// The characters mail uses for layout rather than for words: padding that +// hides a preheader from the inbox preview, and the spaces that hold table +// cells apart without letting them wrap. +const ( + softHyphen = '\u00ad' + noBreakSpace = '\u00a0' + zeroWidthSpace = '\u200b' + zeroWidthNonJoiner = '\u200c' + zeroWidthJoiner = '\u200d' + narrowNoBreakSpace = '\u202f' + wordJoiner = '\u2060' + figureSpace = '\u2007' + byteOrderMark = '\ufeff' +) + +// isHidden reports whether a browser would leave the element out of the page. +func isHidden(node *html.Node) bool { + for _, attr := range node.Attr { + switch attr.Key { + case "hidden": + return true + case "style": + if isHidingStyle(spaceStripper.Replace(strings.ToLower(attr.Val))) { + return true + } + } + } + + return false +} + +func isHidingStyle(style string) bool { + for _, declaration := range []string{"display:none", "visibility:hidden", "mso-hide:all"} { + if strings.Contains(style, declaration) { + return true + } + } + + // font-size:0 hides a preheader too, but only when the zero is the whole + // value: font-size:0.9em is an ordinary line of text. + if index := strings.Index(style, "font-size:0"); index >= 0 { + rest := strings.TrimPrefix(style[index+len("font-size:0"):], "px") + rest = strings.TrimPrefix(rest, "pt") + + return rest == "" || strings.HasPrefix(rest, ";") + } + + return false +} + +var spaceStripper = strings.NewReplacer(" ", "", "\t", "", "\n", "", "\r", "") + +func attrValue(node *html.Node, key string) string { + for _, attr := range node.Attr { + if attr.Key == key { + return attr.Val + } + } + + return "" +} + +func setAttr(node *html.Node, key, value string) { + for i := range node.Attr { + if node.Attr[i].Key == key { + node.Attr[i].Val = value + + return + } + } + + node.Attr = append(node.Attr, html.Attribute{Key: key, Val: value}) +} diff --git a/html_test.go b/html_test.go new file mode 100644 index 0000000..4bbdaf9 --- /dev/null +++ b/html_test.go @@ -0,0 +1,248 @@ +package smtp2shoutrrr + +import ( + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/require" +) + +func render(t *testing.T, source string) string { + t.Helper() + + out, err := renderMarkdown(source) + require.NoError(t, err) + + return out +} + +// The conversion itself belongs to html-to-markdown; what these pin is the +// contract this package keeps on top of it, which is what a notification +// should carry rather than what the document said. +func TestRenderMarkdown(t *testing.T) { + for name, tc := range map[string]struct{ source, want string }{ + "paragraphs are separated by a blank line": { + source: "

first

second

", + want: "first\n\nsecond", + }, + "headings keep their level": { + source: "

one

three

", + want: "# one\n\n### three", + }, + "emphasis": { + source: "

bold italic gone x=1

", + want: "**bold** *italic* ~~gone~~ `x=1`", + }, + "two spans of the same emphasis meeting head on become one": { + // What every Word and Outlook export does to a bolded label. + source: "

Total: 49.99

", + want: "**Total: 49.99**", + }, + "emphasis nested inside itself does not double its markers": { + source: "

ab

", + want: "**ab**", + }, + "links keep their destination": { + source: `

see the docs

`, + want: "see [the docs](https://example.com/x)", + }, + "a link with nothing to click on falls back to its destination": { + // Otherwise a link wrapped around a tracking pixel renders as + // "[](url)", which the reader cannot see at all. + source: `

`, + want: "[https://example.com/x](https://example.com/x)", + }, + "a link around nothing but a spacer falls back the same way": { + source: `

`, + want: "[https://example.com/x](https://example.com/x)", + }, + "unusable destinations are dropped but their text is kept": { + source: `up click img`, + want: "up click img", + }, + "line breaks inside a URL are removed rather than forwarded": { + // A newline in an href is invisible in the document and a + // fabricated line in the notification. + source: "open report", + want: "[open report](https://ok.example/rURGENT:%20wire%20funds)", + }, + "unordered lists": { + source: "
  • one
  • two
", + want: "- one\n- two", + }, + "ordered lists count from their start attribute": { + source: `
  1. three
  2. four
`, + want: "3. three\n4. four", + }, + "blockquotes mark every line": { + source: "

before

quoted

after

", + want: "before\n\n> quoted\n\nafter", + }, + "a code block is fenced past the backticks it holds": { + // Otherwise the sender closes the block and everything after it — + // a heading, a link — renders as live Markdown in a message the + // reader takes for a forwarded notification. + source: "
ok\n```\n## Your account is locked\n
", + want: "````\nok\n```\n## Your account is locked\n````", + }, + "table rows become lines and cells stay apart": { + // Mail lays itself out in tables far more often than it tabulates + // anything, and a converter with no table rules would otherwise + // run a row together as "Total4Failed0". + source: "
Total4
Failed0
", + want: "Total 4 \nFailed 0", + }, + "images without alt text are dropped": { + source: `

ab

`, + want: "ab", + }, + "images with alt text are kept": { + source: `Logo`, + want: "![Logo](https://x.example/logo.png)", + }, + "an inline attachment keeps its alt text without a destination": { + source: `Chart`, + want: "Chart", + }, + "alt text is collapsed onto one line": { + source: "\"a\n", + want: "![a long alt](https://x.example/i.png)", + }, + "script and style content never reaches the reader": { + source: "

body

", + want: "body", + }, + "hidden preheaders are left out": { + source: `
inbox preview

real body

`, + want: "real body", + }, + "the other ways a preheader hides are recognised too": { + source: `
a
b
` + + `
c
d

body

`, + want: "d\n\nbody", + }, + "entities and non-breaking spaces become ordinary text": { + source: "

4m 12s & counting — done

", + want: "4m 12s & counting — done", + }, + "zero width padding is dropped": { + source: "

​a​b​

", + want: "ab", + }, + "text that looks like markup is escaped": { + source: "

2 * 3, a [b] c, `tick`

", + want: "2 * 3, a \\[b] c, \\`tick\\`", + }, + "markers are only defused where they would take effect": { + source: "

- not a bullet

# not a heading

1. not a list

a - b

", + want: "\\- not a bullet\n\n\\# not a heading\n\n1\\. not a list\n\na - b", + }, + "a document with nothing to say renders to nothing": { + source: ``, + want: "", + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tc.want, render(t, tc.source)) + }) + } +} + +// The parser recovers from anything, so the conversion has to as well: a +// notification is worth more than a report that the markup was invalid. +func TestRenderMarkdownSurvivesBrokenMarkup(t *testing.T) { + for name, tc := range map[string]struct{ source, want string }{ + "unclosed tags": {"

boldboth

next", "**bold*both***\n\n***next***"}, + "stray closing tags": {"

text", "text"}, + "no markup at all": {"just some words", "just some words"}, + "an empty document": {"", ""}, + "an unterminated tag": {"

text<", "text<"}, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tc.want, render(t, tc.source)) + }) + } +} + +// Regression: a line prefix is re-emitted on every line of every level it +// nests, so depth multiplies against line count. One message at the server's +// own 1 MB limit rendered to 131 MB over two and a quarter minutes — the same +// class of fault as the remote DoS closed in 7565618. Neither the converter +// nor any of the alternatives bounds this on its own. +func TestRenderMarkdownBoundsPathologicalInput(t *testing.T) { + // Sized to overrun the cap several times over once the nesting is + // flattened, rather than to reproduce the original 1 MB message: the + // assertion is the same and the suite stays quick. + lines := strings.Repeat("

x

", maxRenderedBytes/maxNestingDepth) + + for name, source := range map[string]string{ + "quotes nested far past anything real": strings.Repeat("
", 250) + lines, + "lists nested far past anything real": strings.Repeat("
  • ", 250) + lines, + } { + t.Run(name, func(t *testing.T) { + out, err := renderMarkdown(source) + require.NoError(t, err) + require.LessOrEqual(t, len(out), maxRenderedBytes+len(truncationMarker)+1) + require.True(t, strings.HasSuffix(out, truncationMarker), + "a capped message says so rather than ending mid-sentence") + require.True(t, utf8.ValidString(out), "the cap must not fall inside a rune") + }) + } +} + +// Nesting under the limit is left alone, so an ordinary quoted reply still +// reads as one. +func TestRenderMarkdownKeepsNestingUnderTheLimit(t *testing.T) { + source := strings.Repeat("
    ", maxNestingDepth) + "

    deep

    " + + strings.Repeat("
    ", maxNestingDepth) + + require.Equal(t, strings.Repeat("> ", maxNestingDepth)+"deep", render(t, source)) +} + +// A realistic transactional message, pinned end to end. The faults this +// feature has had lived in combinations of features rather than in one of +// them, which is what a fixture catches and a unit test does not. +func TestRenderMarkdownOnARealisticMessage(t *testing.T) { + require.Equal(t, strings.Join([]string{ + "[![Example Billing](https://cdn.example.com/logo.png)](https://billing.example.com)", + "", + "## Invoice #2026-0231", + "", + "Hi Ferran,", + "", + "Your invoice for **February 2026 (30 days)** is ready. The total is " + + "**€42.00**, charged to the card ending 4242 on *1 March*.", + "", + "Description Qty Amount ", + "Hosting — small 1 €30.00 ", + "Backups 2 €12.00", + "", + "Reference: `inv_2026*0231` ", + "[View invoice](https://billing.example.com/invoices/2026-0231?utm_source=email&utm_medium=cta)", + "", + "You are receiving this because you have an account. " + + "[Unsubscribe](https://billing.example.com/unsubscribe?t=abc) · Back to top", + }, "\n"), render(t, marketingMessage)) +} + +// Shaped like the transactional mail this feature exists for: a hidden +// preheader, a layout table, a logo wrapped in a link, an Outlook-style split +// bold run, a tracking pixel and an in-page anchor. +const marketingMessage = `Invoice + +Your February invoice is ready​​ +
    + + + + + + + +
    Example Billing

    Invoice #2026-0231

    Hi Ferran,

    Your invoice for February 2026 (30 days) is ready. +The total is €42.00, charged to the card ending 4242 on 1 March.
    + +
    DescriptionQtyAmount
    Hosting — small1€30.00
    Backups2€12.00
    Reference: inv_2026*0231
    View invoice

    You are receiving this because you have an account. +Unsubscribe · Back to top

    +
    `