108 lines
2.9 KiB
Go
108 lines
2.9 KiB
Go
package extractors
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
readability "codeberg.org/readeck/go-readability"
|
|
md "github.com/JohannesKaufmann/html-to-markdown"
|
|
)
|
|
|
|
// ReadableExtractor extracts readable content from HTML files using go-readability
|
|
type ReadableExtractor struct {
|
|
available bool
|
|
initError error
|
|
}
|
|
|
|
// NewReadableExtractor creates a new readable extractor
|
|
func NewReadableExtractor() *ReadableExtractor {
|
|
return &ReadableExtractor{}
|
|
}
|
|
|
|
// Key returns the unique identifier for this extractor
|
|
func (e *ReadableExtractor) Key() string {
|
|
return "readable"
|
|
}
|
|
|
|
// Name returns the human-readable name for this extractor
|
|
func (e *ReadableExtractor) Name() string {
|
|
return "Readable Content Extractor"
|
|
}
|
|
|
|
// SupportedMimeTypes returns the MIME types this extractor supports
|
|
func (e *ReadableExtractor) SupportedMimeTypes() []string {
|
|
return []string{"text/html", "application/xhtml+xml"}
|
|
}
|
|
|
|
// Init initializes the extractor
|
|
// Since go-readability is a Go library, this should always succeed
|
|
func (e *ReadableExtractor) Init() error {
|
|
e.available = true
|
|
e.initError = nil
|
|
return nil
|
|
}
|
|
|
|
// Extract extracts readable content from an HTML file and saves it as markdown
|
|
func (e *ReadableExtractor) Extract(ctx context.Context, filePath string) (string, error) {
|
|
if !e.available {
|
|
return "", fmt.Errorf("readable extractor is not available: %w", e.initError)
|
|
}
|
|
|
|
// Open the HTML file
|
|
file, err := os.Open(filePath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to open file: %w", err)
|
|
}
|
|
defer func() { _ = file.Close() }()
|
|
|
|
// Parse base URL from file path or use a default
|
|
// Extract directory to construct a file:// URL
|
|
baseURL, err := url.Parse("https://example.com")
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to parse base URL: %w", err)
|
|
}
|
|
|
|
// Extract readable content using go-readability
|
|
article, err := readability.FromReader(file, baseURL)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to extract readable content: %w", err)
|
|
}
|
|
|
|
// Try to convert HTML content to markdown
|
|
var content string
|
|
converter := md.NewConverter("", true, nil)
|
|
markdownContent, err := converter.ConvertString(article.Content)
|
|
if err != nil {
|
|
// If markdown conversion fails, fall back to plain text
|
|
content = article.TextContent
|
|
} else {
|
|
content = markdownContent
|
|
}
|
|
|
|
// Save markdown/text file next to HTML file
|
|
// Replace extension with .md
|
|
dir := filepath.Dir(filePath)
|
|
baseName := filepath.Base(filePath)
|
|
ext := filepath.Ext(baseName)
|
|
markdownPath := filepath.Join(dir, strings.TrimSuffix(baseName, ext)+".md")
|
|
|
|
if err := os.WriteFile(markdownPath, []byte(content), 0644); err != nil {
|
|
return "", fmt.Errorf("failed to write markdown file: %w", err)
|
|
}
|
|
|
|
return content, nil
|
|
}
|
|
|
|
// GetInitError returns the initialization error if any
|
|
func (e *ReadableExtractor) GetInitError() error {
|
|
return e.initError
|
|
}
|
|
|
|
// IsAvailable returns whether the extractor is available
|
|
func (e *ReadableExtractor) IsAvailable() bool {
|
|
return e.available
|
|
}
|