package extractors import ( "context" "fmt" "net/url" "os" "path/filepath" "strings" readability "codeberg.org/readeck/go-readability" md "github.com/JohannesKaufmann/html-to-markdown" ) // ReadableExtractor extracts readable content from HTML files using go-readability type ReadableExtractor struct { available bool initError error } // NewReadableExtractor creates a new readable extractor func NewReadableExtractor() *ReadableExtractor { return &ReadableExtractor{} } // Key returns the unique identifier for this extractor func (e *ReadableExtractor) Key() string { return "readable" } // Name returns the human-readable name for this extractor func (e *ReadableExtractor) Name() string { return "Readable Content Extractor" } // SupportedMimeTypes returns the MIME types this extractor supports func (e *ReadableExtractor) SupportedMimeTypes() []string { return []string{"text/html", "application/xhtml+xml"} } // Init initializes the extractor // Since go-readability is a Go library, this should always succeed func (e *ReadableExtractor) Init() error { e.available = true e.initError = nil return nil } // Extract extracts readable content from an HTML file and saves it as markdown func (e *ReadableExtractor) Extract(ctx context.Context, filePath string) (string, error) { if !e.available { return "", fmt.Errorf("readable extractor is not available: %w", e.initError) } // Open the HTML file file, err := os.Open(filePath) if err != nil { return "", fmt.Errorf("failed to open file: %w", err) } defer func() { _ = file.Close() }() // Parse base URL from file path or use a default // Extract directory to construct a file:// URL baseURL, err := url.Parse("https://example.com") if err != nil { return "", fmt.Errorf("failed to parse base URL: %w", err) } // Extract readable content using go-readability article, err := readability.FromReader(file, baseURL) if err != nil { return "", fmt.Errorf("failed to extract readable content: %w", err) } // Try to convert HTML content to markdown var content string converter := md.NewConverter("", true, nil) markdownContent, err := converter.ConvertString(article.Content) if err != nil { // If markdown conversion fails, fall back to plain text content = article.TextContent } else { content = markdownContent } // Save markdown/text file next to HTML file // Replace extension with .md dir := filepath.Dir(filePath) baseName := filepath.Base(filePath) ext := filepath.Ext(baseName) markdownPath := filepath.Join(dir, strings.TrimSuffix(baseName, ext)+".md") if err := os.WriteFile(markdownPath, []byte(content), 0644); err != nil { return "", fmt.Errorf("failed to write markdown file: %w", err) } return content, nil } // GetInitError returns the initialization error if any func (e *ReadableExtractor) GetInitError() error { return e.initError } // IsAvailable returns whether the extractor is available func (e *ReadableExtractor) IsAvailable() bool { return e.available }