diff --git a/internal/archival/archiver/yt_dlp.go b/internal/archival/archiver/yt_dlp.go index 9f0f87f..ce1bc67 100644 --- a/internal/archival/archiver/yt_dlp.go +++ b/internal/archival/archiver/yt_dlp.go @@ -7,9 +7,11 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "os" "os/exec" "path/filepath" + "regexp" "strings" "time" @@ -23,6 +25,9 @@ type YtDlpConfig struct { WriteThumbnail bool `json:"write_thumbnail"` WriteSubs bool `json:"write_subs"` // Download subtitles as separate files EmbedSubs bool `json:"embed_subs"` // Embed subtitles into the video file + // IgnoreErrors archives the video even when other assets (subtitles, thumbnail) + // fail; those failures are logged as warnings instead of failing the archival. + IgnoreErrors bool `json:"ignore_errors"` } // ToMap converts the YtDlpConfig to a map[string]any for use with ApplyConfig @@ -52,6 +57,7 @@ func NewYtDlpArchiver() *YtDlpArchiver { WriteThumbnail: true, WriteSubs: true, EmbedSubs: true, + IgnoreErrors: true, }, } } @@ -93,6 +99,7 @@ func (a *YtDlpArchiver) GetDefaultConfig() any { WriteThumbnail: true, WriteSubs: true, EmbedSubs: true, + IgnoreErrors: true, } } @@ -113,6 +120,9 @@ func (a *YtDlpArchiver) ApplyConfig(config map[string]any) error { if v, ok := config["embed_subs"].(bool); ok { a.config.EmbedSubs = v } + if v, ok := config["ignore_errors"].(bool); ok { + a.config.IgnoreErrors = v + } return nil } @@ -164,9 +174,11 @@ func (a *YtDlpArchiver) Archive(ctx context.Context, link *model.Link, stor stor outputTemplate := filepath.Join(tempDir, "%(title)s.%(ext)s") args := []string{ "-o", outputTemplate, - "--no-warnings", "--no-playlist", } + if a.config.IgnoreErrors { + args = append(args, "--ignore-errors") + } if a.config.WriteThumbnail { args = append(args, "--write-thumbnail") } @@ -186,12 +198,17 @@ func (a *YtDlpArchiver) Archive(ctx context.Context, link *model.Link, stor stor cmd.Stdout = &combinedOutput cmd.Stderr = &combinedOutput - if err := cmd.Run(); err != nil { - errMsg := err.Error() - if exitErr, ok := err.(*exec.ExitError); ok { + runErr := cmd.Run() + output := strings.TrimSpace(combinedOutput.String()) + if runErr != nil && (!a.config.IgnoreErrors || !hasMediaFile(tempDir)) { + errMsg := runErr.Error() + if exitErr, ok := runErr.(*exec.ExitError); ok { errMsg = fmt.Sprintf("exit code %d", exitErr.ExitCode()) } - return nil, fmt.Errorf("yt-dlp failed (%s): %s", errMsg, strings.TrimSpace(combinedOutput.String())) + return nil, fmt.Errorf("yt-dlp failed (%s): %s", errMsg, output) + } + if problems := ytDlpProblems(output); len(problems) > 0 { + slog.Warn("yt-dlp archived the video with missing assets", "url", link.URL, "problems", problems) } var files []FileInfo @@ -239,7 +256,7 @@ func (a *YtDlpArchiver) Archive(ctx context.Context, link *model.Link, stor stor } if len(files) == 0 { - return nil, fmt.Errorf("yt-dlp produced no files. output: %s", strings.TrimSpace(combinedOutput.String())) + return nil, fmt.Errorf("yt-dlp produced no files. output: %s", output) } if title == "" { @@ -269,3 +286,36 @@ func mimeTypeFromExt(ext string) string { func isImageMimeType(mimeType string) bool { return strings.HasPrefix(mimeType, "image/") } + +// ytDlpFormatPart matches the per-format streams (title.f400.mp4, title.f251-drc.webm) +// that yt-dlp keeps only when merging them into the final file failed. +// ponytail: numeric format ids only (YouTube), extend for extractors with named ids like fhls-720p. +var ytDlpFormatPart = regexp.MustCompile(`\.f\d+[^.]*\.[^.]+$`) + +func hasMediaFile(dir string) bool { + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, entry := range entries { + if ytDlpFormatPart.MatchString(entry.Name()) { + continue + } + mimeType := mimeTypeFromExt(filepath.Ext(entry.Name())) + if strings.HasPrefix(mimeType, "video/") || strings.HasPrefix(mimeType, "audio/") { + return true + } + } + return false +} + +// ytDlpProblems returns the WARNING and ERROR lines of yt-dlp output. +func ytDlpProblems(output string) []string { + var problems []string + for _, line := range strings.Split(output, "\n") { + if strings.HasPrefix(line, "WARNING:") || strings.HasPrefix(line, "ERROR:") { + problems = append(problems, line) + } + } + return problems +} diff --git a/internal/archival/archiver/yt_dlp_test.go b/internal/archival/archiver/yt_dlp_test.go index 4426a77..884906b 100644 --- a/internal/archival/archiver/yt_dlp_test.go +++ b/internal/archival/archiver/yt_dlp_test.go @@ -1,6 +1,8 @@ package archiver import ( + "os" + "path/filepath" "testing" ) @@ -127,3 +129,48 @@ func TestYtDlpConfig_ToMap(t *testing.T) { t.Errorf("Expected embed_subs true, got %v", m["embed_subs"]) } } + +func TestYtDlpArchiver_IgnoreErrorsDefaultAndConfig(t *testing.T) { + a := NewYtDlpArchiver() + if !a.config.IgnoreErrors || !a.GetDefaultConfig().(YtDlpConfig).IgnoreErrors { + t.Fatal("Expected IgnoreErrors to be true by default") + } + if err := a.ApplyConfig(map[string]any{"ignore_errors": false}); err != nil { + t.Fatalf("ApplyConfig failed: %v", err) + } + if a.config.IgnoreErrors { + t.Error("Expected IgnoreErrors to be false after config") + } +} + +func TestHasMediaFile(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "video.en.vtt"), nil, 0o600); err != nil { + t.Fatal(err) + } + if hasMediaFile(dir) { + t.Error("Expected no media file with only subtitles") + } + for _, name := range []string{"video.f400.mp4", "video.f251-drc.webm", "video.mp4.part"} { + if err := os.WriteFile(filepath.Join(dir, name), nil, 0o600); err != nil { + t.Fatal(err) + } + } + if hasMediaFile(dir) { + t.Error("Expected no media file with only unmerged streams and partial downloads") + } + if err := os.WriteFile(filepath.Join(dir, "video.webm"), nil, 0o600); err != nil { + t.Fatal(err) + } + if !hasMediaFile(dir) { + t.Error("Expected a media file after adding a video") + } +} + +func TestYtDlpProblems(t *testing.T) { + output := "[youtube] abc: Downloading webpage\nWARNING: Unable to download video subtitles for 'en': HTTP Error 429\nERROR: Postprocessing: something\n[info] done" + problems := ytDlpProblems(output) + if len(problems) != 2 { + t.Fatalf("Expected 2 problems, got %d: %v", len(problems), problems) + } +}