0afef05099
Add a new archiver to download videos using yt-dlp. The new extractor should call the yt-dlp binary to download the video and we should track progress in the output and return code. The default rules should be updated so youtube videos are extracted using this extractor. The extractor default config should get the thumbnail as well (so we don't depend on the thumbnail extractor) and subtitles. Update the dockerfile accordingly so we not only have yt-dlp but it's required dependencies as well. Prefer installing from packages, if possible.
59 lines
1.7 KiB
Go
59 lines
1.7 KiB
Go
package rules
|
|
|
|
// GetDefaultRulesConfig returns the default rules configuration
|
|
func GetDefaultRulesConfig() *RuleConfig {
|
|
return &RuleConfig{
|
|
Rules: []Rule{
|
|
// Rule 1: Social networks (excluding YouTube) - skip archiving (empty archivers)
|
|
&OrRule{
|
|
Rules: []Rule{
|
|
&HostnameRule{Hostname: "twitter.com"},
|
|
&HostnameRule{Hostname: "x.com"},
|
|
&HostnameRule{Hostname: "facebook.com"},
|
|
&HostnameRule{Hostname: "instagram.com"},
|
|
&HostnameRule{Hostname: "linkedin.com"},
|
|
&HostnameRule{Hostname: "reddit.com"},
|
|
&HostnameRule{Hostname: "tiktok.com"},
|
|
&HostnameRule{Hostname: "pinterest.com"},
|
|
&HostnameRule{Hostname: "snapchat.com"},
|
|
&HostnameRule{Hostname: "discord.com"},
|
|
&HostnameRule{Hostname: "discord.gg"},
|
|
},
|
|
Archivers: []ArchiverConfig{}, // Empty extractors = skip archiving
|
|
},
|
|
// Rule 2: YouTube videos -> yt-dlp (downloads video, thumbnail, subtitles)
|
|
&OrRule{
|
|
Rules: []Rule{
|
|
&HostnameRule{Hostname: "youtube.com"},
|
|
&HostnameRule{Hostname: "youtu.be"},
|
|
},
|
|
Archivers: []ArchiverConfig{
|
|
{Key: "yt_dlp"},
|
|
},
|
|
},
|
|
// Rule 3: application/* OR image/* OR text/markdown mimetypes -> direct_download
|
|
&OrRule{
|
|
Rules: []Rule{
|
|
&MimetypeRule{Mimetype: "application/*"},
|
|
&MimetypeRule{Mimetype: "image/*"},
|
|
&MimetypeRule{Mimetype: "text/markdown"},
|
|
},
|
|
Archivers: []ArchiverConfig{
|
|
{Key: "direct_download"},
|
|
},
|
|
},
|
|
// Rule 4: text/html mimetype -> obelisk and thumbnail
|
|
&MimetypeRule{
|
|
Mimetype: "text/html",
|
|
Archivers: []ArchiverConfig{
|
|
{Key: "obelisk"},
|
|
{Key: "thumbnail"},
|
|
},
|
|
},
|
|
},
|
|
DefaultArchivers: []ArchiverConfig{
|
|
{Key: "obelisk"},
|
|
{Key: "thumbnail"},
|
|
},
|
|
}
|
|
}
|