package rules import ( "context" "testing" ) func TestGetDefaultRulesConfig(t *testing.T) { config := GetDefaultRulesConfig() // Check default extractors (should be obelisk and thumbnail) if len(config.DefaultArchivers) != 2 { t.Errorf("DefaultArchivers length = %d, want 2", len(config.DefaultArchivers)) } if config.DefaultArchivers[0].Key != "obelisk" { t.Errorf("DefaultArchivers[0].Key = %q, want %q", config.DefaultArchivers[0].Key, "obelisk") } if config.DefaultArchivers[1].Key != "thumbnail" { t.Errorf("DefaultArchivers[1].Key = %q, want %q", config.DefaultArchivers[1].Key, "thumbnail") } // Check that we have rules if len(config.Rules) == 0 { t.Error("Expected at least one rule in default config") } // Validate all rules for i, rule := range config.Rules { if err := rule.IsValid(); err != nil { t.Errorf("Default rule %d is invalid: %v", i, err) } } } func TestDefaultRules_SocialNetworks(t *testing.T) { config := GetDefaultRulesConfig() ctx := context.Background() // Find the social networks rule (first rule should be the OR rule with hostnames) socialRule, ok := config.Rules[0].(*OrRule) if !ok { t.Fatal("First rule should be OrRule for social networks") } // Test various social network domains (excluding YouTube - it has its own yt-dlp rule) socialDomains := []string{ "twitter.com", "www.twitter.com", "api.twitter.com", "x.com", "www.x.com", "facebook.com", "www.facebook.com", "instagram.com", "www.instagram.com", "linkedin.com", "www.linkedin.com", "reddit.com", "www.reddit.com", "tiktok.com", "www.tiktok.com", "pinterest.com", "www.pinterest.com", "snapchat.com", "discord.com", "discord.gg", } for _, domain := range socialDomains { t.Run(domain, func(t *testing.T) { metadata := &URLMetadata{ Domain: domain, MimeType: "text/html", } extractors, err := socialRule.Matches(ctx, metadata) if err != nil { t.Fatalf("Matches() error = %v", err) } // Should match but return empty extractors (skip archiving) if extractors == nil { t.Error("Expected match but got nil") } if len(extractors) != 0 { t.Errorf("Expected empty extractors (skip archiving), got %d extractors", len(extractors)) } }) } // Test non-social domain should not match metadata := &URLMetadata{ Domain: "example.com", MimeType: "text/html", } extractors, err := socialRule.Matches(ctx, metadata) if err != nil { t.Fatalf("Matches() error = %v", err) } if extractors != nil { t.Error("Non-social domain should not match social networks rule") } } func TestDefaultRules_YouTubeYtDlp(t *testing.T) { config := GetDefaultRulesConfig() ctx := context.Background() engine := NewEngine(config) for _, domain := range []string{"youtube.com", "www.youtube.com", "youtu.be"} { t.Run(domain, func(t *testing.T) { metadata := &URLMetadata{Domain: domain, MimeType: "text/html"} extractors, err := engine.Evaluate(ctx, metadata) if err != nil { t.Fatalf("Evaluate() error = %v", err) } if len(extractors) != 1 || extractors[0].Key != "yt_dlp" { t.Errorf("YouTube should use yt_dlp, got %v", extractors) } }) } } func TestDefaultRules_MimetypeDirectDownload(t *testing.T) { config := GetDefaultRulesConfig() ctx := context.Background() // Rule 2 is YouTube; rule 3 is the mimetype OR rule for direct_download mimetypeRule, ok := config.Rules[2].(*OrRule) if !ok { t.Fatal("Third rule should be OrRule for mimetypes") } tests := []struct { name string mimetype string shouldMatch bool }{ {"application/pdf", "application/pdf", true}, {"application/json", "application/json", true}, {"image/png", "image/png", true}, {"image/jpeg", "image/jpeg", true}, {"text/markdown", "text/markdown", true}, {"text/html", "text/html", false}, {"video/mp4", "video/mp4", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { metadata := &URLMetadata{ Domain: "example.com", MimeType: tt.mimetype, } extractors, err := mimetypeRule.Matches(ctx, metadata) if err != nil { t.Fatalf("Matches() error = %v", err) } matched := extractors != nil if matched != tt.shouldMatch { t.Errorf("Matches() = %v, want %v", matched, tt.shouldMatch) } if matched { if len(extractors) != 1 || extractors[0].Key != "direct_download" { t.Errorf("Archivers = %v, want [{key: direct_download}]", extractors) } } }) } } func TestDefaultRules_DefaultExtractor(t *testing.T) { config := GetDefaultRulesConfig() ctx := context.Background() engine := NewEngine(config) // Test URL that doesn't match any rules should use default extractors // Note: text/html matches rule 3, so we need a different mimetype metadata := &URLMetadata{ Domain: "example.com", MimeType: "video/mp4", // Doesn't match any rule, will use defaults } extractors, err := engine.Evaluate(ctx, metadata) if err != nil { t.Fatalf("Evaluate() error = %v", err) } // Should use default extractors (obelisk and thumbnail) if len(extractors) != 2 { t.Errorf("Archivers length = %d, want 2", len(extractors)) } if extractors[0].Key != "obelisk" { t.Errorf("Archivers[0].Key = %q, want %q", extractors[0].Key, "obelisk") } if extractors[1].Key != "thumbnail" { t.Errorf("Archivers[1].Key = %q, want %q", extractors[1].Key, "thumbnail") } } func TestDefaultRules_Order(t *testing.T) { config := GetDefaultRulesConfig() ctx := context.Background() engine := NewEngine(config) // Test that social network rule is checked first (even if mimetype matches) metadata := &URLMetadata{ Domain: "twitter.com", MimeType: "application/pdf", // Would match mimetype rule, but social network should win } extractors, err := engine.Evaluate(ctx, metadata) if err != nil { t.Fatalf("Evaluate() error = %v", err) } // Should return empty extractors (skip archiving) because social network rule matches first if len(extractors) != 0 { t.Errorf("Expected empty extractors for social network, got %v", extractors) } }