hako/internal/archival/rules/url_cleaner_test.go

package rules

import (
	"strings"
	"testing"
)

func TestCleanURL(t *testing.T) {
	tests := []struct {
		name     string
		input    string
		expected string
	}{
		{
			name:     "URL with UTM parameters",
			input:    "https://example.com/page?utm_source=google&utm_medium=cpc&utm_campaign=test&id=123",
			expected: "https://example.com/page?id=123",
		},
		{
			name:     "URL with all UTM parameters",
			input:    "https://example.com/page?utm_source=google&utm_medium=cpc&utm_campaign=test&utm_term=keyword&utm_content=ad",
			expected: "https://example.com/page",
		},
		{
			name:     "URL with fbclid",
			input:    "https://example.com/page?fbclid=abc123&ref=article",
			expected: "https://example.com/page?ref=article",
		},
		{
			name:     "URL with gclid",
			input:    "https://example.com/page?gclid=xyz789&source=newsletter",
			expected: "https://example.com/page?source=newsletter",
		},
		{
			name:     "URL with both fbclid and gclid",
			input:    "https://example.com/page?fbclid=abc123&gclid=xyz789&page=1",
			expected: "https://example.com/page?page=1",
		},
		{
			name:     "URL with all tracking parameters",
			input:    "https://example.com/page?utm_source=google&utm_medium=cpc&fbclid=abc&gclid=xyz&id=123",
			expected: "https://example.com/page?id=123",
		},
		{
			name:     "URL without tracking parameters",
			input:    "https://example.com/page?id=123&name=test",
			expected: "https://example.com/page?id=123&name=test",
		},
		{
			name:     "URL without query parameters",
			input:    "https://example.com/page",
			expected: "https://example.com/page",
		},
		{
			name:     "URL with only tracking parameters",
			input:    "https://example.com/page?utm_source=google&fbclid=abc",
			expected: "https://example.com/page",
		},
		{
			name:     "URL with fragment",
			input:    "https://example.com/page?utm_source=google#section",
			expected: "https://example.com/page#section",
		},
		{
			name:     "URL with path and tracking parameters",
			input:    "https://example.com/articles/2024/post?utm_source=twitter&utm_medium=social&slug=my-post",
			expected: "https://example.com/articles/2024/post?slug=my-post",
		},
		{
			name:     "URL with port",
			input:    "https://example.com:8080/page?utm_source=google&id=123",
			expected: "https://example.com:8080/page?id=123",
		},
		{
			name:     "URL with user info",
			input:    "https://user:pass@example.com/page?utm_source=google&id=123",
			expected: "https://user:pass@example.com/page?id=123",
		},
		{
			name:     "HTTP URL",
			input:    "http://example.com/page?utm_source=google&id=123",
			expected: "http://example.com/page?id=123",
		},
		{
			name:     "URL with empty query parameter values",
			input:    "https://example.com/page?utm_source=&id=123",
			expected: "https://example.com/page?id=123",
		},
		{
			name:     "URL with multiple values for same parameter",
			input:    "https://example.com/page?utm_source=google&utm_source=facebook&id=123",
			expected: "https://example.com/page?id=123",
		},
		{
			name:     "Relative URL with tracking parameters",
			input:    "not-a-valid-url?utm_source=google",
			expected: "not-a-valid-url",
		},
		{
			name:     "URL with special characters in parameters",
			input:    "https://example.com/page?utm_source=test%20value&id=123",
			expected: "https://example.com/page?id=123",
		},
		{
			name:     "URL with case-sensitive tracking parameters",
			input:    "https://example.com/page?UTM_SOURCE=google&utm_source=facebook&id=123",
			expected: "https://example.com/page?UTM_SOURCE=google&id=123",
		},
		{
			name:     "URL with Twitter ref_src parameter",
			input:    "https://x.com/user/status/123?ref_src=twsrc%5Etfw&id=456",
			expected: "https://x.com/user/status/123?id=456",
		},
		{
			name:     "URL with Twitter ref_url parameter",
			input:    "https://x.com/user/status/123?ref_url=https://example.com&id=456",
			expected: "https://x.com/user/status/123?id=456",
		},
		{
			name:     "URL with both Twitter parameters",
			input:    "https://x.com/user/status/123?ref_src=twsrc%5Etfw&ref_url=https://example.com&id=456",
			expected: "https://x.com/user/status/123?id=456",
		},
		{
			name:     "Twitter URL with all tracking parameters",
			input:    "https://x.com/NateTheHate2/status/2009691766191165501?ref_src=twsrc%5Etfw%7Ctwcamp%5Etweetembed%7Ctwterm%5E2009691766191165501%7Ctwgr%5E%7Ctwcon%5Es1_c10&ref_url=",
			expected: "https://x.com/NateTheHate2/status/2009691766191165501",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := CleanURL(tt.input)
			if result != tt.expected {
				t.Errorf("CleanURL(%q) = %q, want %q", tt.input, result, tt.expected)
			}
		})
	}
}

func TestCleanURL_PreservesOtherParameters(t *testing.T) {
	tests := []struct {
		name     string
		input    string
		contains []string // Parameters that should still be present
	}{
		{
			name:     "Preserves id parameter",
			input:    "https://example.com/page?utm_source=google&id=123",
			contains: []string{"id=123"},
		},
		{
			name:     "Preserves multiple non-tracking parameters",
			input:    "https://example.com/page?utm_source=google&id=123&name=test&page=1",
			contains: []string{"id=123", "name=test", "page=1"},
		},
		{
			name:     "Preserves ref parameter",
			input:    "https://example.com/page?fbclid=abc&ref=article",
			contains: []string{"ref=article"},
		},
		{
			name:     "Preserves source parameter",
			input:    "https://example.com/page?gclid=xyz&source=newsletter",
			contains: []string{"source=newsletter"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := CleanURL(tt.input)
			for _, param := range tt.contains {
				if !strings.Contains(result, param) {
					t.Errorf("CleanURL(%q) = %q, should contain %q", tt.input, result, param)
				}
			}
		})
	}
}

func TestCleanURL_RemovesTrackingParameters(t *testing.T) {
	tests := []struct {
		name        string
		input       string
		notContains []string // Parameters that should NOT be present
	}{
		{
			name:        "Removes utm_source",
			input:       "https://example.com/page?utm_source=google&id=123",
			notContains: []string{"utm_source"},
		},
		{
			name:        "Removes all UTM parameters",
			input:       "https://example.com/page?utm_source=google&utm_medium=cpc&utm_campaign=test&utm_term=keyword&utm_content=ad&id=123",
			notContains: []string{"utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content"},
		},
		{
			name:        "Removes fbclid",
			input:       "https://example.com/page?fbclid=abc123&id=123",
			notContains: []string{"fbclid"},
		},
		{
			name:        "Removes gclid",
			input:       "https://example.com/page?gclid=xyz789&id=123",
			notContains: []string{"gclid"},
		},
		{
			name:        "Removes all tracking parameters",
			input:       "https://example.com/page?utm_source=google&fbclid=abc&gclid=xyz&id=123",
			notContains: []string{"utm_source", "fbclid", "gclid"},
		},
		{
			name:        "Removes ref_src",
			input:       "https://x.com/user/status/123?ref_src=twsrc%5Etfw&id=456",
			notContains: []string{"ref_src"},
		},
		{
			name:        "Removes ref_url",
			input:       "https://x.com/user/status/123?ref_url=https://example.com&id=456",
			notContains: []string{"ref_url"},
		},
		{
			name:        "Removes all Twitter tracking parameters",
			input:       "https://x.com/user/status/123?ref_src=twsrc%5Etfw&ref_url=https://example.com&id=456",
			notContains: []string{"ref_src", "ref_url"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := CleanURL(tt.input)
			for _, param := range tt.notContains {
				if strings.Contains(result, param+"=") {
					t.Errorf("CleanURL(%q) = %q, should not contain %q", tt.input, result, param)
				}
			}
		})
	}
}