package rules import ( "context" "fmt" "net/http" "net/url" "strings" "time" ) // URLMetadata contains metadata extracted from a URL via HEAD request type URLMetadata struct { Domain string MimeType string Headers http.Header } // AnalyzeURL performs a HEAD request to the URL and extracts metadata func AnalyzeURL(ctx context.Context, urlStr string) (*URLMetadata, error) { // Parse URL to extract domain parsedURL, err := url.Parse(urlStr) if err != nil { return nil, fmt.Errorf("failed to parse URL: %w", err) } domain := parsedURL.Host // Remove port if present if idx := strings.Index(domain, ":"); idx != -1 { domain = domain[:idx] } // Create HTTP client with timeout client := &http.Client{ Timeout: 30 * time.Second, } // Create HEAD request req, err := http.NewRequestWithContext(ctx, http.MethodHead, urlStr, nil) if err != nil { return nil, fmt.Errorf("failed to create HEAD request: %w", err) } // Set user agent req.Header.Set("User-Agent", "Hako/1.0") // Perform HEAD request resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("HEAD request failed: %w", err) } defer func() { _ = resp.Body.Close() }() // Extract MIME type from Content-Type header mimeType := resp.Header.Get("Content-Type") // Remove charset and other parameters (e.g., "text/html; charset=utf-8" -> "text/html") if idx := strings.Index(mimeType, ";"); idx != -1 { mimeType = strings.TrimSpace(mimeType[:idx]) } return &URLMetadata{ Domain: domain, MimeType: mimeType, Headers: resp.Header, }, nil }