Add a new archiver to download videos using yt-dlp. The new extractor should call the yt-dlp binary to download the video and we should track progress in the output and return code. The default rules should be updated so youtube videos are extracted using this extractor. The extractor default config should get the thumbnail as well (so we don't depend on the thumbnail extractor) and subtitles. Update the dockerfile accordingly so we not only have yt-dlp but it's required dependencies as well. Prefer installing from packages, if possible.
152 lines
5.1 KiB
Go
152 lines
5.1 KiB
Go
package testutil
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.nakama.town/fmartingr/hako/internal/archival/archiver"
|
|
archivalRules "git.nakama.town/fmartingr/hako/internal/archival/rules"
|
|
archivalStore "git.nakama.town/fmartingr/hako/internal/archival/store"
|
|
"git.nakama.town/fmartingr/hako/internal/auth"
|
|
"git.nakama.town/fmartingr/hako/internal/auth/store"
|
|
"git.nakama.town/fmartingr/hako/internal/config"
|
|
"git.nakama.town/fmartingr/hako/internal/database"
|
|
"git.nakama.town/fmartingr/hako/internal/dependencies"
|
|
"git.nakama.town/fmartingr/hako/internal/extractors"
|
|
"git.nakama.town/fmartingr/hako/internal/jobs"
|
|
"git.nakama.town/fmartingr/hako/internal/storage"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// TestDependencies holds all dependencies needed for testing
|
|
type TestDependencies struct {
|
|
Config *config.Config
|
|
DBConnections *database.Connections
|
|
Dependencies *dependencies.Dependencies
|
|
ArchiveStore *archivalStore.ArchiveStore
|
|
ArchiveFileStore *archivalStore.ArchiveFileStore
|
|
LinkCategoryStore *archivalStore.LinkCategoryStore
|
|
Worker *jobs.Worker
|
|
Logger *slog.Logger
|
|
}
|
|
|
|
// GetTestConfigurationAndDependencies creates test configuration and dependencies
|
|
// with a temporary SQLite database that is cleaned up after tests
|
|
func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context) *TestDependencies {
|
|
t.Helper()
|
|
|
|
// Create temporary database file
|
|
tmpDB, err := os.CreateTemp("", "hako_test_*.db")
|
|
require.NoError(t, err)
|
|
_ = tmpDB.Close()
|
|
|
|
// Create temporary storage directory
|
|
tmpStorageDir, err := os.MkdirTemp("", "hako_test_storage_*")
|
|
require.NoError(t, err)
|
|
|
|
// Clean up after test
|
|
t.Cleanup(func() {
|
|
_ = os.Remove(tmpDB.Name())
|
|
_ = os.Remove(tmpDB.Name() + "-wal")
|
|
_ = os.Remove(tmpDB.Name() + "-shm")
|
|
_ = os.RemoveAll(tmpStorageDir)
|
|
})
|
|
|
|
dbURL := "sqlite:" + tmpDB.Name()
|
|
|
|
// Create logger
|
|
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelError}))
|
|
|
|
// Create test configuration
|
|
cfg := &config.Config{
|
|
DatabaseURL: dbURL,
|
|
DatabaseReadURL: dbURL,
|
|
JWTSecret: "test-secret-key-for-testing-only",
|
|
JWTExpiration: 24 * time.Hour,
|
|
ServerPort: 8080,
|
|
ArchiveStoragePath: tmpStorageDir,
|
|
}
|
|
|
|
// Create database connections
|
|
dbConnections, err := database.NewConnections(cfg.DatabaseReadURL, cfg.DatabaseURL)
|
|
require.NoError(t, err)
|
|
|
|
// Clean up database connections after test
|
|
t.Cleanup(func() {
|
|
_ = dbConnections.Close()
|
|
})
|
|
|
|
// Run migrations
|
|
err = database.InitSchema(dbConnections)
|
|
require.NoError(t, err)
|
|
|
|
// Initialize store layer
|
|
userStore := store.NewUserStore(dbConnections.Read, dbConnections.Write)
|
|
|
|
// Create dependencies
|
|
deps := dependencies.NewDependencies(logger, dbConnections, cfg)
|
|
|
|
// Initialize JWT service
|
|
jwtService := auth.NewJWTService(cfg.JWTSecret, cfg.JWTExpiration)
|
|
|
|
// Initialize storage
|
|
stor := storage.NewLocalStorage(cfg.ArchiveStoragePath)
|
|
|
|
// Initialize archiver manager
|
|
archiverMgr := archiver.NewManager()
|
|
_ = archiverMgr.Register(archiver.NewDirectDownloadExtractor())
|
|
_ = archiverMgr.Register(archiver.NewYtDlpArchiver())
|
|
|
|
// Initialize extractor manager
|
|
extractorMgr := extractors.NewManager(logger)
|
|
_ = extractorMgr.Register(extractors.NewPDFExtractor())
|
|
|
|
// Initialize archival stores
|
|
linkStore := archivalStore.NewLinkStore(dbConnections.Read, dbConnections.Write)
|
|
archiveStore := archivalStore.NewArchiveStore(dbConnections.Read, dbConnections.Write)
|
|
archiveFileStore := archivalStore.NewArchiveFileStore(dbConnections.Read, dbConnections.Write)
|
|
categoryStore := archivalStore.NewCategoryStore(dbConnections.Read, dbConnections.Write)
|
|
linkCategoryStore := archivalStore.NewLinkCategoryStore(dbConnections.Read, dbConnections.Write)
|
|
archiverConfigStore := archivalStore.NewArchiverConfigStore(dbConnections.Read, dbConnections.Write)
|
|
settingsStore := archivalStore.NewSettingsStore(dbConnections.Read, dbConnections.Write)
|
|
|
|
// Initialize rule engine with default rules
|
|
rulesConfig := archivalRules.GetDefaultRulesConfig()
|
|
ruleEngine := archivalRules.NewEngine(rulesConfig)
|
|
|
|
// Initialize job queue
|
|
queue := jobs.NewMemoryQueue()
|
|
|
|
// Set stores and other dependencies in dependencies
|
|
deps.LinkStore = linkStore
|
|
deps.ArchiveStore = archiveStore
|
|
deps.ArchiveFileStore = archiveFileStore
|
|
deps.LinkCategoryStore = linkCategoryStore
|
|
deps.CategoryStore = categoryStore
|
|
deps.ArchiverConfigStore = archiverConfigStore
|
|
deps.SettingsStore = settingsStore
|
|
deps.UserStore = userStore
|
|
deps.Storage = stor
|
|
deps.Queue = queue
|
|
deps.ArchiverMgr = archiverMgr
|
|
deps.ExtractorMgr = extractorMgr
|
|
deps.RuleEngine = ruleEngine
|
|
deps.JWTService = jwtService
|
|
|
|
// Note: Domains should be initialized by the test itself to avoid import cycles
|
|
// Worker will be created by tests that need it after domains are initialized
|
|
|
|
return &TestDependencies{
|
|
Config: cfg,
|
|
DBConnections: dbConnections,
|
|
Dependencies: deps,
|
|
ArchiveStore: archiveStore,
|
|
ArchiveFileStore: archiveFileStore,
|
|
LinkCategoryStore: linkCategoryStore,
|
|
Worker: nil, // Worker should be created by tests after domains are initialized
|
|
Logger: logger,
|
|
}
|
|
}
|