diff --git a/database.go b/database.go index 54e97bd..8a9136f 100644 --- a/database.go +++ b/database.go @@ -1,101 +1,31 @@ package main import ( - "database/sql" "embed" - "fmt" - _ "github.com/mattn/go-sqlite3" + "gitea.seeseepuff.be/seeseemelk/mysqlite" "log" "os" - "strconv" - "strings" ) //go:embed migrations/*.sql var embeddedMigrations embed.FS -func openDatabase() *sql.DB { +func openDatabase() *mysqlite.Db { // Get database file databaseSource := os.Getenv("VIVAPLUS_DATABASE") if databaseSource == "" { databaseSource = "videos.db3" } - return openDatabaseSource(databaseSource) -} -func openDatabaseSource(databaseSource string) *sql.DB { - // Initialize the database connection - db, err := sql.Open("sqlite3", databaseSource) + db, err := mysqlite.OpenDb(databaseSource) if err != nil { log.Fatalf("error opening database: %v", err) } - // Read all migrations - migrationFiles, err := embeddedMigrations.ReadDir("migrations") + err = db.MigrateDb(embeddedMigrations, "migrations") if err != nil { - log.Fatalf("error reading migration files: %v", err) - } - var migrations = make(map[int]string) - latestVersion := 0 - for _, f := range migrationFiles { - versionStr := f.Name() - version, err := strconv.Atoi(strings.SplitN(versionStr, "_", 2)[0]) - if err != nil { - log.Fatalf("invalid version number for migration script: %v", err) - } - migrations[version] = versionStr - latestVersion = max(latestVersion, version) + log.Fatalf("error migrating database: %v", err) } - // Get current migration version from user_version - var currentVersion int - err = db.QueryRow("PRAGMA user_version").Scan(¤tVersion) - if err != nil { - log.Fatalf("error getting current version: %v", err) - } - log.Printf("Current database migration version is %d, latest version is %d", currentVersion, latestVersion) - - // If we are no up-to-date, bring the db up-to-date - for currentVersion != latestVersion { - targetVersion := currentVersion + 1 - migrationFile := migrations[targetVersion] - log.Printf("migration to version %s", migrationFile) - migrationScript, err := embeddedMigrations.ReadFile("migrations/" + migrationFile) - if err != nil { - log.Fatalf("error opening migration script %s: %v", migrationScript, err) - } - - tx, err := db.Begin() - if err != nil { - log.Fatalf("error beginning transaction: %v", err) - } - - _, err = tx.Exec(string(migrationScript)) - if err != nil { - rollbackIgnoringErrors(tx) - log.Fatalf("error performing migration: %v", err) - } - - _, err = tx.Exec(fmt.Sprintf("PRAGMA user_version = %d", targetVersion)) - if err != nil { - rollbackIgnoringErrors(tx) - log.Fatalf("error updating version: %v", err) - } - - err = tx.Commit() - if err != nil { - log.Fatalf("error commiting transaction: %v", err) - } - currentVersion = targetVersion - } - - log.Println("All migrations applied") return db } - -func rollbackIgnoringErrors(tx *sql.Tx) { - err := tx.Rollback() - if err != nil { - log.Printf("error rolling back: %v", err) - } -} diff --git a/downloader.go b/downloader.go index b92e332..9b1933a 100644 --- a/downloader.go +++ b/downloader.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "errors" "fmt" + "gitea.seeseepuff.be/seeseemelk/mysqlite" "io" "log" "net/http" @@ -15,7 +16,7 @@ import ( "time" ) -func DownloadAllVideos(db *sql.DB) error { +func DownloadAllVideos(db *mysqlite.Db) error { // Ensure no partial videos exist tempDir := os.Getenv("VIVAPLUS_DOWNLOAD") if tempDir == "" { @@ -43,10 +44,10 @@ func DownloadAllVideos(db *sql.DB) error { log.Printf("Starting downloads...") for { // Fetch the next record from the database - row := db.QueryRow("select id, url, `cast`, title, description, upload_date, thumbnail, episode from videos where state = 'pending' and episode is not null order by year, episode limit 1") var id, episode int var href, cast, title, description, uploadDateStr, thumbnailUrl string - err = row.Scan(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl, &episode) + err = db.Query("select id, url, `cast`, title, description, upload_date, thumbnail, episode from videos where state = 'pending' and episode is not null order by year, episode limit 1"). + ScanSingle(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl, &episode) if errors.Is(err, sql.ErrNoRows) { log.Printf("No videos found for downloading") return nil diff --git a/go.mod b/go.mod index 77be89a..21e4ba1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module vivaplusdl -go 1.23 +go 1.24 + +toolchain go1.24.0 require ( github.com/mattn/go-sqlite3 v1.14.24 @@ -8,7 +10,18 @@ require ( ) require ( + gitea.seeseepuff.be/seeseemelk/mysqlite v0.3.0 // indirect github.com/deckarep/golang-set/v2 v2.7.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-stack/stack v1.8.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.22.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/sqlite v1.33.1 // indirect ) diff --git a/go.sum b/go.sum index 6ed8d67..04a8f57 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,33 @@ +gitea.seeseepuff.be/seeseemelk/mysqlite v0.3.0 h1:T2TjZj7hvb3KF0zYf/uzidprcDJIBPFi3QcN+gmyutM= +gitea.seeseepuff.be/seeseemelk/mysqlite v0.3.0/go.mod h1:cgswydOxJjMlNwfcBIXnKjr47LwXnMT9BInkiHb0tXE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.7.0 h1:gIloKvD7yH2oip4VLhsv3JyLLFnC0Y2mlusgcvJYW5k= github.com/deckarep/golang-set/v2 v2.7.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/playwright-community/playwright-go v0.4902.0 h1:SslPUKmc35YgTBZKTLhokxrqTsVk3/mirj+TkqR6dC0= github.com/playwright-community/playwright-go v0.4902.0/go.mod h1:kBNWs/w2aJ2ZUp1wEOOFLXgOqvppFngM5OS+qyhl+ZM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= @@ -40,8 +52,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -62,3 +77,13 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= +modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= +zombiezen.com/go/sqlite v1.4.0 h1:N1s3RIljwtp4541Y8rM880qgGIgq3fTD2yks1xftnKU= +zombiezen.com/go/sqlite v1.4.0/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik= diff --git a/vivaweb.go b/vivaweb.go index 25c2f40..9b3accc 100644 --- a/vivaweb.go +++ b/vivaweb.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "gitea.seeseepuff.be/seeseemelk/mysqlite" "github.com/playwright-community/playwright-go" "log" "regexp" @@ -100,7 +101,7 @@ func parseDateString(dateStr string) (time.Time, error) { return t, nil } -func (w *WebClient) DiscoverAllVideos(db *sql.DB) error { +func (w *WebClient) DiscoverAllVideos(db *mysqlite.Db) error { log.Printf("Loading list of all videos...") _, err := w.page.Goto("https://vivaplus.tv/supporters/videos/all?order=desc") @@ -116,12 +117,11 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error { if err != nil { return fmt.Errorf("error starting transaction: %w", err) } - defer tx.Rollback() + defer tx.MustRollback() // Find the next run number var currentRun int - row := tx.QueryRow("select max(run) from videos") - err = row.Scan(¤tRun) + err = tx.Query("select max(run) from videos").ScanSingle(¤tRun) if err != nil { return fmt.Errorf("error retrieving current run: %w", err) } @@ -165,9 +165,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error { // Ensure the record does not already exist. If it does, we've fetched // all new videos - result := tx.QueryRow("select count(1) from videos where url = :url", href) - var count int - err = result.Scan(&count) + err = tx.Query("select count(1) from videos where url = :url").Bind(href).ScanSingle(&count) if err != nil { return fmt.Errorf("error fetching data from db: %w", err) } @@ -178,7 +176,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error { // Insert it into the database log.Printf("Adding video %s", href) - _, err = tx.Exec("insert into videos(url, thumbnail, run) values (?, ?, ?)", href, thumbnail, currentRun) + err = tx.Query("insert into videos(url, thumbnail, run) values (?, ?, ?)").Bind(href, thumbnail, currentRun).Exec() if err != nil { return fmt.Errorf("error inserting into db: %w", err) } @@ -204,14 +202,13 @@ func isRelativeTimeFormat(input string) bool { return re.MatchString(input) } -func (w *WebClient) FetchVideoMetadata(db *sql.DB) error { +func (w *WebClient) FetchVideoMetadata(db *mysqlite.Db) error { log.Printf("Fetching video metadata...") for { // Fetch the next record from the database - row := db.QueryRow("select id, url from videos where `cast` is null limit 1") var id int var href string - err := row.Scan(&id, &href) + err := db.Query("select id, url from videos where `cast` is null limit 1").ScanSingle(&id, &href) if errors.Is(err, sql.ErrNoRows) { log.Printf("Fetched all metadata") return nil @@ -263,29 +260,31 @@ func (w *WebClient) FetchVideoMetadata(db *sql.DB) error { castSource, err := videoElement.GetAttribute("cast-src") // Store info in database - tx, err := db.Begin() + err = updateVideoMetadata(db, id, title, description, castSource, uploadDate.Format(time.DateOnly), uploadDate.Year()) if err != nil { - return fmt.Errorf("error starting transaction: %w", err) - } - defer tx.Rollback() - result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ?, year = ? where id = ?", title, description, castSource, uploadDate.Format(time.DateOnly), uploadDate.Year(), id) - if err != nil { - return fmt.Errorf("error updating database: %w", err) - } - rowsAffected, err := result.RowsAffected() - if err != nil { - return fmt.Errorf("error retrieving affected rows: %w", err) - } - if rowsAffected != 1 { - return fmt.Errorf("unexpected number of rows affected: %d", rowsAffected) - } - err = tx.Commit() - if err != nil { - return fmt.Errorf("error commiting changeds: %w", err) + return err } } } +func updateVideoMetadata(db *mysqlite.Db, id int, title, description, castSource, uploadDate string, year int) error { + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("error starting transaction: %w", err) + } + defer tx.MustRollback() + err = tx.Query("update videos set title = ?, description = ?, cast = ?, upload_date = ?, year = ? where id = ?"). + Bind(title, description, castSource, uploadDate, year, id).Exec() + if err != nil { + return fmt.Errorf("error updating database: %w", err) + } + err = tx.Commit() + if err != nil { + return fmt.Errorf("error commiting changeds: %w", err) + } + return nil +} + func (w *WebClient) getInnerText(selector string) (string, error) { // Get video title titleElement, err := w.page.QuerySelector(selector)