Working on porting from sqlite to mysqlite

database.go

@@ -1,101 +1,31 @@
 package main
 
 import (
-	"database/sql"
 	"embed"
-	"fmt"
+	"gitea.seeseepuff.be/seeseemelk/mysqlite"
-	_ "github.com/mattn/go-sqlite3"
 	"log"
 	"os"
-	"strconv"
-	"strings"
 )
 
 //go:embed migrations/*.sql
 var embeddedMigrations embed.FS
 
-func openDatabase() *sql.DB {
+func openDatabase() *mysqlite.Db {
 	// Get database file
 	databaseSource := os.Getenv("VIVAPLUS_DATABASE")
 	if databaseSource == "" {
 		databaseSource = "videos.db3"
 	}
-	return openDatabaseSource(databaseSource)
-}
 
-func openDatabaseSource(databaseSource string) *sql.DB {
+	db, err := mysqlite.OpenDb(databaseSource)
-	// Initialize the database connection
-	db, err := sql.Open("sqlite3", databaseSource)
 	if err != nil {
 		log.Fatalf("error opening database: %v", err)
 	}
 
-	// Read all migrations
+	err = db.MigrateDb(embeddedMigrations, "migrations")
-	migrationFiles, err := embeddedMigrations.ReadDir("migrations")
 	if err != nil {
-		log.Fatalf("error reading migration files: %v", err)
+		log.Fatalf("error migrating database: %v", err)
-	}
-	var migrations = make(map[int]string)
-	latestVersion := 0
-	for _, f := range migrationFiles {
-		versionStr := f.Name()
-		version, err := strconv.Atoi(strings.SplitN(versionStr, "_", 2)[0])
-		if err != nil {
-			log.Fatalf("invalid version number for migration script: %v", err)
-		}
-		migrations[version] = versionStr
-		latestVersion = max(latestVersion, version)
 	}
 
-	// Get current migration version from user_version
-	var currentVersion int
-	err = db.QueryRow("PRAGMA user_version").Scan(&currentVersion)
-	if err != nil {
-		log.Fatalf("error getting current version: %v", err)
-	}
-	log.Printf("Current database migration version is %d, latest version is %d", currentVersion, latestVersion)
-
-	// If we are no up-to-date, bring the db up-to-date
-	for currentVersion != latestVersion {
-		targetVersion := currentVersion + 1
-		migrationFile := migrations[targetVersion]
-		log.Printf("migration to version %s", migrationFile)
-		migrationScript, err := embeddedMigrations.ReadFile("migrations/" + migrationFile)
-		if err != nil {
-			log.Fatalf("error opening migration script %s: %v", migrationScript, err)
-		}
-
-		tx, err := db.Begin()
-		if err != nil {
-			log.Fatalf("error beginning transaction: %v", err)
-		}
-
-		_, err = tx.Exec(string(migrationScript))
-		if err != nil {
-			rollbackIgnoringErrors(tx)
-			log.Fatalf("error performing migration: %v", err)
-		}
-
-		_, err = tx.Exec(fmt.Sprintf("PRAGMA user_version = %d", targetVersion))
-		if err != nil {
-			rollbackIgnoringErrors(tx)
-			log.Fatalf("error updating version: %v", err)
-		}
-
-		err = tx.Commit()
-		if err != nil {
-			log.Fatalf("error commiting transaction: %v", err)
-		}
-		currentVersion = targetVersion
-	}
-
-	log.Println("All migrations applied")
 	return db
 }
-
-func rollbackIgnoringErrors(tx *sql.Tx) {
-	err := tx.Rollback()
-	if err != nil {
-		log.Printf("error rolling back: %v", err)
-	}
-}

downloader.go

@@ -5,6 +5,7 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"gitea.seeseepuff.be/seeseemelk/mysqlite"
 	"io"
 	"log"
 	"net/http"
@@ -15,7 +16,7 @@ import (
 	"time"
 )
 
-func DownloadAllVideos(db *sql.DB) error {
+func DownloadAllVideos(db *mysqlite.Db) error {
 	// Ensure no partial videos exist
 	tempDir := os.Getenv("VIVAPLUS_DOWNLOAD")
 	if tempDir == "" {
@@ -43,10 +44,10 @@ func DownloadAllVideos(db *sql.DB) error {
 	log.Printf("Starting downloads...")
 	for {
 		// Fetch the next record from the database
-		row := db.QueryRow("select id, url, `cast`, title, description, upload_date, thumbnail, episode from videos where state = 'pending' and episode is not null order by year, episode limit 1")
 		var id, episode int
 		var href, cast, title, description, uploadDateStr, thumbnailUrl string
-		err = row.Scan(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl, &episode)
+		err = db.Query("select id, url, `cast`, title, description, upload_date, thumbnail, episode from videos where state = 'pending' and episode is not null order by year, episode limit 1").
+			ScanSingle(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl, &episode)
 		if errors.Is(err, sql.ErrNoRows) {
 			log.Printf("No videos found for downloading")
 			return nil

go.mod

@@ -1,6 +1,8 @@
 module vivaplusdl
 
-go 1.23
+go 1.24
+
+toolchain go1.24.0
 
 require (
 	github.com/mattn/go-sqlite3 v1.14.24
@@ -8,7 +10,18 @@ require (
 )
 
 require (
+	gitea.seeseepuff.be/seeseemelk/mysqlite v0.3.0 // indirect
 	github.com/deckarep/golang-set/v2 v2.7.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.3 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	modernc.org/libc v1.55.3 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.8.0 // indirect
+	modernc.org/sqlite v1.33.1 // indirect
 )

go.sum

@@ -1,21 +1,33 @@
+gitea.seeseepuff.be/seeseemelk/mysqlite v0.3.0 h1:T2TjZj7hvb3KF0zYf/uzidprcDJIBPFi3QcN+gmyutM=
+gitea.seeseepuff.be/seeseemelk/mysqlite v0.3.0/go.mod h1:cgswydOxJjMlNwfcBIXnKjr47LwXnMT9BInkiHb0tXE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/deckarep/golang-set/v2 v2.7.0 h1:gIloKvD7yH2oip4VLhsv3JyLLFnC0Y2mlusgcvJYW5k=
 github.com/deckarep/golang-set/v2 v2.7.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=
 github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
 github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
 github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
 github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/playwright-community/playwright-go v0.4902.0 h1:SslPUKmc35YgTBZKTLhokxrqTsVk3/mirj+TkqR6dC0=
 github.com/playwright-community/playwright-go v0.4902.0/go.mod h1:kBNWs/w2aJ2ZUp1wEOOFLXgOqvppFngM5OS+qyhl+ZM=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
@@ -40,8 +52,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -62,3 +77,13 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
+modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
+modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
+modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM=
+modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k=
+zombiezen.com/go/sqlite v1.4.0 h1:N1s3RIljwtp4541Y8rM880qgGIgq3fTD2yks1xftnKU=
+zombiezen.com/go/sqlite v1.4.0/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik=

vivaweb.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"gitea.seeseepuff.be/seeseemelk/mysqlite"
 	"github.com/playwright-community/playwright-go"
 	"log"
 	"regexp"
@@ -100,7 +101,7 @@ func parseDateString(dateStr string) (time.Time, error) {
 	return t, nil
 }
 
-func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
+func (w *WebClient) DiscoverAllVideos(db *mysqlite.Db) error {
 	log.Printf("Loading list of all videos...")
 
 	_, err := w.page.Goto("https://vivaplus.tv/supporters/videos/all?order=desc")
@@ -116,12 +117,11 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
 	if err != nil {
 		return fmt.Errorf("error starting transaction: %w", err)
 	}
-	defer tx.Rollback()
+	defer tx.MustRollback()
 
 	// Find the next run number
 	var currentRun int
-	row := tx.QueryRow("select max(run) from videos")
+	err = tx.Query("select max(run) from videos").ScanSingle(&currentRun)
-	err = row.Scan(&currentRun)
 	if err != nil {
 		return fmt.Errorf("error retrieving current run: %w", err)
 	}
@@ -165,9 +165,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
 
 			// Ensure the record does not already exist. If it does, we've fetched
 			// all new videos
-			result := tx.QueryRow("select count(1) from videos where url = :url", href)
+			err = tx.Query("select count(1) from videos  where url = :url").Bind(href).ScanSingle(&count)
-			var count int
-			err = result.Scan(&count)
 			if err != nil {
 				return fmt.Errorf("error fetching data from db: %w", err)
 			}
@@ -178,7 +176,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
 
 			// Insert it into the database
 			log.Printf("Adding video %s", href)
-			_, err = tx.Exec("insert into videos(url, thumbnail, run) values (?, ?, ?)", href, thumbnail, currentRun)
+			err = tx.Query("insert into videos(url, thumbnail, run) values (?, ?, ?)").Bind(href, thumbnail, currentRun).Exec()
 			if err != nil {
 				return fmt.Errorf("error inserting into db: %w", err)
 			}
@@ -204,14 +202,13 @@ func isRelativeTimeFormat(input string) bool {
 	return re.MatchString(input)
 }
 
-func (w *WebClient) FetchVideoMetadata(db *sql.DB) error {
+func (w *WebClient) FetchVideoMetadata(db *mysqlite.Db) error {
 	log.Printf("Fetching video metadata...")
 	for {
 		// Fetch the next record from the database
-		row := db.QueryRow("select id, url from videos where `cast` is null limit 1")
 		var id int
 		var href string
-		err := row.Scan(&id, &href)
+		err := db.Query("select id, url from videos where `cast` is null limit 1").ScanSingle(&id, &href)
 		if errors.Is(err, sql.ErrNoRows) {
 			log.Printf("Fetched all metadata")
 			return nil
@@ -263,29 +260,31 @@ func (w *WebClient) FetchVideoMetadata(db *sql.DB) error {
 		castSource, err := videoElement.GetAttribute("cast-src")
 
 		// Store info in database
-		tx, err := db.Begin()
+		err = updateVideoMetadata(db, id, title, description, castSource, uploadDate.Format(time.DateOnly), uploadDate.Year())
 		if err != nil {
-			return fmt.Errorf("error starting transaction: %w", err)
+			return err
-		}
-		defer tx.Rollback()
-		result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ?, year = ? where id = ?", title, description, castSource, uploadDate.Format(time.DateOnly), uploadDate.Year(), id)
-		if err != nil {
-			return fmt.Errorf("error updating database: %w", err)
-		}
-		rowsAffected, err := result.RowsAffected()
-		if err != nil {
-			return fmt.Errorf("error retrieving affected rows: %w", err)
-		}
-		if rowsAffected != 1 {
-			return fmt.Errorf("unexpected number of rows affected: %d", rowsAffected)
-		}
-		err = tx.Commit()
-		if err != nil {
-			return fmt.Errorf("error commiting changeds: %w", err)
 		}
 	}
 }
 
+func updateVideoMetadata(db *mysqlite.Db, id int, title, description, castSource, uploadDate string, year int) error {
+	tx, err := db.Begin()
+	if err != nil {
+		return fmt.Errorf("error starting transaction: %w", err)
+	}
+	defer tx.MustRollback()
+	err = tx.Query("update videos set title = ?, description = ?, cast = ?, upload_date = ?, year = ? where id = ?").
+		Bind(title, description, castSource, uploadDate, year, id).Exec()
+	if err != nil {
+		return fmt.Errorf("error updating database: %w", err)
+	}
+	err = tx.Commit()
+	if err != nil {
+		return fmt.Errorf("error commiting changeds: %w", err)
+	}
+	return nil
+}
+
 func (w *WebClient) getInnerText(selector string) (string, error) {
 	// Get video title
 	titleElement, err := w.page.QuerySelector(selector)