Working on porting from sqlite to mysqlite
Some checks failed
Build / build (push) Failing after 1m24s
Some checks failed
Build / build (push) Failing after 1m24s
This commit is contained in:
parent
a928d79504
commit
4b42e4f998
80
database.go
80
database.go
@ -1,101 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"embed"
|
||||
"fmt"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"gitea.seeseepuff.be/seeseemelk/mysqlite"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var embeddedMigrations embed.FS
|
||||
|
||||
func openDatabase() *sql.DB {
|
||||
func openDatabase() *mysqlite.Db {
|
||||
// Get database file
|
||||
databaseSource := os.Getenv("VIVAPLUS_DATABASE")
|
||||
if databaseSource == "" {
|
||||
databaseSource = "videos.db3"
|
||||
}
|
||||
return openDatabaseSource(databaseSource)
|
||||
}
|
||||
|
||||
func openDatabaseSource(databaseSource string) *sql.DB {
|
||||
// Initialize the database connection
|
||||
db, err := sql.Open("sqlite3", databaseSource)
|
||||
db, err := mysqlite.OpenDb(databaseSource)
|
||||
if err != nil {
|
||||
log.Fatalf("error opening database: %v", err)
|
||||
}
|
||||
|
||||
// Read all migrations
|
||||
migrationFiles, err := embeddedMigrations.ReadDir("migrations")
|
||||
err = db.MigrateDb(embeddedMigrations, "migrations")
|
||||
if err != nil {
|
||||
log.Fatalf("error reading migration files: %v", err)
|
||||
}
|
||||
var migrations = make(map[int]string)
|
||||
latestVersion := 0
|
||||
for _, f := range migrationFiles {
|
||||
versionStr := f.Name()
|
||||
version, err := strconv.Atoi(strings.SplitN(versionStr, "_", 2)[0])
|
||||
if err != nil {
|
||||
log.Fatalf("invalid version number for migration script: %v", err)
|
||||
}
|
||||
migrations[version] = versionStr
|
||||
latestVersion = max(latestVersion, version)
|
||||
log.Fatalf("error migrating database: %v", err)
|
||||
}
|
||||
|
||||
// Get current migration version from user_version
|
||||
var currentVersion int
|
||||
err = db.QueryRow("PRAGMA user_version").Scan(¤tVersion)
|
||||
if err != nil {
|
||||
log.Fatalf("error getting current version: %v", err)
|
||||
}
|
||||
log.Printf("Current database migration version is %d, latest version is %d", currentVersion, latestVersion)
|
||||
|
||||
// If we are no up-to-date, bring the db up-to-date
|
||||
for currentVersion != latestVersion {
|
||||
targetVersion := currentVersion + 1
|
||||
migrationFile := migrations[targetVersion]
|
||||
log.Printf("migration to version %s", migrationFile)
|
||||
migrationScript, err := embeddedMigrations.ReadFile("migrations/" + migrationFile)
|
||||
if err != nil {
|
||||
log.Fatalf("error opening migration script %s: %v", migrationScript, err)
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
log.Fatalf("error beginning transaction: %v", err)
|
||||
}
|
||||
|
||||
_, err = tx.Exec(string(migrationScript))
|
||||
if err != nil {
|
||||
rollbackIgnoringErrors(tx)
|
||||
log.Fatalf("error performing migration: %v", err)
|
||||
}
|
||||
|
||||
_, err = tx.Exec(fmt.Sprintf("PRAGMA user_version = %d", targetVersion))
|
||||
if err != nil {
|
||||
rollbackIgnoringErrors(tx)
|
||||
log.Fatalf("error updating version: %v", err)
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
log.Fatalf("error commiting transaction: %v", err)
|
||||
}
|
||||
currentVersion = targetVersion
|
||||
}
|
||||
|
||||
log.Println("All migrations applied")
|
||||
return db
|
||||
}
|
||||
|
||||
func rollbackIgnoringErrors(tx *sql.Tx) {
|
||||
err := tx.Rollback()
|
||||
if err != nil {
|
||||
log.Printf("error rolling back: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.seeseepuff.be/seeseemelk/mysqlite"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
@ -15,7 +16,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func DownloadAllVideos(db *sql.DB) error {
|
||||
func DownloadAllVideos(db *mysqlite.Db) error {
|
||||
// Ensure no partial videos exist
|
||||
tempDir := os.Getenv("VIVAPLUS_DOWNLOAD")
|
||||
if tempDir == "" {
|
||||
@ -43,10 +44,10 @@ func DownloadAllVideos(db *sql.DB) error {
|
||||
log.Printf("Starting downloads...")
|
||||
for {
|
||||
// Fetch the next record from the database
|
||||
row := db.QueryRow("select id, url, `cast`, title, description, upload_date, thumbnail, episode from videos where state = 'pending' and episode is not null order by year, episode limit 1")
|
||||
var id, episode int
|
||||
var href, cast, title, description, uploadDateStr, thumbnailUrl string
|
||||
err = row.Scan(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl, &episode)
|
||||
err = db.Query("select id, url, `cast`, title, description, upload_date, thumbnail, episode from videos where state = 'pending' and episode is not null order by year, episode limit 1").
|
||||
ScanSingle(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl, &episode)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
log.Printf("No videos found for downloading")
|
||||
return nil
|
||||
|
||||
15
go.mod
15
go.mod
@ -1,6 +1,8 @@
|
||||
module vivaplusdl
|
||||
|
||||
go 1.23
|
||||
go 1.24
|
||||
|
||||
toolchain go1.24.0
|
||||
|
||||
require (
|
||||
github.com/mattn/go-sqlite3 v1.14.24
|
||||
@ -8,7 +10,18 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
gitea.seeseepuff.be/seeseemelk/mysqlite v0.3.0 // indirect
|
||||
github.com/deckarep/golang-set/v2 v2.7.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-jose/go-jose/v3 v3.0.3 // indirect
|
||||
github.com/go-stack/stack v1.8.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.8.0 // indirect
|
||||
modernc.org/sqlite v1.33.1 // indirect
|
||||
)
|
||||
|
||||
25
go.sum
25
go.sum
@ -1,21 +1,33 @@
|
||||
gitea.seeseepuff.be/seeseemelk/mysqlite v0.3.0 h1:T2TjZj7hvb3KF0zYf/uzidprcDJIBPFi3QcN+gmyutM=
|
||||
gitea.seeseepuff.be/seeseemelk/mysqlite v0.3.0/go.mod h1:cgswydOxJjMlNwfcBIXnKjr47LwXnMT9BInkiHb0tXE=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/deckarep/golang-set/v2 v2.7.0 h1:gIloKvD7yH2oip4VLhsv3JyLLFnC0Y2mlusgcvJYW5k=
|
||||
github.com/deckarep/golang-set/v2 v2.7.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=
|
||||
github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
|
||||
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
|
||||
github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
|
||||
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
|
||||
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/playwright-community/playwright-go v0.4902.0 h1:SslPUKmc35YgTBZKTLhokxrqTsVk3/mirj+TkqR6dC0=
|
||||
github.com/playwright-community/playwright-go v0.4902.0/go.mod h1:kBNWs/w2aJ2ZUp1wEOOFLXgOqvppFngM5OS+qyhl+ZM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
@ -40,8 +52,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
@ -62,3 +77,13 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
|
||||
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
|
||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
||||
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
|
||||
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
|
||||
modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM=
|
||||
modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k=
|
||||
zombiezen.com/go/sqlite v1.4.0 h1:N1s3RIljwtp4541Y8rM880qgGIgq3fTD2yks1xftnKU=
|
||||
zombiezen.com/go/sqlite v1.4.0/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik=
|
||||
|
||||
57
vivaweb.go
57
vivaweb.go
@ -4,6 +4,7 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.seeseepuff.be/seeseemelk/mysqlite"
|
||||
"github.com/playwright-community/playwright-go"
|
||||
"log"
|
||||
"regexp"
|
||||
@ -100,7 +101,7 @@ func parseDateString(dateStr string) (time.Time, error) {
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
|
||||
func (w *WebClient) DiscoverAllVideos(db *mysqlite.Db) error {
|
||||
log.Printf("Loading list of all videos...")
|
||||
|
||||
_, err := w.page.Goto("https://vivaplus.tv/supporters/videos/all?order=desc")
|
||||
@ -116,12 +117,11 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("error starting transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
defer tx.MustRollback()
|
||||
|
||||
// Find the next run number
|
||||
var currentRun int
|
||||
row := tx.QueryRow("select max(run) from videos")
|
||||
err = row.Scan(¤tRun)
|
||||
err = tx.Query("select max(run) from videos").ScanSingle(¤tRun)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error retrieving current run: %w", err)
|
||||
}
|
||||
@ -165,9 +165,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
|
||||
|
||||
// Ensure the record does not already exist. If it does, we've fetched
|
||||
// all new videos
|
||||
result := tx.QueryRow("select count(1) from videos where url = :url", href)
|
||||
var count int
|
||||
err = result.Scan(&count)
|
||||
err = tx.Query("select count(1) from videos where url = :url").Bind(href).ScanSingle(&count)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error fetching data from db: %w", err)
|
||||
}
|
||||
@ -178,7 +176,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
|
||||
|
||||
// Insert it into the database
|
||||
log.Printf("Adding video %s", href)
|
||||
_, err = tx.Exec("insert into videos(url, thumbnail, run) values (?, ?, ?)", href, thumbnail, currentRun)
|
||||
err = tx.Query("insert into videos(url, thumbnail, run) values (?, ?, ?)").Bind(href, thumbnail, currentRun).Exec()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error inserting into db: %w", err)
|
||||
}
|
||||
@ -204,14 +202,13 @@ func isRelativeTimeFormat(input string) bool {
|
||||
return re.MatchString(input)
|
||||
}
|
||||
|
||||
func (w *WebClient) FetchVideoMetadata(db *sql.DB) error {
|
||||
func (w *WebClient) FetchVideoMetadata(db *mysqlite.Db) error {
|
||||
log.Printf("Fetching video metadata...")
|
||||
for {
|
||||
// Fetch the next record from the database
|
||||
row := db.QueryRow("select id, url from videos where `cast` is null limit 1")
|
||||
var id int
|
||||
var href string
|
||||
err := row.Scan(&id, &href)
|
||||
err := db.Query("select id, url from videos where `cast` is null limit 1").ScanSingle(&id, &href)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
log.Printf("Fetched all metadata")
|
||||
return nil
|
||||
@ -263,29 +260,31 @@ func (w *WebClient) FetchVideoMetadata(db *sql.DB) error {
|
||||
castSource, err := videoElement.GetAttribute("cast-src")
|
||||
|
||||
// Store info in database
|
||||
tx, err := db.Begin()
|
||||
err = updateVideoMetadata(db, id, title, description, castSource, uploadDate.Format(time.DateOnly), uploadDate.Year())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error starting transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ?, year = ? where id = ?", title, description, castSource, uploadDate.Format(time.DateOnly), uploadDate.Year(), id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error updating database: %w", err)
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error retrieving affected rows: %w", err)
|
||||
}
|
||||
if rowsAffected != 1 {
|
||||
return fmt.Errorf("unexpected number of rows affected: %d", rowsAffected)
|
||||
}
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error commiting changeds: %w", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func updateVideoMetadata(db *mysqlite.Db, id int, title, description, castSource, uploadDate string, year int) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error starting transaction: %w", err)
|
||||
}
|
||||
defer tx.MustRollback()
|
||||
err = tx.Query("update videos set title = ?, description = ?, cast = ?, upload_date = ?, year = ? where id = ?").
|
||||
Bind(title, description, castSource, uploadDate, year, id).Exec()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error updating database: %w", err)
|
||||
}
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error commiting changeds: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *WebClient) getInnerText(selector string) (string, error) {
|
||||
// Get video title
|
||||
titleElement, err := w.page.QuerySelector(selector)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user