From a27ca2c8282e3876578fffb9ba35f0da8e368a57 Mon Sep 17 00:00:00 2001 From: Sebastiaan de Schaetzen Date: Fri, 7 Feb 2025 15:35:46 +0100 Subject: [PATCH] Improve episode numbering system --- downloader.go | 10 ++-- episodes.go | 63 ++++++++++++++++++++ main.go | 5 ++ migrations/1_init.sql | 12 +++- migrations/2_add_metadata_columns.sql | 3 - migrations/3_add_state_thumbnail_columns.sql | 4 -- vivaweb.go | 17 ++++-- 7 files changed, 94 insertions(+), 20 deletions(-) create mode 100644 episodes.go delete mode 100644 migrations/2_add_metadata_columns.sql delete mode 100644 migrations/3_add_state_thumbnail_columns.sql diff --git a/downloader.go b/downloader.go index d42f887..8f80aa8 100644 --- a/downloader.go +++ b/downloader.go @@ -43,10 +43,10 @@ func DownloadAllVideos(db *sql.DB) error { log.Printf("Starting downloads...") for { // Fetch the next record from the database - row := db.QueryRow("select id, url, `cast`, title, description, upload_date, thumbnail from videos where state = 'pending' order by id desc limit 1") - var id int + row := db.QueryRow("select id, url, `cast`, title, description, upload_date, thumbnail, episode from videos where state = 'pending' and episode is not null order by year, episode limit 1") + var id, episode int var href, cast, title, description, uploadDateStr, thumbnailUrl string - err = row.Scan(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl) + err = row.Scan(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl, &episode) if errors.Is(err, sql.ErrNoRows) { log.Printf("No videos found for downloading") return nil @@ -60,7 +60,7 @@ func DownloadAllVideos(db *sql.DB) error { } // Download the actual video - baseFileName := fmt.Sprintf("/s%de1%02d%02d", uploadDate.Year(), uploadDate.Month(), uploadDate.Day()) + baseFileName := fmt.Sprintf("/S%dE%03d", uploadDate.Year(), episode) log.Printf("Downloading %s", href) cmd := exec.Command("yt-dlp", cast, "-o", filepath.Join(tempDir, baseFileName+".%(ext)s")) cmd.Stderr = os.Stderr @@ -93,10 +93,10 @@ func DownloadAllVideos(db *sql.DB) error { // Write XML sidecar nfo := NFO{} - nfo.Aired = uploadDateStr nfo.Plot = description nfo.Title = title nfo.Year = uploadDate.Year() + nfo.Aired = uploadDate.Format(time.DateOnly) nfoData, err := xml.MarshalIndent(nfo, "", " ") if err != nil { return fmt.Errorf("error marshalling NFO data: %w", err) diff --git a/episodes.go b/episodes.go new file mode 100644 index 0000000..ec4bccc --- /dev/null +++ b/episodes.go @@ -0,0 +1,63 @@ +package main + +import ( + "database/sql" + "fmt" + "log" +) + +func CalculateEpisodeNumbers(db *sql.DB) error { + log.Printf("Calculating episode numbers") + + // Start a transaction + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("error starting transaction: %w", err) + } + defer tx.Rollback() + + // First find all years that still need episodes numbers + rows, err := tx.Query("select year, max(ifnull(episode, 0)) from videos where episode is null group by year") + if err != nil { + return fmt.Errorf("error retrieving rows: %w", err) + } + + // Loop over each year and set the missing episode numbers + for rows.Next() { + var year, lastEpisode int + err = rows.Scan(&year, &lastEpisode) + if err != nil { + return fmt.Errorf("error getting row: %w", err) + } + + // Find all episodes that need updating + episodes, err := tx.Query("select id from videos where year = ? and episode is null order by run, id desc", year) + if err != nil { + return fmt.Errorf("error retrieving episodes: %w", err) + } + + // Update each episode + for episodes.Next() { + var episodeId int + err = episodes.Scan(&episodeId) + if err != nil { + return fmt.Errorf("error retrieving episode id: %w", err) + } + + // Set the episode ID + lastEpisode++ + _, err = tx.Exec("update videos set episode = ? where id = ?", &lastEpisode, &episodeId) + if err != nil { + return fmt.Errorf("error updating episode id: %w", err) + } + } + } + + // Commit all changes + err = tx.Commit() + if err != nil { + return fmt.Errorf("error commiting changes: %w", err) + } + + return nil +} diff --git a/main.go b/main.go index a777833..1621f1d 100644 --- a/main.go +++ b/main.go @@ -53,6 +53,11 @@ func main() { panic(err) } + err = CalculateEpisodeNumbers(db) + if err != nil { + panic(err) + } + err = DownloadAllVideos(db) if err != nil { panic(err) diff --git a/migrations/1_init.sql b/migrations/1_init.sql index 443b691..dbb77dd 100644 --- a/migrations/1_init.sql +++ b/migrations/1_init.sql @@ -2,7 +2,15 @@ CREATE TABLE IF NOT EXISTS videos ( id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, url TEXT NOT NULL UNIQUE, - inserted_on DATETIME DEFAULT CURRENT_TIMESTAMP + inserted_on TEXT DEFAULT CURRENT_TIMESTAMP, + upload_date TEXT, + cast TEXT, + description TEXT, + year INTEGER, + episode INTEGER, + run INTEGER, + state TEXT NOT NULL DEFAULT 'pending', + thumbnail TEXT ); -INSERT INTO videos (url) values ('/supporters/videos/81886'); +INSERT INTO videos (url, run, thumbnail) values ('/supporters/videos/81886', 0, 'https://imgproxy.fourthwall.com/ymbpbC7b9fO7tWlsg3e8sEdv8b3TuqXDGjr5FNM-uIg/rs:fill:607:342:0/sm:1/enc/N2U3ODcwN2I3Y2Iw/ZTZlYgoXdyrvsfaC/wje2R4f5TYcCwB4Y/ilNrHvoVDCjpBH8z/xnY0ho0FU6HlR9bX/Pk1pfFXOIFCqKnm6/2SKEU7hk5GgZTVXP/vDclApaRvgHzNXsF/pT8enEz076L7hSD-/0rENKD99989AC33R/5ZkKjg1ZZSc.webp'); diff --git a/migrations/2_add_metadata_columns.sql b/migrations/2_add_metadata_columns.sql deleted file mode 100644 index 60d062d..0000000 --- a/migrations/2_add_metadata_columns.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE videos ADD COLUMN upload_date TEXT; -ALTER TABLE videos ADD COLUMN cast TEXT; -ALTER TABLE videos ADD COLUMN description TEXT; diff --git a/migrations/3_add_state_thumbnail_columns.sql b/migrations/3_add_state_thumbnail_columns.sql deleted file mode 100644 index 2458783..0000000 --- a/migrations/3_add_state_thumbnail_columns.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE videos ADD COLUMN state TEXT NOT NULL DEFAULT 'pending'; -ALTER TABLE videos ADD COLUMN thumbnail TEXT; - -UPDATE videos SET thumbnail='https://imgproxy.fourthwall.com/ymbpbC7b9fO7tWlsg3e8sEdv8b3TuqXDGjr5FNM-uIg/rs:fill:607:342:0/sm:1/enc/N2U3ODcwN2I3Y2Iw/ZTZlYgoXdyrvsfaC/wje2R4f5TYcCwB4Y/ilNrHvoVDCjpBH8z/xnY0ho0FU6HlR9bX/Pk1pfFXOIFCqKnm6/2SKEU7hk5GgZTVXP/vDclApaRvgHzNXsF/pT8enEz076L7hSD-/0rENKD99989AC33R/5ZkKjg1ZZSc.webp' WHERE url = '/supporters/videos/81886'; diff --git a/vivaweb.go b/vivaweb.go index 2718cd7..7f18771 100644 --- a/vivaweb.go +++ b/vivaweb.go @@ -108,11 +108,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error { return fmt.Errorf("error opening page: %w", err) } - //page.Keyboard().Press("End") - //time.Sleep(3 * time.Second) - count, err := w.page.GetByTestId("VideoCatalog.Video").Count() - //count, err := el.Count() println(count, err) println("Looping over videos...") @@ -122,6 +118,15 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error { } defer tx.Rollback() + // Find the next run number + var currentRun int + row := tx.QueryRow("select max(run) from videos") + err = row.Scan(¤tRun) + if err != nil { + return fmt.Errorf("error retrieving current run: %w", err) + } + currentRun++ + previousUrls := make(map[string]struct{}) for { @@ -173,7 +178,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error { // Insert it into the database log.Printf("Adding video %s", href) - _, err = tx.Exec("insert into videos(url, thumbnail) values (?, ?)", href, thumbnail) + _, err = tx.Exec("insert into videos(url, thumbnail, run) values (?, ?, ?)", href, thumbnail, currentRun) if err != nil { return fmt.Errorf("error inserting into db: %w", err) } @@ -263,7 +268,7 @@ func (w *WebClient) FetchVideoMetadata(db *sql.DB) error { return fmt.Errorf("error starting transaction: %w", err) } defer tx.Rollback() - result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ? where id = ?", title, description, castSource, uploadDate.Format(time.DateOnly), id) + result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ?, year = ? where id = ?", title, description, castSource, uploadDate.Format(time.DateOnly), uploadDate.Year(), id) if err != nil { return fmt.Errorf("error updating database: %w", err) }