Improve episode numbering system

downloader.go

@@ -43,10 +43,10 @@ func DownloadAllVideos(db *sql.DB) error {
 	log.Printf("Starting downloads...")
 	for {
 		// Fetch the next record from the database
-		row := db.QueryRow("select id, url, `cast`, title, description, upload_date, thumbnail from videos where state = 'pending' order by id desc limit 1")
+		row := db.QueryRow("select id, url, `cast`, title, description, upload_date, thumbnail, episode from videos where state = 'pending' and episode is not null order by year, episode limit 1")
-		var id int
+		var id, episode int
 		var href, cast, title, description, uploadDateStr, thumbnailUrl string
-		err = row.Scan(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl)
+		err = row.Scan(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl, &episode)
 		if errors.Is(err, sql.ErrNoRows) {
 			log.Printf("No videos found for downloading")
 			return nil
@@ -60,7 +60,7 @@ func DownloadAllVideos(db *sql.DB) error {
 		}
 
 		// Download the actual video
-		baseFileName := fmt.Sprintf("/s%de1%02d%02d", uploadDate.Year(), uploadDate.Month(), uploadDate.Day())
+		baseFileName := fmt.Sprintf("/S%dE%03d", uploadDate.Year(), episode)
 		log.Printf("Downloading %s", href)
 		cmd := exec.Command("yt-dlp", cast, "-o", filepath.Join(tempDir, baseFileName+".%(ext)s"))
 		cmd.Stderr = os.Stderr
@@ -93,10 +93,10 @@ func DownloadAllVideos(db *sql.DB) error {
 
 		// Write XML sidecar
 		nfo := NFO{}
-		nfo.Aired = uploadDateStr
 		nfo.Plot = description
 		nfo.Title = title
 		nfo.Year = uploadDate.Year()
+		nfo.Aired = uploadDate.Format(time.DateOnly)
 		nfoData, err := xml.MarshalIndent(nfo, "", "    ")
 		if err != nil {
 			return fmt.Errorf("error marshalling NFO data: %w", err)

episodes.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+)
+
+func CalculateEpisodeNumbers(db *sql.DB) error {
+	log.Printf("Calculating episode numbers")
+
+	// Start a transaction
+	tx, err := db.Begin()
+	if err != nil {
+		return fmt.Errorf("error starting transaction: %w", err)
+	}
+	defer tx.Rollback()
+
+	// First find all years that still need episodes numbers
+	rows, err := tx.Query("select year, max(ifnull(episode, 0)) from videos where episode is null group by year")
+	if err != nil {
+		return fmt.Errorf("error retrieving rows: %w", err)
+	}
+
+	// Loop over each year and set the missing episode numbers
+	for rows.Next() {
+		var year, lastEpisode int
+		err = rows.Scan(&year, &lastEpisode)
+		if err != nil {
+			return fmt.Errorf("error getting row: %w", err)
+		}
+
+		// Find all episodes that need updating
+		episodes, err := tx.Query("select id from videos where year = ? and episode is null order by run, id desc", year)
+		if err != nil {
+			return fmt.Errorf("error retrieving episodes: %w", err)
+		}
+
+		// Update each episode
+		for episodes.Next() {
+			var episodeId int
+			err = episodes.Scan(&episodeId)
+			if err != nil {
+				return fmt.Errorf("error retrieving episode id: %w", err)
+			}
+
+			// Set the episode ID
+			lastEpisode++
+			_, err = tx.Exec("update videos set episode = ? where id = ?", &lastEpisode, &episodeId)
+			if err != nil {
+				return fmt.Errorf("error updating episode id: %w", err)
+			}
+		}
+	}
+
+	// Commit all changes
+	err = tx.Commit()
+	if err != nil {
+		return fmt.Errorf("error commiting changes: %w", err)
+	}
+
+	return nil
+}

main.go

@@ -53,6 +53,11 @@ func main() {
 			panic(err)
 		}
 
+		err = CalculateEpisodeNumbers(db)
+		if err != nil {
+			panic(err)
+		}
+
 		err = DownloadAllVideos(db)
 		if err != nil {
 			panic(err)

migrations/1_init.sql

@@ -2,7 +2,15 @@ CREATE TABLE IF NOT EXISTS videos (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     title TEXT,
     url TEXT NOT NULL UNIQUE,
-    inserted_on DATETIME DEFAULT CURRENT_TIMESTAMP
+    inserted_on TEXT DEFAULT CURRENT_TIMESTAMP,
+    upload_date TEXT,
+    cast TEXT,
+    description TEXT,
+    year INTEGER,
+    episode INTEGER,
+    run INTEGER,
+    state TEXT NOT NULL DEFAULT 'pending',
+    thumbnail TEXT
 );
 
-INSERT INTO videos (url) values ('/supporters/videos/81886');
+INSERT INTO videos (url, run, thumbnail) values ('/supporters/videos/81886', 0, 'https://imgproxy.fourthwall.com/ymbpbC7b9fO7tWlsg3e8sEdv8b3TuqXDGjr5FNM-uIg/rs:fill:607:342:0/sm:1/enc/N2U3ODcwN2I3Y2Iw/ZTZlYgoXdyrvsfaC/wje2R4f5TYcCwB4Y/ilNrHvoVDCjpBH8z/xnY0ho0FU6HlR9bX/Pk1pfFXOIFCqKnm6/2SKEU7hk5GgZTVXP/vDclApaRvgHzNXsF/pT8enEz076L7hSD-/0rENKD99989AC33R/5ZkKjg1ZZSc.webp');

migrations/2_add_metadata_columns.sql

@@ -1,3 +0,0 @@
-ALTER TABLE videos ADD COLUMN upload_date TEXT;
-ALTER TABLE videos ADD COLUMN cast TEXT;
-ALTER TABLE videos ADD COLUMN description TEXT;

migrations/3_add_state_thumbnail_columns.sql

@@ -1,4 +0,0 @@
-ALTER TABLE videos ADD COLUMN state TEXT NOT NULL DEFAULT 'pending';
-ALTER TABLE videos ADD COLUMN thumbnail TEXT;
-
-UPDATE videos SET thumbnail='https://imgproxy.fourthwall.com/ymbpbC7b9fO7tWlsg3e8sEdv8b3TuqXDGjr5FNM-uIg/rs:fill:607:342:0/sm:1/enc/N2U3ODcwN2I3Y2Iw/ZTZlYgoXdyrvsfaC/wje2R4f5TYcCwB4Y/ilNrHvoVDCjpBH8z/xnY0ho0FU6HlR9bX/Pk1pfFXOIFCqKnm6/2SKEU7hk5GgZTVXP/vDclApaRvgHzNXsF/pT8enEz076L7hSD-/0rENKD99989AC33R/5ZkKjg1ZZSc.webp' WHERE url = '/supporters/videos/81886';

vivaweb.go

@@ -108,11 +108,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
 		return fmt.Errorf("error opening page: %w", err)
 	}
 
-	//page.Keyboard().Press("End")
-	//time.Sleep(3 * time.Second)
-
 	count, err := w.page.GetByTestId("VideoCatalog.Video").Count()
-	//count, err := el.Count()
 	println(count, err)
 
 	println("Looping over videos...")
@@ -122,6 +118,15 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
 	}
 	defer tx.Rollback()
 
+	// Find the next run number
+	var currentRun int
+	row := tx.QueryRow("select max(run) from videos")
+	err = row.Scan(&currentRun)
+	if err != nil {
+		return fmt.Errorf("error retrieving current run: %w", err)
+	}
+	currentRun++
+
 	previousUrls := make(map[string]struct{})
 
 	for {
@@ -173,7 +178,7 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
 
 			// Insert it into the database
 			log.Printf("Adding video %s", href)
-			_, err = tx.Exec("insert into videos(url, thumbnail) values (?, ?)", href, thumbnail)
+			_, err = tx.Exec("insert into videos(url, thumbnail, run) values (?, ?, ?)", href, thumbnail, currentRun)
 			if err != nil {
 				return fmt.Errorf("error inserting into db: %w", err)
 			}
@@ -263,7 +268,7 @@ func (w *WebClient) FetchVideoMetadata(db *sql.DB) error {
 			return fmt.Errorf("error starting transaction: %w", err)
 		}
 		defer tx.Rollback()
-		result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ? where id = ?", title, description, castSource, uploadDate.Format(time.DateOnly), id)
+		result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ?, year = ? where id = ?", title, description, castSource, uploadDate.Format(time.DateOnly), uploadDate.Year(), id)
 		if err != nil {
 			return fmt.Errorf("error updating database: %w", err)
 		}