Can now actually download files

downloader.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+)
+
+func DownloadAllVideos(db *sql.DB) error {
+	// Ensure no partial videos exist
+	tempDir := os.Getenv("TMP_DOWNLOAD")
+	if tempDir == "" {
+		tempDir = "./temp" // Replace with your desired default path
+	}
+	err := os.RemoveAll(tempDir)
+	if err != nil {
+		return fmt.Errorf("error cleaning temporary directory: %w", err)
+	}
+	err = os.MkdirAll(tempDir, 0750)
+	if err != nil {
+		return fmt.Errorf("error creating temporary directory: %w", err)
+	}
+
+	// Find the target directory
+	downloadDir := os.Getenv("DESTINATION")
+	if downloadDir == "" {
+		downloadDir = "./downloads"
+	}
+	err = os.MkdirAll(downloadDir, 0750)
+	if err != nil {
+		return fmt.Errorf("error creating destination directory: %w", err)
+	}
+
+	log.Printf("Starting downloads...")
+	for {
+		// Fetch the next record from the database
+		row := db.QueryRow("select id, url, `cast`, title, description, upload_date, thumbnail from videos where state = 'pending' limit 1")
+		var id int
+		var href, cast, title, description, uploadDateStr, thumbnailUrl string
+		err = row.Scan(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl)
+		if errors.Is(err, sql.ErrNoRows) {
+			log.Printf("No videos found for downloading")
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("error fetching record: %w", err)
+		}
+		uploadDate, err := time.Parse(time.DateOnly, uploadDateStr)
+		if err != nil {
+			return fmt.Errorf("error parsing upload date '%s': %w", uploadDateStr, err)
+		}
+
+		// Download the actual video
+		baseFileName := fmt.Sprintf("/s%de1%02d%02d", uploadDate.Year(), uploadDate.Month(), uploadDate.Day())
+		log.Printf("Downloading %s", href)
+		cmd := exec.Command("yt-dlp", cast, "-o", filepath.Join(tempDir, baseFileName+".%(ext)s"))
+		cmd.Stderr = os.Stderr
+		cmd.Stdout = os.Stdout
+		err = cmd.Run()
+		if err != nil {
+			return fmt.Errorf("error downloading video: %w", err)
+		}
+
+		// Move it into the destination directory
+		destinationDir := filepath.Join(downloadDir, fmt.Sprintf("Season %d", uploadDate.Year()))
+		err = os.MkdirAll(destinationDir, 0750)
+		if err != nil {
+			return fmt.Errorf("error creating target directory: %w", err)
+		}
+		files, err := os.ReadDir(tempDir)
+		if err != nil {
+			return fmt.Errorf("error retrieving downloaded files: %w", err)
+		}
+
+		for _, file := range files {
+			srcPath := filepath.Join(tempDir, file.Name())
+			destPath := filepath.Join(destinationDir, file.Name())
+			fmt.Printf("Moving %s to %s", srcPath, destPath)
+			err = os.Rename(srcPath, destPath)
+			if err != nil {
+				return fmt.Errorf("error moving file: %w", err)
+			}
+		}
+
+		// Write XML sidecar
+		nfo := NFO{}
+		nfo.Aired = uploadDateStr
+		nfo.Plot = description
+		nfo.Title = title
+		nfo.Year = uploadDate.Year()
+		nfoData, err := xml.MarshalIndent(nfo, "", "    ")
+		if err != nil {
+			return fmt.Errorf("error marshalling NFO data: %w", err)
+		}
+		nfoString := xml.Header + string(nfoData)
+		nfoFile, err := os.Create(filepath.Join(destinationDir, baseFileName+".nfo"))
+		defer nfoFile.Close()
+		_, err = nfoFile.WriteString(nfoString)
+		if err != nil {
+			return fmt.Errorf("error writing NFO file: %w", err)
+		}
+
+		// Write thumbnail
+		thumbnailExt := filepath.Ext(thumbnailUrl)
+		thumbnailFile, err := os.Create(filepath.Join(destinationDir, baseFileName+"-thumb"+thumbnailExt))
+		if err != nil {
+			return fmt.Errorf("error creating thumbnail: %w", err)
+		}
+		defer thumbnailFile.Close()
+		thumbnailResp, err := http.Get(thumbnailUrl)
+		if err != nil {
+			return fmt.Errorf("error fetching thumbnail: %w", err)
+		}
+		defer thumbnailResp.Body.Close()
+		_, err = io.Copy(thumbnailFile, thumbnailResp.Body)
+		if err != nil {
+			return fmt.Errorf("error writing thumbnail: %w", err)
+		}
+
+		// Set the database state to done
+		tx, err := db.Begin()
+		if err != nil {
+			return fmt.Errorf("error starting transaction: %w", err)
+		}
+		defer tx.Rollback()
+		result, err := tx.Exec("update videos set state = 'done' where id = ?", id)
+		if err != nil {
+			return fmt.Errorf("error updating database: %w", err)
+		}
+		count, err := result.RowsAffected()
+		if err != nil {
+			return fmt.Errorf("error getting number of rows affected: %d", err)
+		}
+		if count != 1 {
+			return fmt.Errorf("unexpected number of rows changed (expected exactly 1): %d", count)
+		}
+		err = tx.Commit()
+		if err != nil {
+			return fmt.Errorf("error commiting transaction: %w", err)
+		}
+
+		fmt.Printf("Finished downloading file")
+	}
+}

main.go

@@ -5,6 +5,8 @@ import (
 	"github.com/playwright-community/playwright-go"
 	"log"
 	"os"
+	"strconv"
+	"time"
 )
 
 func main() {
@@ -19,15 +21,23 @@ func main() {
 	db := openDatabase()
 	defer db.Close()
 
-	w := NewWebClient(options)
+	sleepTimeStr := os.Getenv("VIVAPLUS_SLEEPTIME")
+	sleepTime := 15
+	if sleepTimeStr != "" {
+		sleepTime, err = strconv.Atoi(sleepTimeStr)
+		if err != nil {
+			log.Fatalf("error parsing sleep time: %v", err)
+		}
+	}
 
+	w := NewWebClient(options)
 	username := os.Getenv("VIVAPLUS_USER")
-	//password := os.Getenv("VIVAPLUS_PASS")
 	password, err := base64.StdEncoding.DecodeString(os.Getenv("VIVAPLUS_PASS"))
 	if err != nil {
 		log.Fatalf("error decoding password: %v", err)
 	}
 
+	for {
 		err = w.Login(username, string(password))
 		if err != nil {
 			log.Fatalf("error login in: %v", err)
@@ -42,4 +52,13 @@ func main() {
 		if err != nil {
 			panic(err)
 		}
+
+		err = DownloadAllVideos(db)
+		if err != nil {
+			panic(err)
+		}
+
+		log.Printf("Sleeping %d minutes until next run", sleepTime)
+		time.Sleep(time.Duration(sleepTime) * time.Minute)
+	}
 }

migrations/1_init.sql

@@ -5,4 +5,4 @@ CREATE TABLE IF NOT EXISTS videos (
     inserted_on DATETIME DEFAULT CURRENT_TIMESTAMP
 );
 
-INSERT INTO videos (url) values ('/supporters/videos/81886')
+INSERT INTO videos (url) values ('/supporters/videos/81886');

migrations/2_add_columns.sql

@@ -1,3 +0,0 @@
-ALTER TABLE videos ADD COLUMN upload_date;
-ALTER TABLE videos ADD COLUMN cast;
-ALTER TABLE videos ADD COLUMN description;

migrations/2_add_metadata_columns.sql

@@ -0,0 +1,3 @@
+ALTER TABLE videos ADD COLUMN upload_date TEXT;
+ALTER TABLE videos ADD COLUMN cast TEXT;
+ALTER TABLE videos ADD COLUMN description TEXT;

migrations/3_add_state_thumbnail_columns.sql

@@ -0,0 +1,4 @@
+ALTER TABLE videos ADD COLUMN state TEXT NOT NULL DEFAULT 'pending';
+ALTER TABLE videos ADD COLUMN thumbnail TEXT;
+
+UPDATE videos SET thumbnail='https://imgproxy.fourthwall.com/ymbpbC7b9fO7tWlsg3e8sEdv8b3TuqXDGjr5FNM-uIg/rs:fill:607:342:0/sm:1/enc/N2U3ODcwN2I3Y2Iw/ZTZlYgoXdyrvsfaC/wje2R4f5TYcCwB4Y/ilNrHvoVDCjpBH8z/xnY0ho0FU6HlR9bX/Pk1pfFXOIFCqKnm6/2SKEU7hk5GgZTVXP/vDclApaRvgHzNXsF/pT8enEz076L7hSD-/0rENKD99989AC33R/5ZkKjg1ZZSc.webp' WHERE url = '/supporters/videos/81886';

nfo.go

@@ -0,0 +1,9 @@
+package main
+
+type NFO struct {
+	XMLName struct{} `xml:"episodedetails"`
+	Title   string   `xml:"title"`
+	Year    int      `xml:"year"`
+	Aired   string   `xml:"aired"`
+	Plot    string   `xml:"plot"`
+}

vivaweb.go

@@ -151,7 +151,15 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
 				return fmt.Errorf("url has bad format: %s", href)
 			}
 
-			// Insert it into the database
+			// Get thumbnail
+			thumbnailEl := l.Locator(".video__image:first-child")
+			thumbnail, err := thumbnailEl.GetAttribute("src")
+			if err != nil {
+				return fmt.Errorf("error retrieving thumbnail: %w", err)
+			}
+
+			// Ensure the record does not already exist. If it does, we've fetched
+			// all new videos
 			result := tx.QueryRow("select count(1) from videos where url = :url", href)
 			var count int
 			err = result.Scan(&count)
@@ -163,8 +171,9 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error {
 				goto finish
 			}
 
+			// Insert it into the database
 			log.Printf("Adding video %s", href)
-			_, err = tx.Exec("insert into videos(url) values (:url)", href)
+			_, err = tx.Exec("insert into videos(url, thumbnail) values (?, ?)", href, thumbnail)
 			if err != nil {
 				return fmt.Errorf("error inserting into db: %w", err)
 			}
@@ -205,7 +214,7 @@ func (w *WebClient) FetchVideoMetadata(db *sql.DB) error {
 		if err != nil {
 			return fmt.Errorf("error fetching record: %w", err)
 		}
-		log.Printf("Fetching data from %s", href)
+		log.Printf("Fetching data for %s", href)
 
 		// Fetch the video metadata from the web page
 		_, err = w.page.Goto(BASE_URL + href)
@@ -254,7 +263,7 @@ func (w *WebClient) FetchVideoMetadata(db *sql.DB) error {
 			return fmt.Errorf("error starting transaction: %w", err)
 		}
 		defer tx.Rollback()
-		result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ? where id = ?", title, description, castSource, uploadDate, id)
+		result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ? where id = ?", title, description, castSource, uploadDate.Format(time.DateOnly), id)
 		if err != nil {
 			return fmt.Errorf("error updating database: %w", err)
 		}