Add ability to fetch video links

main.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"encoding/base64"
 	"github.com/playwright-community/playwright-go"
 	"log"
+	"os"
 )
 
 func main() {
@@ -19,17 +21,18 @@ func main() {
 
 	w := NewWebClient(options)
 
-	//username := os.Getenv("VIVAPLUS_USER")
-	////password := os.Getenv("VIVAPLUS_PASS")
-	//password, err := base64.StdEncoding.DecodeString(os.Getenv("VIVAPLUS_PASS"))
-	//if err != nil {
-	//	log.Fatalf("error decoding password: %v", err)
-	//}
-	//
-	//err = w.VivaLogin(username, string(password))
-	//if err != nil {
-	//	log.Fatalf("error login in: %v", err)
-	//}
+	username := os.Getenv("VIVAPLUS_USER")
+	//password := os.Getenv("VIVAPLUS_PASS")
+	password, err := base64.StdEncoding.DecodeString(os.Getenv("VIVAPLUS_PASS"))
+	if err != nil {
+		log.Fatalf("error decoding password: %v", err)
+	}
 
-	w.VivaFindAllVideos()
+	err = w.VivaLogin(username, string(password))
+	if err != nil {
+		log.Fatalf("error login in: %v", err)
+	}
+
+	w.VivaDiscoverAllVideos(db)
+	//w.VivaAddMetadata(db)
 }

migrations/1_init.sql

@@ -1,6 +1,8 @@
 CREATE TABLE IF NOT EXISTS videos (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
-    title TEXT NOT NULL,
+    title TEXT,
     url TEXT NOT NULL UNIQUE,
-    uploaded_at DATETIME DEFAULT CURRENT_TIMESTAMP
+    inserted_on DATETIME DEFAULT CURRENT_TIMESTAMP
 );
+
+INSERT INTO videos (url) values ('/supporters/videos/81886')

vivaweb.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"database/sql"
 	"fmt"
 	"github.com/playwright-community/playwright-go"
 	"log"
@@ -12,6 +13,7 @@ const SIGN_IN_URL = BASE_URL + "supporters/sign_in"
 type WebClient struct {
 	pw      *playwright.Playwright
 	browser playwright.Browser
+	page    playwright.Page
 	//browser *browser.Browser
 	//csrfToken string
 	//cookies   *cookiejar.Jar
@@ -20,63 +22,122 @@ type WebClient struct {
 func NewWebClient(options *playwright.RunOptions) *WebClient {
 	pw, err := playwright.Run(options)
 	if err != nil {
-		log.Fatalf("error running playwright: %w", pw)
+		log.Fatalf("error running playwright: %w", err)
 	}
 	browser, err := pw.Firefox.Launch()
 	if err != nil {
-		log.Fatalf("error running firefox: %w", pw)
+		log.Fatalf("error running firefox: %w", err)
+	}
+	page, err := browser.NewPage()
+	if err != nil {
+		log.Fatalf("error creating page: %w", err)
 	}
 	return &WebClient{
 		pw:      pw,
 		browser: browser,
+		page:    page,
 	}
 }
 
 func (w *WebClient) VivaLogin(username, password string) error {
-	page, err := w.browser.NewPage()
-	if err != nil {
-		return fmt.Errorf("error creating page: %w", err)
-	}
-
-	_, err = page.Goto(SIGN_IN_URL)
+	_, err := w.page.Goto(SIGN_IN_URL)
 	if err != nil {
 		return fmt.Errorf("error navigating to sign-in page: %w", err)
 	}
 
-	err = page.GetByTestId("SupporterLogin.EmailInput").Fill(username)
+	err = w.page.GetByTestId("SupporterLogin.EmailInput").Fill(username)
 	if err != nil {
 		return fmt.Errorf("error filling in email: %w", err)
 	}
-	err = page.GetByTestId("SupporterLogin.PasswordInput").Fill(password)
+	err = w.page.GetByTestId("SupporterLogin.PasswordInput").Fill(password)
 	if err != nil {
 		return fmt.Errorf("error filling in email: %w", err)
 	}
 
-	err = page.GetByTestId("SupporterLogin.SubmitButton").Click()
+	log.Printf("Clicking login...")
+	err = w.page.GetByTestId("SupporterLogin.SubmitButton").Click()
+	if err != nil {
+		return fmt.Errorf("error clicking login: %w", err)
+	}
+
+	err = w.page.WaitForURL("https://vivaplus.tv/supporters")
+	if err != nil {
+		return fmt.Errorf("error waiting for url: %w", err)
+	}
+	log.Printf("Logged in!")
+
 	return nil
 }
 
-func (w *WebClient) VivaFindAllVideos() error {
+func (w *WebClient) VivaDiscoverAllVideos(db *sql.DB) error {
 	log.Printf("Loading list of all videos...")
-	page, err := w.browser.NewPage()
-	if err != nil {
-		return fmt.Errorf("error creating page: %w", err)
-	}
 
-	_, err = page.Goto("https://vivaplus.tv/supporters/videos/all?order=asc")
+	_, err := w.page.Goto("https://vivaplus.tv/supporters/videos/all?order=desc")
 	if err != nil {
 		return fmt.Errorf("error opening page: %w", err)
 	}
 
-	count, err := page.GetByTestId("VideoCatalog.Video").Count()
+	//page.Keyboard().Press("End")
+	//time.Sleep(3 * time.Second)
+
+	count, err := w.page.GetByTestId("VideoCatalog.Video").Count()
 	//count, err := el.Count()
 	println(count, err)
-	//attr, err := el.GetAttribute("busy")
-	//println(attr, err)
-	//err = el.ScrollIntoViewIfNeeded()
-	//println(err)
-	//attr, err = el.GetAttribute("busy")
-	//println(attr, err)
 
+	println("Looping over videos...")
+	tx, err := db.Begin()
+	if err != nil {
+		return fmt.Errorf("error starting transaction: %w", err)
+	}
+	defer tx.Rollback()
+
+	previousUrls := make(map[string]struct{})
+
+	for {
+		locators, err := w.page.GetByTestId("VideoCatalog.Video").All()
+		if err != nil {
+			return err
+		}
+
+		for _, l := range locators {
+			href, err := l.GetAttribute("href")
+			if err != nil {
+				return err
+			}
+
+			if _, exists := previousUrls[href]; exists {
+				continue // The item was already scanned.
+			}
+			previousUrls[href] = struct{}{}
+
+			result := tx.QueryRow("select count(1) from videos where url = :url", href)
+			var count int
+			err = result.Scan(&count)
+			if err != nil {
+				return fmt.Errorf("error fetching data from db: %w", err)
+			}
+			if count == 1 {
+				log.Printf("Video already exists: %s", href)
+				goto finish
+			}
+
+			log.Printf("Adding video %s", href)
+			_, err = tx.Exec("insert into videos(url) values (:url)", href)
+			if err != nil {
+				return fmt.Errorf("error inserting into db: %w", err)
+			}
+		}
+		log.Printf("Scrolling...")
+		err = w.page.Keyboard().Press("End")
+		if err != nil {
+			return fmt.Errorf("error scrolling to end of page: %w", err)
+		}
+		//w.page.tim
+	}
+finish:
+	err = tx.Commit()
+	if err != nil {
+		return fmt.Errorf("error committing: %w", err)
+	}
 	return nil
 }