From 4170686ec612a4a50b498cd82fa1d2f5226ca2b9 Mon Sep 17 00:00:00 2001 From: Sebastiaan de Schaetzen Date: Tue, 4 Feb 2025 20:02:11 +0100 Subject: [PATCH] Add ability to fetch video links --- main.go | 27 ++++++----- migrations/1_init.sql | 6 ++- vivaweb.go | 109 ++++++++++++++++++++++++++++++++---------- 3 files changed, 104 insertions(+), 38 deletions(-) diff --git a/main.go b/main.go index 1c00b2e..752b3cf 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,10 @@ package main import ( + "encoding/base64" "github.com/playwright-community/playwright-go" "log" + "os" ) func main() { @@ -19,17 +21,18 @@ func main() { w := NewWebClient(options) - //username := os.Getenv("VIVAPLUS_USER") - ////password := os.Getenv("VIVAPLUS_PASS") - //password, err := base64.StdEncoding.DecodeString(os.Getenv("VIVAPLUS_PASS")) - //if err != nil { - // log.Fatalf("error decoding password: %v", err) - //} - // - //err = w.VivaLogin(username, string(password)) - //if err != nil { - // log.Fatalf("error login in: %v", err) - //} + username := os.Getenv("VIVAPLUS_USER") + //password := os.Getenv("VIVAPLUS_PASS") + password, err := base64.StdEncoding.DecodeString(os.Getenv("VIVAPLUS_PASS")) + if err != nil { + log.Fatalf("error decoding password: %v", err) + } - w.VivaFindAllVideos() + err = w.VivaLogin(username, string(password)) + if err != nil { + log.Fatalf("error login in: %v", err) + } + + w.VivaDiscoverAllVideos(db) + //w.VivaAddMetadata(db) } diff --git a/migrations/1_init.sql b/migrations/1_init.sql index 3e2b345..72a99ff 100644 --- a/migrations/1_init.sql +++ b/migrations/1_init.sql @@ -1,6 +1,8 @@ CREATE TABLE IF NOT EXISTS videos ( id INTEGER PRIMARY KEY AUTOINCREMENT, - title TEXT NOT NULL, + title TEXT, url TEXT NOT NULL UNIQUE, - uploaded_at DATETIME DEFAULT CURRENT_TIMESTAMP + inserted_on DATETIME DEFAULT CURRENT_TIMESTAMP ); + +INSERT INTO videos (url) values ('/supporters/videos/81886') diff --git a/vivaweb.go b/vivaweb.go index 6cb79c4..16fb5c5 100644 --- a/vivaweb.go +++ b/vivaweb.go @@ -1,6 +1,7 @@ package main import ( + "database/sql" "fmt" "github.com/playwright-community/playwright-go" "log" @@ -12,6 +13,7 @@ const SIGN_IN_URL = BASE_URL + "supporters/sign_in" type WebClient struct { pw *playwright.Playwright browser playwright.Browser + page playwright.Page //browser *browser.Browser //csrfToken string //cookies *cookiejar.Jar @@ -20,63 +22,122 @@ type WebClient struct { func NewWebClient(options *playwright.RunOptions) *WebClient { pw, err := playwright.Run(options) if err != nil { - log.Fatalf("error running playwright: %w", pw) + log.Fatalf("error running playwright: %w", err) } browser, err := pw.Firefox.Launch() if err != nil { - log.Fatalf("error running firefox: %w", pw) + log.Fatalf("error running firefox: %w", err) + } + page, err := browser.NewPage() + if err != nil { + log.Fatalf("error creating page: %w", err) } return &WebClient{ pw: pw, browser: browser, + page: page, } } func (w *WebClient) VivaLogin(username, password string) error { - page, err := w.browser.NewPage() - if err != nil { - return fmt.Errorf("error creating page: %w", err) - } - - _, err = page.Goto(SIGN_IN_URL) + _, err := w.page.Goto(SIGN_IN_URL) if err != nil { return fmt.Errorf("error navigating to sign-in page: %w", err) } - err = page.GetByTestId("SupporterLogin.EmailInput").Fill(username) + err = w.page.GetByTestId("SupporterLogin.EmailInput").Fill(username) if err != nil { return fmt.Errorf("error filling in email: %w", err) } - err = page.GetByTestId("SupporterLogin.PasswordInput").Fill(password) + err = w.page.GetByTestId("SupporterLogin.PasswordInput").Fill(password) if err != nil { return fmt.Errorf("error filling in email: %w", err) } - err = page.GetByTestId("SupporterLogin.SubmitButton").Click() + log.Printf("Clicking login...") + err = w.page.GetByTestId("SupporterLogin.SubmitButton").Click() + if err != nil { + return fmt.Errorf("error clicking login: %w", err) + } + + err = w.page.WaitForURL("https://vivaplus.tv/supporters") + if err != nil { + return fmt.Errorf("error waiting for url: %w", err) + } + log.Printf("Logged in!") + return nil } -func (w *WebClient) VivaFindAllVideos() error { +func (w *WebClient) VivaDiscoverAllVideos(db *sql.DB) error { log.Printf("Loading list of all videos...") - page, err := w.browser.NewPage() - if err != nil { - return fmt.Errorf("error creating page: %w", err) - } - _, err = page.Goto("https://vivaplus.tv/supporters/videos/all?order=asc") + _, err := w.page.Goto("https://vivaplus.tv/supporters/videos/all?order=desc") if err != nil { return fmt.Errorf("error opening page: %w", err) } - count, err := page.GetByTestId("VideoCatalog.Video").Count() + //page.Keyboard().Press("End") + //time.Sleep(3 * time.Second) + + count, err := w.page.GetByTestId("VideoCatalog.Video").Count() //count, err := el.Count() println(count, err) - //attr, err := el.GetAttribute("busy") - //println(attr, err) - //err = el.ScrollIntoViewIfNeeded() - //println(err) - //attr, err = el.GetAttribute("busy") - //println(attr, err) + println("Looping over videos...") + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("error starting transaction: %w", err) + } + defer tx.Rollback() + + previousUrls := make(map[string]struct{}) + + for { + locators, err := w.page.GetByTestId("VideoCatalog.Video").All() + if err != nil { + return err + } + + for _, l := range locators { + href, err := l.GetAttribute("href") + if err != nil { + return err + } + + if _, exists := previousUrls[href]; exists { + continue // The item was already scanned. + } + previousUrls[href] = struct{}{} + + result := tx.QueryRow("select count(1) from videos where url = :url", href) + var count int + err = result.Scan(&count) + if err != nil { + return fmt.Errorf("error fetching data from db: %w", err) + } + if count == 1 { + log.Printf("Video already exists: %s", href) + goto finish + } + + log.Printf("Adding video %s", href) + _, err = tx.Exec("insert into videos(url) values (:url)", href) + if err != nil { + return fmt.Errorf("error inserting into db: %w", err) + } + } + log.Printf("Scrolling...") + err = w.page.Keyboard().Press("End") + if err != nil { + return fmt.Errorf("error scrolling to end of page: %w", err) + } + //w.page.tim + } +finish: + err = tx.Commit() + if err != nil { + return fmt.Errorf("error committing: %w", err) + } return nil }