144 lines
3.4 KiB
Go
144 lines
3.4 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"github.com/playwright-community/playwright-go"
|
|
"log"
|
|
)
|
|
|
|
const BASE_URL = "https://vivaplus.tv/"
|
|
const SIGN_IN_URL = BASE_URL + "supporters/sign_in"
|
|
|
|
type WebClient struct {
|
|
pw *playwright.Playwright
|
|
browser playwright.Browser
|
|
page playwright.Page
|
|
//browser *browser.Browser
|
|
//csrfToken string
|
|
//cookies *cookiejar.Jar
|
|
}
|
|
|
|
func NewWebClient(options *playwright.RunOptions) *WebClient {
|
|
pw, err := playwright.Run(options)
|
|
if err != nil {
|
|
log.Fatalf("error running playwright: %w", err)
|
|
}
|
|
browser, err := pw.Firefox.Launch()
|
|
if err != nil {
|
|
log.Fatalf("error running firefox: %w", err)
|
|
}
|
|
page, err := browser.NewPage()
|
|
if err != nil {
|
|
log.Fatalf("error creating page: %w", err)
|
|
}
|
|
return &WebClient{
|
|
pw: pw,
|
|
browser: browser,
|
|
page: page,
|
|
}
|
|
}
|
|
|
|
func (w *WebClient) VivaLogin(username, password string) error {
|
|
_, err := w.page.Goto(SIGN_IN_URL)
|
|
if err != nil {
|
|
return fmt.Errorf("error navigating to sign-in page: %w", err)
|
|
}
|
|
|
|
err = w.page.GetByTestId("SupporterLogin.EmailInput").Fill(username)
|
|
if err != nil {
|
|
return fmt.Errorf("error filling in email: %w", err)
|
|
}
|
|
err = w.page.GetByTestId("SupporterLogin.PasswordInput").Fill(password)
|
|
if err != nil {
|
|
return fmt.Errorf("error filling in email: %w", err)
|
|
}
|
|
|
|
log.Printf("Clicking login...")
|
|
err = w.page.GetByTestId("SupporterLogin.SubmitButton").Click()
|
|
if err != nil {
|
|
return fmt.Errorf("error clicking login: %w", err)
|
|
}
|
|
|
|
err = w.page.WaitForURL("https://vivaplus.tv/supporters")
|
|
if err != nil {
|
|
return fmt.Errorf("error waiting for url: %w", err)
|
|
}
|
|
log.Printf("Logged in!")
|
|
|
|
return nil
|
|
}
|
|
|
|
func (w *WebClient) VivaDiscoverAllVideos(db *sql.DB) error {
|
|
log.Printf("Loading list of all videos...")
|
|
|
|
_, err := w.page.Goto("https://vivaplus.tv/supporters/videos/all?order=desc")
|
|
if err != nil {
|
|
return fmt.Errorf("error opening page: %w", err)
|
|
}
|
|
|
|
//page.Keyboard().Press("End")
|
|
//time.Sleep(3 * time.Second)
|
|
|
|
count, err := w.page.GetByTestId("VideoCatalog.Video").Count()
|
|
//count, err := el.Count()
|
|
println(count, err)
|
|
|
|
println("Looping over videos...")
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return fmt.Errorf("error starting transaction: %w", err)
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
previousUrls := make(map[string]struct{})
|
|
|
|
for {
|
|
locators, err := w.page.GetByTestId("VideoCatalog.Video").All()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, l := range locators {
|
|
href, err := l.GetAttribute("href")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, exists := previousUrls[href]; exists {
|
|
continue // The item was already scanned.
|
|
}
|
|
previousUrls[href] = struct{}{}
|
|
|
|
result := tx.QueryRow("select count(1) from videos where url = :url", href)
|
|
var count int
|
|
err = result.Scan(&count)
|
|
if err != nil {
|
|
return fmt.Errorf("error fetching data from db: %w", err)
|
|
}
|
|
if count == 1 {
|
|
log.Printf("Video already exists: %s", href)
|
|
goto finish
|
|
}
|
|
|
|
log.Printf("Adding video %s", href)
|
|
_, err = tx.Exec("insert into videos(url) values (:url)", href)
|
|
if err != nil {
|
|
return fmt.Errorf("error inserting into db: %w", err)
|
|
}
|
|
}
|
|
log.Printf("Scrolling...")
|
|
err = w.page.Keyboard().Press("End")
|
|
if err != nil {
|
|
return fmt.Errorf("error scrolling to end of page: %w", err)
|
|
}
|
|
//w.page.tim
|
|
}
|
|
finish:
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
return fmt.Errorf("error committing: %w", err)
|
|
}
|
|
return nil
|
|
}
|