package main import ( "database/sql" "fmt" "github.com/playwright-community/playwright-go" "log" ) const BASE_URL = "https://vivaplus.tv/" const SIGN_IN_URL = BASE_URL + "supporters/sign_in" type WebClient struct { pw *playwright.Playwright browser playwright.Browser page playwright.Page //browser *browser.Browser //csrfToken string //cookies *cookiejar.Jar } func NewWebClient(options *playwright.RunOptions) *WebClient { pw, err := playwright.Run(options) if err != nil { log.Fatalf("error running playwright: %w", err) } browser, err := pw.Firefox.Launch() if err != nil { log.Fatalf("error running firefox: %w", err) } page, err := browser.NewPage() if err != nil { log.Fatalf("error creating page: %w", err) } return &WebClient{ pw: pw, browser: browser, page: page, } } func (w *WebClient) VivaLogin(username, password string) error { _, err := w.page.Goto(SIGN_IN_URL) if err != nil { return fmt.Errorf("error navigating to sign-in page: %w", err) } err = w.page.GetByTestId("SupporterLogin.EmailInput").Fill(username) if err != nil { return fmt.Errorf("error filling in email: %w", err) } err = w.page.GetByTestId("SupporterLogin.PasswordInput").Fill(password) if err != nil { return fmt.Errorf("error filling in email: %w", err) } log.Printf("Clicking login...") err = w.page.GetByTestId("SupporterLogin.SubmitButton").Click() if err != nil { return fmt.Errorf("error clicking login: %w", err) } err = w.page.WaitForURL("https://vivaplus.tv/supporters") if err != nil { return fmt.Errorf("error waiting for url: %w", err) } log.Printf("Logged in!") return nil } func (w *WebClient) VivaDiscoverAllVideos(db *sql.DB) error { log.Printf("Loading list of all videos...") _, err := w.page.Goto("https://vivaplus.tv/supporters/videos/all?order=desc") if err != nil { return fmt.Errorf("error opening page: %w", err) } //page.Keyboard().Press("End") //time.Sleep(3 * time.Second) count, err := w.page.GetByTestId("VideoCatalog.Video").Count() //count, err := el.Count() println(count, err) println("Looping over videos...") tx, err := db.Begin() if err != nil { return fmt.Errorf("error starting transaction: %w", err) } defer tx.Rollback() previousUrls := make(map[string]struct{}) for { locators, err := w.page.GetByTestId("VideoCatalog.Video").All() if err != nil { return err } for _, l := range locators { href, err := l.GetAttribute("href") if err != nil { return err } if _, exists := previousUrls[href]; exists { continue // The item was already scanned. } previousUrls[href] = struct{}{} result := tx.QueryRow("select count(1) from videos where url = :url", href) var count int err = result.Scan(&count) if err != nil { return fmt.Errorf("error fetching data from db: %w", err) } if count == 1 { log.Printf("Video already exists: %s", href) goto finish } log.Printf("Adding video %s", href) _, err = tx.Exec("insert into videos(url) values (:url)", href) if err != nil { return fmt.Errorf("error inserting into db: %w", err) } } log.Printf("Scrolling...") err = w.page.Keyboard().Press("End") if err != nil { return fmt.Errorf("error scrolling to end of page: %w", err) } //w.page.tim } finish: err = tx.Commit() if err != nil { return fmt.Errorf("error committing: %w", err) } return nil }