seeseemelk/vivaplusdl
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Add ability to fetch video links

Browse Source
This commit is contained in:
Sebastiaan de Schaetzen
2025-02-04 20:02:11 +01:00
parent 27c58d06e0
commit 4170686ec6
3 changed files with 104 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
main.go
View File

@@ -1,8 +1,10 @@
package main
import (
"encoding/base64"
"github.com/playwright-community/playwright-go"
"log"
"os"
)
func main() {
@@ -19,17 +21,18 @@ func main() {
w := NewWebClient(options)
//username := os.Getenv("VIVAPLUS_USER")
////password := os.Getenv("VIVAPLUS_PASS")
//password, err := base64.StdEncoding.DecodeString(os.Getenv("VIVAPLUS_PASS"))
//if err != nil {
// log.Fatalf("error decoding password: %v", err)
//}
//
//err = w.VivaLogin(username, string(password))
//if err != nil {
// log.Fatalf("error login in: %v", err)
//}
username := os.Getenv("VIVAPLUS_USER")
//password := os.Getenv("VIVAPLUS_PASS")
password, err := base64.StdEncoding.DecodeString(os.Getenv("VIVAPLUS_PASS"))
if err != nil {
log.Fatalf("error decoding password: %v", err)
}
w.VivaFindAllVideos()
err = w.VivaLogin(username, string(password))
if err != nil {
log.Fatalf("error login in: %v", err)
}
w.VivaDiscoverAllVideos(db)
//w.VivaAddMetadata(db)
}

6
migrations/1_init.sql
View File

@@ -1,6 +1,8 @@
CREATE TABLE IF NOT EXISTS videos (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
title TEXT,
url TEXT NOT NULL UNIQUE,
uploaded_at DATETIME DEFAULT CURRENT_TIMESTAMP
inserted_on DATETIME DEFAULT CURRENT_TIMESTAMP
);
INSERT INTO videos (url) values ('/supporters/videos/81886')

109
vivaweb.go
View File

@@ -1,6 +1,7 @@
package main
import (
"database/sql"
"fmt"
"github.com/playwright-community/playwright-go"
"log"
@@ -12,6 +13,7 @@ const SIGN_IN_URL = BASE_URL + "supporters/sign_in"
type WebClient struct {
pw *playwright.Playwright
browser playwright.Browser
page playwright.Page
//browser *browser.Browser
//csrfToken string
//cookies *cookiejar.Jar
@@ -20,63 +22,122 @@ type WebClient struct {
func NewWebClient(options *playwright.RunOptions) *WebClient {
pw, err := playwright.Run(options)
if err != nil {
log.Fatalf("error running playwright: %w", pw)
log.Fatalf("error running playwright: %w", err)
}
browser, err := pw.Firefox.Launch()
if err != nil {
log.Fatalf("error running firefox: %w", pw)
log.Fatalf("error running firefox: %w", err)
}
page, err := browser.NewPage()
if err != nil {
log.Fatalf("error creating page: %w", err)
}
return &WebClient{
pw: pw,
browser: browser,
page: page,
}
}
func (w *WebClient) VivaLogin(username, password string) error {
page, err := w.browser.NewPage()
if err != nil {
return fmt.Errorf("error creating page: %w", err)
}
_, err = page.Goto(SIGN_IN_URL)
_, err := w.page.Goto(SIGN_IN_URL)
if err != nil {
return fmt.Errorf("error navigating to sign-in page: %w", err)
}
err = page.GetByTestId("SupporterLogin.EmailInput").Fill(username)
err = w.page.GetByTestId("SupporterLogin.EmailInput").Fill(username)
if err != nil {
return fmt.Errorf("error filling in email: %w", err)
}
err = page.GetByTestId("SupporterLogin.PasswordInput").Fill(password)
err = w.page.GetByTestId("SupporterLogin.PasswordInput").Fill(password)
if err != nil {
return fmt.Errorf("error filling in email: %w", err)
}
err = page.GetByTestId("SupporterLogin.SubmitButton").Click()
log.Printf("Clicking login...")
err = w.page.GetByTestId("SupporterLogin.SubmitButton").Click()
if err != nil {
return fmt.Errorf("error clicking login: %w", err)
}
err = w.page.WaitForURL("https://vivaplus.tv/supporters")
if err != nil {
return fmt.Errorf("error waiting for url: %w", err)
}
log.Printf("Logged in!")
return nil
}
func (w *WebClient) VivaFindAllVideos() error {
func (w *WebClient) VivaDiscoverAllVideos(db *sql.DB) error {
log.Printf("Loading list of all videos...")
page, err := w.browser.NewPage()
if err != nil {
return fmt.Errorf("error creating page: %w", err)
}
_, err = page.Goto("https://vivaplus.tv/supporters/videos/all?order=asc")
_, err := w.page.Goto("https://vivaplus.tv/supporters/videos/all?order=desc")
if err != nil {
return fmt.Errorf("error opening page: %w", err)
}
count, err := page.GetByTestId("VideoCatalog.Video").Count()
//page.Keyboard().Press("End")
//time.Sleep(3 * time.Second)
count, err := w.page.GetByTestId("VideoCatalog.Video").Count()
//count, err := el.Count()
println(count, err)
//attr, err := el.GetAttribute("busy")
//println(attr, err)
//err = el.ScrollIntoViewIfNeeded()
//println(err)
//attr, err = el.GetAttribute("busy")
//println(attr, err)
println("Looping over videos...")
tx, err := db.Begin()
if err != nil {
return fmt.Errorf("error starting transaction: %w", err)
}
defer tx.Rollback()
previousUrls := make(map[string]struct{})
for {
locators, err := w.page.GetByTestId("VideoCatalog.Video").All()
if err != nil {
return err
}
for _, l := range locators {
href, err := l.GetAttribute("href")
if err != nil {
return err
}
if _, exists := previousUrls[href]; exists {
continue // The item was already scanned.
}
previousUrls[href] = struct{}{}
result := tx.QueryRow("select count(1) from videos where url = :url", href)
var count int
err = result.Scan(&count)
if err != nil {
return fmt.Errorf("error fetching data from db: %w", err)
}
if count == 1 {
log.Printf("Video already exists: %s", href)
goto finish
}
log.Printf("Adding video %s", href)
_, err = tx.Exec("insert into videos(url) values (:url)", href)
if err != nil {
return fmt.Errorf("error inserting into db: %w", err)
}
}
log.Printf("Scrolling...")
err = w.page.Keyboard().Press("End")
if err != nil {
return fmt.Errorf("error scrolling to end of page: %w", err)
}
//w.page.tim
}
finish:
err = tx.Commit()
if err != nil {
return fmt.Errorf("error committing: %w", err)
}
return nil
}