diff --git a/downloader.go b/downloader.go new file mode 100644 index 0000000..939c36e --- /dev/null +++ b/downloader.go @@ -0,0 +1,152 @@ +package main + +import ( + "database/sql" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "time" +) + +func DownloadAllVideos(db *sql.DB) error { + // Ensure no partial videos exist + tempDir := os.Getenv("TMP_DOWNLOAD") + if tempDir == "" { + tempDir = "./temp" // Replace with your desired default path + } + err := os.RemoveAll(tempDir) + if err != nil { + return fmt.Errorf("error cleaning temporary directory: %w", err) + } + err = os.MkdirAll(tempDir, 0750) + if err != nil { + return fmt.Errorf("error creating temporary directory: %w", err) + } + + // Find the target directory + downloadDir := os.Getenv("DESTINATION") + if downloadDir == "" { + downloadDir = "./downloads" + } + err = os.MkdirAll(downloadDir, 0750) + if err != nil { + return fmt.Errorf("error creating destination directory: %w", err) + } + + log.Printf("Starting downloads...") + for { + // Fetch the next record from the database + row := db.QueryRow("select id, url, `cast`, title, description, upload_date, thumbnail from videos where state = 'pending' limit 1") + var id int + var href, cast, title, description, uploadDateStr, thumbnailUrl string + err = row.Scan(&id, &href, &cast, &title, &description, &uploadDateStr, &thumbnailUrl) + if errors.Is(err, sql.ErrNoRows) { + log.Printf("No videos found for downloading") + return nil + } + if err != nil { + return fmt.Errorf("error fetching record: %w", err) + } + uploadDate, err := time.Parse(time.DateOnly, uploadDateStr) + if err != nil { + return fmt.Errorf("error parsing upload date '%s': %w", uploadDateStr, err) + } + + // Download the actual video + baseFileName := fmt.Sprintf("/s%de1%02d%02d", uploadDate.Year(), uploadDate.Month(), uploadDate.Day()) + log.Printf("Downloading %s", href) + cmd := exec.Command("yt-dlp", cast, "-o", filepath.Join(tempDir, baseFileName+".%(ext)s")) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + err = cmd.Run() + if err != nil { + return fmt.Errorf("error downloading video: %w", err) + } + + // Move it into the destination directory + destinationDir := filepath.Join(downloadDir, fmt.Sprintf("Season %d", uploadDate.Year())) + err = os.MkdirAll(destinationDir, 0750) + if err != nil { + return fmt.Errorf("error creating target directory: %w", err) + } + files, err := os.ReadDir(tempDir) + if err != nil { + return fmt.Errorf("error retrieving downloaded files: %w", err) + } + + for _, file := range files { + srcPath := filepath.Join(tempDir, file.Name()) + destPath := filepath.Join(destinationDir, file.Name()) + fmt.Printf("Moving %s to %s", srcPath, destPath) + err = os.Rename(srcPath, destPath) + if err != nil { + return fmt.Errorf("error moving file: %w", err) + } + } + + // Write XML sidecar + nfo := NFO{} + nfo.Aired = uploadDateStr + nfo.Plot = description + nfo.Title = title + nfo.Year = uploadDate.Year() + nfoData, err := xml.MarshalIndent(nfo, "", " ") + if err != nil { + return fmt.Errorf("error marshalling NFO data: %w", err) + } + nfoString := xml.Header + string(nfoData) + nfoFile, err := os.Create(filepath.Join(destinationDir, baseFileName+".nfo")) + defer nfoFile.Close() + _, err = nfoFile.WriteString(nfoString) + if err != nil { + return fmt.Errorf("error writing NFO file: %w", err) + } + + // Write thumbnail + thumbnailExt := filepath.Ext(thumbnailUrl) + thumbnailFile, err := os.Create(filepath.Join(destinationDir, baseFileName+"-thumb"+thumbnailExt)) + if err != nil { + return fmt.Errorf("error creating thumbnail: %w", err) + } + defer thumbnailFile.Close() + thumbnailResp, err := http.Get(thumbnailUrl) + if err != nil { + return fmt.Errorf("error fetching thumbnail: %w", err) + } + defer thumbnailResp.Body.Close() + _, err = io.Copy(thumbnailFile, thumbnailResp.Body) + if err != nil { + return fmt.Errorf("error writing thumbnail: %w", err) + } + + // Set the database state to done + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("error starting transaction: %w", err) + } + defer tx.Rollback() + result, err := tx.Exec("update videos set state = 'done' where id = ?", id) + if err != nil { + return fmt.Errorf("error updating database: %w", err) + } + count, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("error getting number of rows affected: %d", err) + } + if count != 1 { + return fmt.Errorf("unexpected number of rows changed (expected exactly 1): %d", count) + } + err = tx.Commit() + if err != nil { + return fmt.Errorf("error commiting transaction: %w", err) + } + + fmt.Printf("Finished downloading file") + } +} diff --git a/main.go b/main.go index 5806ae4..8b4ddbf 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,8 @@ import ( "github.com/playwright-community/playwright-go" "log" "os" + "strconv" + "time" ) func main() { @@ -19,27 +21,44 @@ func main() { db := openDatabase() defer db.Close() - w := NewWebClient(options) + sleepTimeStr := os.Getenv("VIVAPLUS_SLEEPTIME") + sleepTime := 15 + if sleepTimeStr != "" { + sleepTime, err = strconv.Atoi(sleepTimeStr) + if err != nil { + log.Fatalf("error parsing sleep time: %v", err) + } + } + w := NewWebClient(options) username := os.Getenv("VIVAPLUS_USER") - //password := os.Getenv("VIVAPLUS_PASS") password, err := base64.StdEncoding.DecodeString(os.Getenv("VIVAPLUS_PASS")) if err != nil { log.Fatalf("error decoding password: %v", err) } - err = w.Login(username, string(password)) - if err != nil { - log.Fatalf("error login in: %v", err) - } + for { + err = w.Login(username, string(password)) + if err != nil { + log.Fatalf("error login in: %v", err) + } - err = w.DiscoverAllVideos(db) - if err != nil { - panic(err) - } + err = w.DiscoverAllVideos(db) + if err != nil { + panic(err) + } - err = w.FetchVideoMetadata(db) - if err != nil { - panic(err) + err = w.FetchVideoMetadata(db) + if err != nil { + panic(err) + } + + err = DownloadAllVideos(db) + if err != nil { + panic(err) + } + + log.Printf("Sleeping %d minutes until next run", sleepTime) + time.Sleep(time.Duration(sleepTime) * time.Minute) } } diff --git a/migrations/1_init.sql b/migrations/1_init.sql index 72a99ff..443b691 100644 --- a/migrations/1_init.sql +++ b/migrations/1_init.sql @@ -5,4 +5,4 @@ CREATE TABLE IF NOT EXISTS videos ( inserted_on DATETIME DEFAULT CURRENT_TIMESTAMP ); -INSERT INTO videos (url) values ('/supporters/videos/81886') +INSERT INTO videos (url) values ('/supporters/videos/81886'); diff --git a/migrations/2_add_columns.sql b/migrations/2_add_columns.sql deleted file mode 100644 index 74c625d..0000000 --- a/migrations/2_add_columns.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE videos ADD COLUMN upload_date; -ALTER TABLE videos ADD COLUMN cast; -ALTER TABLE videos ADD COLUMN description; diff --git a/migrations/2_add_metadata_columns.sql b/migrations/2_add_metadata_columns.sql new file mode 100644 index 0000000..60d062d --- /dev/null +++ b/migrations/2_add_metadata_columns.sql @@ -0,0 +1,3 @@ +ALTER TABLE videos ADD COLUMN upload_date TEXT; +ALTER TABLE videos ADD COLUMN cast TEXT; +ALTER TABLE videos ADD COLUMN description TEXT; diff --git a/migrations/3_add_state_thumbnail_columns.sql b/migrations/3_add_state_thumbnail_columns.sql new file mode 100644 index 0000000..2458783 --- /dev/null +++ b/migrations/3_add_state_thumbnail_columns.sql @@ -0,0 +1,4 @@ +ALTER TABLE videos ADD COLUMN state TEXT NOT NULL DEFAULT 'pending'; +ALTER TABLE videos ADD COLUMN thumbnail TEXT; + +UPDATE videos SET thumbnail='https://imgproxy.fourthwall.com/ymbpbC7b9fO7tWlsg3e8sEdv8b3TuqXDGjr5FNM-uIg/rs:fill:607:342:0/sm:1/enc/N2U3ODcwN2I3Y2Iw/ZTZlYgoXdyrvsfaC/wje2R4f5TYcCwB4Y/ilNrHvoVDCjpBH8z/xnY0ho0FU6HlR9bX/Pk1pfFXOIFCqKnm6/2SKEU7hk5GgZTVXP/vDclApaRvgHzNXsF/pT8enEz076L7hSD-/0rENKD99989AC33R/5ZkKjg1ZZSc.webp' WHERE url = '/supporters/videos/81886'; diff --git a/nfo.go b/nfo.go new file mode 100644 index 0000000..9ccdaac --- /dev/null +++ b/nfo.go @@ -0,0 +1,9 @@ +package main + +type NFO struct { + XMLName struct{} `xml:"episodedetails"` + Title string `xml:"title"` + Year int `xml:"year"` + Aired string `xml:"aired"` + Plot string `xml:"plot"` +} diff --git a/vivaweb.go b/vivaweb.go index c9b79d4..2718cd7 100644 --- a/vivaweb.go +++ b/vivaweb.go @@ -151,7 +151,15 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error { return fmt.Errorf("url has bad format: %s", href) } - // Insert it into the database + // Get thumbnail + thumbnailEl := l.Locator(".video__image:first-child") + thumbnail, err := thumbnailEl.GetAttribute("src") + if err != nil { + return fmt.Errorf("error retrieving thumbnail: %w", err) + } + + // Ensure the record does not already exist. If it does, we've fetched + // all new videos result := tx.QueryRow("select count(1) from videos where url = :url", href) var count int err = result.Scan(&count) @@ -163,8 +171,9 @@ func (w *WebClient) DiscoverAllVideos(db *sql.DB) error { goto finish } + // Insert it into the database log.Printf("Adding video %s", href) - _, err = tx.Exec("insert into videos(url) values (:url)", href) + _, err = tx.Exec("insert into videos(url, thumbnail) values (?, ?)", href, thumbnail) if err != nil { return fmt.Errorf("error inserting into db: %w", err) } @@ -205,7 +214,7 @@ func (w *WebClient) FetchVideoMetadata(db *sql.DB) error { if err != nil { return fmt.Errorf("error fetching record: %w", err) } - log.Printf("Fetching data from %s", href) + log.Printf("Fetching data for %s", href) // Fetch the video metadata from the web page _, err = w.page.Goto(BASE_URL + href) @@ -254,7 +263,7 @@ func (w *WebClient) FetchVideoMetadata(db *sql.DB) error { return fmt.Errorf("error starting transaction: %w", err) } defer tx.Rollback() - result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ? where id = ?", title, description, castSource, uploadDate, id) + result, err := tx.Exec("update videos set title = ?, description = ?, cast = ?, upload_date = ? where id = ?", title, description, castSource, uploadDate.Format(time.DateOnly), id) if err != nil { return fmt.Errorf("error updating database: %w", err) }