cwpearson/ytdlp-site
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support multiple video/audio transcodes

Browse Source
This commit is contained in:
Carl Pearson
2024-09-10 08:21:03 -06:00
parent 456b3d3e56
commit 663a6f5e88
9 changed files with 411 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.dockerignore
View File

@@ -1,4 +1,4 @@
downloads
data
config
go.sum
*.mp4

4
.gitignore vendored
View File

@@ -1,4 +1,4 @@
downloads
config
go.sum
*.mp4
*.mp4
data

4
README.md
View File

@@ -28,10 +28,10 @@ docker run --rm -it \
docker run --rm -it \
-p 3000:8080 \
--env YTDLP_SITE_DOWNLOAD_DIR=/downloads \
--env YTDLP_SITE_DATA_DIR=/data \
--env YTDLP_SITE_CONFIG_DIR=/config \
--env YTDLP_SITE_ADMIN_INITIAL_PASSWORD=abc123 \
-v $(realpath downloads):/downloads \
-v $(realpath data):/data \
server
```

6
config.go
View File

@@ -6,12 +6,12 @@ import (
"os"
)
func getDownloadDir() string {
value, exists := os.LookupEnv("YTDLP_SITE_DOWNLOAD_DIR")
func getDataDir() string {
value, exists := os.LookupEnv("YTDLP_SITE_DATA_DIR")
if exists {
return value
}
return "downloads"
return "data"
}
func getConfigDir() string {

409
handlers.go
View File

@@ -2,6 +2,7 @@ package main
import (
"bytes"
"errors"
"fmt"
"net/http"
"os"
@@ -11,6 +12,7 @@ import (
"strings"
"time"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
"golang.org/x/crypto/bcrypt"
)
@@ -89,22 +91,20 @@ func downloadPostHandler(c echo.Context) error {
url := c.FormValue("url")
userID := c.Get("user_id").(uint)
video := Video{URL: url, UserID: userID, Status: "pending"}
db.Create(&video)
go startDownload(video.ID, url)
original := Original{URL: url, UserID: userID, Status: "pending"}
db.Create(&original)
go startDownload(original.ID, url)
return c.Redirect(http.StatusSeeOther, "/videos")
}
type Meta struct {
title string
ext string
}
func getMeta(url string) (Meta, error) {
cmd := exec.Command("yt-dlp",
"--simulate", "--print", "%(title)s",
url)
cmd := exec.Command("yt-dlp", "--simulate", "--print", "%(title)s.%(ext)s", url)
var stdout bytes.Buffer
cmd.Stdout = &stdout
@@ -113,8 +113,17 @@ func getMeta(url string) (Meta, error) {
fmt.Println("getTitle error:", err, stdout.String())
return Meta{}, err
} else {
isDot := func(r rune) bool {
return r == '.'
}
fields := strings.FieldsFunc(strings.TrimSpace(stdout.String()), isDot)
if len(fields) < 2 {
return Meta{}, errors.New("couldn't parse ytdlp output")
}
return Meta{
title: strings.TrimSpace(stdout.String()),
title: strings.Join(fields[:len(fields)-1], "."),
ext: fields[len(fields)-1],
}, nil
}
}
@@ -139,6 +148,87 @@ func getLength(path string) (float64, error) {
return result, nil
}
func getVideoWidth(path string) (uint, error) {
cmd := exec.Command("ffprobe", "-v", "error", "-select_streams",
"v:0", "-count_packets", "-show_entries",
"stream=width", "-of", "csv=p=0", path)
var stdout bytes.Buffer
cmd.Stdout = &stdout
err := cmd.Run()
if err != nil {
fmt.Println("getVideoWidth cmd error:", err)
return 0, err
}
result, err := strconv.ParseUint(strings.TrimSpace(stdout.String()), 10, 32)
if err != nil {
fmt.Println("getVideoWidth parse error:", err, stdout.String())
}
return uint(result), nil
}
func getVideoHeight(path string) (uint, error) {
cmd := exec.Command("ffprobe", "-v", "error", "-select_streams",
"v:0", "-count_packets", "-show_entries",
"stream=height", "-of", "csv=p=0", path)
var stdout bytes.Buffer
cmd.Stdout = &stdout
err := cmd.Run()
if err != nil {
fmt.Println("getVideoHeight cmd error:", err)
return 0, err
}
result, err := strconv.ParseUint(strings.TrimSpace(stdout.String()), 10, 32)
if err != nil {
fmt.Println("getVideoHeight parse error:", err, stdout.String())
}
return uint(result), nil
}
func getVideoFPS(path string) (float64, error) {
ffprobe := "ffprobe"
args := []string{"-v", "error", "-select_streams",
"v:0", "-count_packets", "-show_entries",
"stream=r_frame_rate", "-of", "csv=p=0", path}
fmt.Println(ffprobe, strings.Join(args, " "))
cmd := exec.Command(ffprobe, args...)
var stdout bytes.Buffer
cmd.Stdout = &stdout
err := cmd.Run()
if err != nil {
fmt.Println("getVideoFPS cmd error:", err)
return -1, err
}
// TODO: this produces a string like "num/denom", do the division
parts := strings.Split(strings.TrimSpace(stdout.String()), "/")
if len(parts) != 2 {
fmt.Println("getVideoFPS split error:", err, stdout.String())
}
num, err := strconv.ParseFloat(parts[0], 64)
if err != nil {
fmt.Println("getVideoFPS numerator parse error:", err, stdout.String())
}
denom, err := strconv.ParseFloat(parts[1], 64)
if err != nil {
fmt.Println("getVideoFPS denominator parse error:", err, stdout.String())
}
if denom == 0 {
fmt.Println("getVideoFPS denominator is zero error:", stdout.String())
}
return num / denom, nil
}
func humanLength(s float64) string {
ss := int64(s)
mm, ss := ss/60, ss%60
@@ -172,104 +262,258 @@ func humanSize(bytes int64) string {
return fmt.Sprintf("%d bytes", bytes)
}
func startDownload(videoID uint, videoURL string) {
db.Model(&Video{}).Where("id = ?", videoID).Update("status", "metadata")
type VideoMeta struct {
width uint
height uint
fps float64
}
meta, err := getMeta(videoURL)
func getVideoMeta(path string) (VideoMeta, error) {
w, err := getVideoWidth(path)
if err != nil {
db.Model(&Video{}).Where("id = ?", videoID).Update("status", "failed")
return
return VideoMeta{}, err
}
fmt.Println("set video title:", meta.title)
db.Model(&Video{}).Where("id = ?", videoID).Update("title", meta.title)
db.Model(&Video{}).Where("id = ?", videoID).Update("status", "downloading")
videoFilename := fmt.Sprintf("%d-%s.mp4", videoID, meta.title)
videoFilepath := filepath.Join(getDownloadDir(), "video", videoFilename)
cmd := exec.Command("yt-dlp",
"-f", "bestvideo[height<=720]+bestaudio/best[height<=720]",
"--recode-video", "mp4",
"-o", videoFilepath,
videoURL)
err = cmd.Run()
h, err := getVideoHeight(path)
if err != nil {
db.Model(&Video{}).Where("id = ?", videoID).Update("status", "failed")
return
return VideoMeta{}, err
}
fps, err := getVideoFPS(path)
if err != nil {
return VideoMeta{}, err
}
return VideoMeta{
width: w,
height: h,
fps: fps,
}, nil
}
db.Model(&Video{}).Where("id = ?", videoID).Update("status", "audio")
audioFilename := fmt.Sprintf("%d-%s.mp3", videoID, meta.title)
audioFilepath := filepath.Join(getDownloadDir(), "audio", audioFilename)
func videoToAudio(audioID uint, bitrate uint, videoFilepath string) {
audioFilename := uuid.Must(uuid.NewV7()).String()
audioFilename = fmt.Sprintf("%s.mp3", audioFilename)
audioFilepath := filepath.Join(getDataDir(), audioFilename)
audioDir := filepath.Dir(audioFilepath)
fmt.Println("Create", audioDir)
err = os.MkdirAll(audioDir, 0700)
err := os.MkdirAll(audioDir, 0700)
if err != nil {
fmt.Println("Error: couldn't create", audioDir)
db.Model(&Video{}).Where("id = ?", videoID).Update("status", "failed")
db.Model(&Audio{}).Where("id = ?", audioID).Update("status", "failed")
return
}
ffmpeg := "ffmpeg"
ffmpegArgs := []string{"-i", videoFilepath, "-vn", "-acodec",
"mp3", "-b:a", "160k", audioFilepath}
fmt.Println(ffmpeg, ffmpegArgs)
cmd = exec.Command(ffmpeg, ffmpegArgs...)
"mp3", "-b:a",
fmt.Sprintf("%dk", bitrate),
audioFilepath}
fmt.Println(ffmpeg, strings.Join(ffmpegArgs, " "))
cmd := exec.Command(ffmpeg, ffmpegArgs...)
err = cmd.Run()
if err != nil {
fmt.Println("Error: convert to audio file", videoFilepath, "->", audioFilepath)
db.Model(&Video{}).Where("id = ?", videoID).Update("status", "failed")
return
}
// FIXME: ensure expected files exist
db.Model(&Video{}).Where("id = ?", videoID).Updates(map[string]interface{}{
"video_filename": videoFilename,
"audio_filename": audioFilename,
"status": "completed",
})
length, err := getLength(videoFilepath)
fileSize, err := getSize(audioFilepath)
if err == nil {
db.Model(&Video{}).Where("id = ?", videoID).Update("length", humanLength(length))
db.Model(&Audio{}).Where("id = ?", audioID).Update("size", humanSize(fileSize))
}
db.Model(&Audio{}).Where("id = ?", audioID).Update("filename", audioFilename)
db.Model(&Audio{}).Where("id = ?", audioID).Update("status", "completed")
}
func videoToVideo(videoID uint, height uint, videoFilepath string) {
dstFilename := uuid.Must(uuid.NewV7()).String()
dstFilename = fmt.Sprintf("%s.mp4", dstFilename)
dstFilepath := filepath.Join(getDataDir(), dstFilename)
dstDir := filepath.Dir(dstFilepath)
fmt.Println("Create", dstDir)
err := os.MkdirAll(dstDir, 0700)
if err != nil {
fmt.Println("Error: couldn't create", dstDir)
db.Model(&Video{}).Where("id = ?", videoID).Update("status", "failed")
return
}
var audioBitrate uint = 160
if height <= 144 {
audioBitrate = 64
} else if height <= 240 {
audioBitrate = 96
} else if height < 720 {
audioBitrate = 128
}
ffmpeg := "ffmpeg"
ffmpegArgs := []string{"-i", videoFilepath,
"-vf", fmt.Sprintf("scale=-2:%d", height), "-c:v", "libx264",
"-crf", "23", "-preset", "veryfast", "-c:a", "aac", "-b:a", fmt.Sprintf("%dk", audioBitrate),
dstFilepath}
fmt.Println(ffmpeg, strings.Join(ffmpegArgs, " "))
cmd := exec.Command(ffmpeg, ffmpegArgs...)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout, cmd.Stderr = &stdout, &stderr
err = cmd.Run()
if err != nil {
fmt.Println("Error: convert to video file", videoFilepath, "->", dstFilepath, stdout.String(), stderr.String())
return
}
db.Model(&Video{}).Where("id = ?", videoID).Update("filename", dstFilename)
fileSize, err := getSize(dstFilepath)
if err == nil {
db.Model(&Video{}).Where("id = ?", videoID).Update("size", humanSize(fileSize))
}
meta, err := getVideoMeta(dstFilepath)
fmt.Println("meta for", dstFilepath, meta)
if err == nil {
db.Model(&Video{}).Where("id = ?", videoID).Update("width", meta.width)
db.Model(&Video{}).Where("id = ?", videoID).Update("height", meta.height)
db.Model(&Video{}).Where("id = ?", videoID).Update("fps", meta.fps)
}
db.Model(&Video{}).Where("id = ?", videoID).Update("status", "completed")
}
func startDownload(originalID uint, videoURL string) {
// metadata phase
db.Model(&Original{}).Where("id = ?", originalID).Update("status", "metadata")
origMeta, err := getMeta(videoURL)
if err != nil {
db.Model(&Original{}).Where("id = ?", originalID).Update("status", "failed")
return
}
fmt.Printf("original metadata %v\n", origMeta)
db.Model(&Original{}).Where("id = ?", originalID).Update("title", origMeta.title)
// download original
db.Model(&Original{}).Where("id = ?", originalID).Update("status", "downloading")
videoFilename := fmt.Sprintf("%d-%s.%s", originalID, origMeta.title, origMeta.ext)
videoFilepath := filepath.Join(getDataDir(), videoFilename)
cmd := exec.Command("yt-dlp",
"-f", "bestvideo+bestaudio/best",
"-o", videoFilepath,
videoURL)
err = cmd.Run()
if err != nil {
db.Model(&Original{}).Where("id = ?", originalID).Update("status", "failed")
return
}
db.Model(&Original{}).Where("id = ?", originalID).Update("status", "completed")
// create video entry for original
video := Video{
OriginalID: originalID,
Filename: videoFilename,
Source: "original",
Type: origMeta.ext,
Status: "completed",
}
if err := db.Create(&video).Error; err != nil {
fmt.Println(err)
}
videoMeta, err := getVideoMeta(videoFilepath)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(videoMeta)
db.Model(&Video{}).Where("id = ?", video.ID).Update("fps", videoMeta.fps)
db.Model(&Video{}).Where("id = ?", video.ID).Update("width", videoMeta.width)
db.Model(&Video{}).Where("id = ?", video.ID).Update("height", videoMeta.height)
}
videoSize, err := getSize(videoFilepath)
if err == nil {
db.Model(&Video{}).Where("id = ?", videoID).Update("video_size", humanSize(videoSize))
db.Model(&Video{}).Where("id = ?", video.ID).Update("size", humanSize(videoSize))
}
audioSize, err := getSize(audioFilepath)
if err == nil {
db.Model(&Video{}).Where("id = ?", videoID).Update("audio_size", humanSize(audioSize))
// create audio transcodes
for _, bitrate := range []uint{64, 96, 128, 160, 192} {
audio := Audio{
OriginalID: originalID,
Rate: fmt.Sprintf("%dk", bitrate),
Type: "mp3",
Status: "pending",
}
db.Create(&audio)
go videoToAudio(audio.ID, bitrate, videoFilepath)
}
// create video transcodes
for _, targetHeight := range []uint{144, 240, 360, 480, 720, 1080} {
if targetHeight <= videoMeta.height {
newVideo := Video{
OriginalID: originalID,
Type: "mp4",
Status: "pending",
Source: "transcode",
}
db.Create(&newVideo)
videoToVideo(newVideo.ID, targetHeight, videoFilepath)
}
}
}
func videosHandler(c echo.Context) error {
userID := c.Get("user_id").(uint)
var videos []Video
db.Where("user_id = ?", userID).Find(&videos)
return c.Render(http.StatusOK, "videos.html", map[string]interface{}{"videos": videos})
var origs []Original
db.Where("user_id = ?", userID).Find(&origs)
return c.Render(http.StatusOK, "videos.html", map[string]interface{}{"videos": origs})
}
type VideoTemplate struct {
Video
TempURL
}
type AudioTemplate struct {
Audio
TempURL
}
func videoHandler(c echo.Context) error {
id, _ := strconv.Atoi(c.Param("id"))
var video Video
if err := db.First(&video, id).Error; err != nil {
var orig Original
if err := db.First(&orig, id).Error; err != nil {
return c.Redirect(http.StatusSeeOther, "/videos")
}
downloadDir := getDownloadDir()
var videos []Video
db.Where("original_id = ?", id).Find(&videos)
tempURL, err := CreateTempURL(filepath.Join(downloadDir, "video", video.VideoFilename))
if err != nil {
return err
var audios []Audio
db.Where("original_id = ?", id).Find(&audios)
dataDir := getDataDir()
// create remporary URLs
var videoURLs []VideoTemplate
var audioURLs []AudioTemplate
for _, video := range videos {
tempURL, err := CreateTempURL(filepath.Join(dataDir, video.Filename))
if err != nil {
continue
}
videoURLs = append(videoURLs, VideoTemplate{video, tempURL})
}
for _, audio := range audios {
tempURL, err := CreateTempURL(filepath.Join(dataDir, audio.Filename))
if err != nil {
continue
}
audioURLs = append(audioURLs, AudioTemplate{audio, tempURL})
}
return c.Render(http.StatusOK, "video.html",
map[string]interface{}{
"video": video,
"downloadDir": downloadDir,
"tempURL": fmt.Sprintf("/temp/%s", tempURL.Token),
"original": orig,
"videos": videoURLs,
"audios": audioURLs,
"dataDir": dataDir,
})
}
@@ -289,35 +533,52 @@ func videoCancelHandler(c echo.Context) error {
func videoRestartHandler(c echo.Context) error {
id, _ := strconv.Atoi(c.Param("id"))
var video Video
if err := db.First(&video, id).Error; err != nil {
var orig Original
if err := db.First(&orig, id).Error; err != nil {
return c.Redirect(http.StatusSeeOther, "/videos")
}
video.Status = "pending"
db.Save(&video)
go startDownload(uint(id), video.URL)
orig.Status = "pending"
db.Save(&orig)
go startDownload(uint(id), orig.URL)
return c.Redirect(http.StatusSeeOther, "/videos")
}
func videoDeleteHandler(c echo.Context) error {
id, _ := strconv.Atoi(c.Param("id"))
var video Video
if err := db.First(&video, id).Error; err != nil {
var orig Original
if err := db.First(&orig, id).Error; err != nil {
return c.Redirect(http.StatusSeeOther, "/videos")
}
// Delete the file
if video.VideoFilename != "" {
os.Remove(filepath.Join(getDownloadDir(), "video", video.VideoFilename))
// TODO: delete all files
var videos []Video
db.Where("original_id = ?", id).Find(&videos)
for _, video := range videos {
path := filepath.Join(getDataDir(), video.Filename)
fmt.Println("remove", path)
err := os.Remove(path)
if err != nil {
fmt.Println("error removing", path, err)
}
}
if video.AudioFilename != "" {
os.Remove(filepath.Join(getDownloadDir(), "audio", video.AudioFilename))
db.Delete(videos)
var audios []Audio
db.Where("original_id = ?", id).Find(&audios)
for _, audio := range audios {
path := filepath.Join(getDataDir(), audio.Filename)
fmt.Println("remove", path)
err := os.Remove(path)
if err != nil {
fmt.Println("error removing", path, err)
}
}
db.Delete(audios)
// Delete from database
db.Delete(&video)
db.Delete(&orig)
return c.Redirect(http.StatusSeeOther, "/videos")
}

6
main.go
View File

@@ -51,7 +51,7 @@ func main() {
}
// Migrate the schema
db.AutoMigrate(&Video{}, &User{}, &TempURL{})
db.AutoMigrate(&Original{}, &Video{}, &Audio{}, &User{}, &TempURL{})
go PeriodicCleanup()
// create a user
@@ -96,9 +96,9 @@ func main() {
e.POST("/video/:id/restart", videoRestartHandler, authMiddleware)
e.POST("/video/:id/delete", videoDeleteHandler, authMiddleware)
staticGroup := e.Group("/downloads")
staticGroup := e.Group("/data")
staticGroup.Use(authMiddleware)
staticGroup.Static("/", getDownloadDir())
staticGroup.Static("/", getDataDir())
e.GET("/temp/:token", tempHandler)
store.Options = &sessions.Options{

41
models.go
View File

@@ -11,17 +11,40 @@ import (
"gorm.io/gorm"
)
type Original struct {
gorm.Model
UserID uint
URL string
Title string
Author string
Status string // "pending", "metadata", "downloading", "completed", "failed", "cancelled"
}
type Video struct {
gorm.Model
URL string
Title string
VideoFilename string
AudioFilename string
UserID uint
Length string
AudioSize string
VideoSize string
Status string // "pending", "downloading", "completed", "failed", "cancelled"
OriginalID uint // Original.ID
Width uint
Height uint
FPS float64
Length string
Size string
Type string
Codec string
Filename string
Status string // "pending", "completed"
Source string // "original", "ffmpeg"
}
type Audio struct {
gorm.Model
OriginalID uint // Original.ID
Rate string
Length string
Size string
Type string
Codec string
Filename string
Status string // "pending", "completed", "failed"
}
type User struct {

36
templates/video.html
View File

@@ -4,7 +4,7 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{{.video.Title}}</title>
<title>{{.original.Title}}</title>
<style>
body {
font-family: Arial, sans-serif;
@@ -42,13 +42,43 @@
</head>
<body>
<h1>{{.video.Title}}</h1>
<h1>{{.original.Title}}</h1>
{{range .videos}}
{{if eq .Status "completed"}}
<h2>{{.Source}} {{.Width}} x {{.Height}} @ {{.FPS}}</h2>
<div class="video-container">
<video controls playsinline preload="metadata">
<source src="{{.tempURL}}" type="video/mp4">
<source src="/temp/{{.Token}}" type="video/mp4">
Your browser does not support the video tag.
</video>
</div>
<div class="video-download">
<a href="/data/{{.Filename}}" download>Download ({{.Size}})</a>
</div>
{{else}}
<h2>Video {{.Source}} {{.Status}}</h2>
{{end}}
{{end}}
{{range .audios}}
{{if eq .Status "completed"}}
<h2>{{.Rate}}</h2>
<div class="audio-container">
<audio controls playsinline preload="metadata">
<source src="/temp/{{.Token}}">
Your browser does not support the audio tag.
</audio>
</div>
<div class="audio-download">
<a href="/data/{{.Filename}}" download>Download ({{.Size}})</a>
</div>
{{else}}
<h2>Audio {{.Status}}</h2>
{{end}}
{{end}}
</body>
</html>

4
templates/videos.html
View File

@@ -31,7 +31,6 @@
<tr>
<th>Title</th>
<th>URL</th>
<th>Length</th>
<th>Status</th>
<th>Actions</th>
</tr>
@@ -45,12 +44,9 @@
{{end}}
</td>
<td><a href="{{.URL}}">{{.URL}}</a></td>
<td>{{.Length}}</td>
<td>{{.Status}}</td>
<td>
{{if eq .Status "completed"}}
<a href="/downloads/video/{{.VideoFilename}}" download>Download Video ({{.VideoSize}})</a> |
<a href="/downloads/audio/{{.AudioFilename}}" download>Download Audio ({{.AudioSize}})</a>
{{else if eq .Status "failed"}}
<form action="/video/{{.ID}}/restart" method="post" style="display:inline;">
<button type="submit">Restart</button>