royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages1 Wiki Activity Actions

small refactor*

Browse source
This commit is contained in:
royalcat 2025-03-22 08:49:14 +04:00
parent b6b541e050
commit 24a4d30275
232 changed files with 2164 additions and 1906 deletions

9
server/pkg/ytdlp/client.go Normal file
View file

@ -0,0 +1,9 @@
package ytdlp
type Client struct {
binary string
}
func New() (*Client, error) {
return &Client{binary: "yt-dlp"}, nil
}

105
server/pkg/ytdlp/download.go Normal file
View file

@ -0,0 +1,105 @@
package ytdlp
import (
"context"
"encoding/json"
"io"
"os/exec"
"strings"
"github.com/royalcat/ctxprogress"
"golang.org/x/sync/errgroup"
)
type DownloadStatus string
const (
StatusDownloading DownloadStatus = "downloading"
StatusFinished DownloadStatus = "finished"
StatusErrored DownloadStatus = "error"
)
// Progress for the Running call
type DownloadProgress struct {
Status DownloadStatus `json:"status"`
Filename string `json:"filename"`
TmpFilename string `json:"tmpfilename"`
DownloadedBytes int64 `json:"downloaded_bytes"`
TotalBytes int64 `json:"total_bytes"`
TotalBytesEstimate float64 `json:"total_bytes_estimate"`
Elapsed float64 `json:"elapsed"`
ETA float64 `json:"eta"`
Speed float64 `json:"speed"`
FragmentIndex int64 `json:"fragment_index"`
FragmentCount int64 `json:"fragment_count"`
}
// Current implements ctxprogress.Progress.
func (d DownloadProgress) Progress() (int, int) {
if d.TotalBytes != -1 && d.TotalBytes != 0 && d.DownloadedBytes != -1 {
return int(d.DownloadedBytes), int(d.TotalBytes)
}
if d.TotalBytesEstimate != -1 && d.TotalBytesEstimate != 0 && d.DownloadedBytes != -1 {
return int(d.DownloadedBytes), int(d.TotalBytesEstimate)
}
return int(d.FragmentIndex), int(d.FragmentCount)
}
const rawProgressTemplate = `download:
%{
"status":"%(progress.status)s",
"eta":%(progress.eta|-1)s,
"speed":%(progress.speed|0)s,
"downloaded_bytes":%(progress.downloaded_bytes|-1)s,
"total_bytes": %(progress.total_bytes|-1)s,
"total_bytes_estimate": %(progress.total_bytes_estimate|-1)s,
"fragment_index":%(progress.fragment_index|-1)s,
"fragment_count":%(progress.fragment_count|-1)s
}`
var progressTemplate = strings.NewReplacer("\n", "", "\t", "", " ", "").Replace(rawProgressTemplate)
func (c *Client) Download(ctx context.Context, url string, w io.Writer) error {
args := []string{
"--progress", "--newline", "--progress-template", progressTemplate,
"-o", "-",
url,
}
group, ctx := errgroup.WithContext(ctx)
stderr, lines, err := lineReader(group)
if err != nil {
return err
}
cmd := exec.CommandContext(ctx, c.binary, args...)
cmd.Stdout = w
cmd.Stderr = stderr
group.Go(func() error {
err := cmd.Run()
stderr.Close()
if err != nil {
return err
}
return nil
})
for line := range lines {
if line, ok := strings.CutPrefix(line, "%"); ok {
p := DownloadProgress{}
err = json.Unmarshal([]byte(line), &p)
if err != nil {
//TODO: handle error
continue
}
ctxprogress.Set(ctx, p)
}
}
return group.Wait()
}

27
server/pkg/ytdlp/download_test.go Normal file
View file

@ -0,0 +1,27 @@
package ytdlp_test
import (
"context"
"fmt"
"io"
"testing"
"git.kmsign.ru/royalcat/tstor/server/pkg/ytdlp"
"github.com/royalcat/ctxprogress"
"github.com/stretchr/testify/require"
)
func TestDownload(t *testing.T) {
require := require.New(t)
ctx := context.Background()
c, err := ytdlp.New()
require.NoError(err)
ctx = ctxprogress.New(ctx)
ctxprogress.AddCallback(ctx, func(p ctxprogress.Progress) {
cur, total := p.Progress()
fmt.Printf("%d/%d\n", cur, total)
})
err = c.Download(ctx, "https://www.youtube.com/watch?v=dQw4w9WgXcQ", io.Discard)
require.NoError(err)
}

33
server/pkg/ytdlp/info.go Normal file
View file

@ -0,0 +1,33 @@
package ytdlp
import (
"bytes"
"context"
"encoding/json"
"os/exec"
)
func (c *Client) Info(ctx context.Context, url string) (*Info, error) {
args := []string{
"-q", "-J", url,
}
cmd := exec.CommandContext(ctx, c.binary, args...)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
return nil, err
}
var info Info
err = json.Unmarshal(stdout.Bytes(), &info)
if err != nil {
return nil, err
}
return &info, nil
}

389
server/pkg/ytdlp/model.go Normal file
View file

@ -0,0 +1,389 @@
package ytdlp
type Info struct {
ID string `json:"id"`
Title string `json:"title"`
Availability string `json:"availability"`
ChannelFollowerCount *int64 `json:"channel_follower_count"`
Description string `json:"description"`
Tags []string `json:"tags"`
Thumbnails []Thumbnail `json:"thumbnails"`
ModifiedDate *string `json:"modified_date,omitempty"`
ViewCount int64 `json:"view_count"`
PlaylistCount *int64 `json:"playlist_count,omitempty"`
Channel string `json:"channel"`
ChannelID string `json:"channel_id"`
UploaderID string `json:"uploader_id"`
Uploader string `json:"uploader"`
ChannelURL string `json:"channel_url"`
UploaderURL string `json:"uploader_url"`
Type string `json:"_type"`
Entries []Entry `json:"entries,omitempty"`
ExtractorKey string `json:"extractor_key"`
Extractor string `json:"extractor"`
WebpageURL string `json:"webpage_url"`
OriginalURL string `json:"original_url"`
WebpageURLBasename string `json:"webpage_url_basename"`
WebpageURLDomain string `json:"webpage_url_domain"`
ReleaseYear interface{} `json:"release_year"`
Epoch int64 `json:"epoch"`
FilesToMove *FilesToMove `json:"__files_to_move,omitempty"`
Version Version `json:"_version"`
Formats []Format `json:"formats,omitempty"`
Thumbnail *string `json:"thumbnail,omitempty"`
Duration *int64 `json:"duration,omitempty"`
AverageRating interface{} `json:"average_rating"`
AgeLimit *int64 `json:"age_limit,omitempty"`
Categories []string `json:"categories,omitempty"`
PlayableInEmbed *bool `json:"playable_in_embed,omitempty"`
LiveStatus *string `json:"live_status,omitempty"`
ReleaseTimestamp interface{} `json:"release_timestamp"`
FormatSortFields []string `json:"_format_sort_fields,omitempty"`
AutomaticCaptions map[string][]AutomaticCaption `json:"automatic_captions,omitempty"`
Subtitles *FilesToMove `json:"subtitles,omitempty"`
CommentCount *int64 `json:"comment_count,omitempty"`
Chapters interface{} `json:"chapters"`
Heatmap []Heatmap `json:"heatmap,omitempty"`
LikeCount *int64 `json:"like_count,omitempty"`
ChannelIsVerified *bool `json:"channel_is_verified,omitempty"`
UploadDate *string `json:"upload_date,omitempty"`
Timestamp *int64 `json:"timestamp,omitempty"`
Playlist interface{} `json:"playlist"`
PlaylistIndex interface{} `json:"playlist_index"`
DisplayID *string `json:"display_id,omitempty"`
Fulltitle *string `json:"fulltitle,omitempty"`
DurationString *string `json:"duration_string,omitempty"`
IsLive *bool `json:"is_live,omitempty"`
WasLive *bool `json:"was_live,omitempty"`
RequestedSubtitles interface{} `json:"requested_subtitles"`
HasDRM interface{} `json:"_has_drm"`
RequestedDownloads []RequestedDownload `json:"requested_downloads,omitempty"`
RequestedFormats []Format `json:"requested_formats,omitempty"`
Format *string `json:"format,omitempty"`
FormatID *string `json:"format_id,omitempty"`
EXT *MediaEXT `json:"ext,omitempty"`
Protocol *string `json:"protocol,omitempty"`
Language *Language `json:"language,omitempty"`
FormatNote *string `json:"format_note,omitempty"`
FilesizeApprox *int64 `json:"filesize_approx,omitempty"`
Tbr *float64 `json:"tbr,omitempty"`
Width *int64 `json:"width,omitempty"`
Height *int64 `json:"height,omitempty"`
Resolution *Resolution `json:"resolution,omitempty"`
FPS *int64 `json:"fps,omitempty"`
DynamicRange *DynamicRange `json:"dynamic_range,omitempty"`
Vcodec *string `json:"vcodec,omitempty"`
Vbr *float64 `json:"vbr,omitempty"`
StretchedRatio interface{} `json:"stretched_ratio"`
AspectRatio *float64 `json:"aspect_ratio,omitempty"`
Acodec *Acodec `json:"acodec,omitempty"`
ABR *float64 `json:"abr,omitempty"`
ASR *int64 `json:"asr,omitempty"`
AudioChannels *int64 `json:"audio_channels,omitempty"`
}
type AutomaticCaption struct {
EXT AutomaticCaptionEXT `json:"ext"`
URL string `json:"url"`
Name string `json:"name"`
}
type Entry struct {
ID string `json:"id"`
Title string `json:"title"`
Formats []Format `json:"formats"`
Thumbnails []Thumbnail `json:"thumbnails"`
Thumbnail string `json:"thumbnail"`
Description string `json:"description"`
ChannelID string `json:"channel_id"`
ChannelURL string `json:"channel_url"`
Duration int64 `json:"duration"`
ViewCount int64 `json:"view_count"`
AverageRating interface{} `json:"average_rating"`
AgeLimit int64 `json:"age_limit"`
WebpageURL string `json:"webpage_url"`
Categories []string `json:"categories"`
Tags []string `json:"tags"`
PlayableInEmbed bool `json:"playable_in_embed"`
LiveStatus string `json:"live_status"`
ReleaseTimestamp interface{} `json:"release_timestamp"`
FormatSortFields []string `json:"_format_sort_fields"`
AutomaticCaptions map[string][]AutomaticCaption `json:"automatic_captions"`
Subtitles FilesToMove `json:"subtitles"`
CommentCount int64 `json:"comment_count"`
Chapters interface{} `json:"chapters"`
Heatmap interface{} `json:"heatmap"`
LikeCount int64 `json:"like_count"`
Channel string `json:"channel"`
ChannelFollowerCount int64 `json:"channel_follower_count"`
Uploader string `json:"uploader"`
UploaderID string `json:"uploader_id"`
UploaderURL string `json:"uploader_url"`
UploadDate string `json:"upload_date"`
Timestamp int64 `json:"timestamp"`
Availability string `json:"availability"`
OriginalURL string `json:"original_url"`
WebpageURLBasename string `json:"webpage_url_basename"`
WebpageURLDomain string `json:"webpage_url_domain"`
Extractor string `json:"extractor"`
ExtractorKey string `json:"extractor_key"`
PlaylistCount int64 `json:"playlist_count"`
Playlist string `json:"playlist"`
PlaylistID string `json:"playlist_id"`
PlaylistTitle string `json:"playlist_title"`
PlaylistUploader string `json:"playlist_uploader"`
PlaylistUploaderID string `json:"playlist_uploader_id"`
NEntries int64 `json:"n_entries"`
PlaylistIndex int64 `json:"playlist_index"`
LastPlaylistIndex int64 `json:"__last_playlist_index"`
PlaylistAutonumber int64 `json:"playlist_autonumber"`
DisplayID string `json:"display_id"`
Fulltitle string `json:"fulltitle"`
DurationString string `json:"duration_string"`
ReleaseYear interface{} `json:"release_year"`
IsLive bool `json:"is_live"`
WasLive bool `json:"was_live"`
RequestedSubtitles interface{} `json:"requested_subtitles"`
HasDRM interface{} `json:"_has_drm"`
Epoch int64 `json:"epoch"`
RequestedDownloads []RequestedDownload `json:"requested_downloads"`
RequestedFormats []Format `json:"requested_formats"`
Format string `json:"format"`
FormatID string `json:"format_id"`
EXT string `json:"ext"`
Protocol string `json:"protocol"`
Language *Language `json:"language"`
FormatNote string `json:"format_note"`
FilesizeApprox int64 `json:"filesize_approx"`
Tbr float64 `json:"tbr"`
Width int64 `json:"width"`
Height int64 `json:"height"`
Resolution Resolution `json:"resolution"`
FPS int64 `json:"fps"`
DynamicRange DynamicRange `json:"dynamic_range"`
Vcodec string `json:"vcodec"`
Vbr float64 `json:"vbr"`
StretchedRatio interface{} `json:"stretched_ratio"`
AspectRatio float64 `json:"aspect_ratio"`
Acodec Acodec `json:"acodec"`
ABR float64 `json:"abr"`
ASR int64 `json:"asr"`
AudioChannels int64 `json:"audio_channels"`
}
type Format struct {
FormatID string `json:"format_id"`
FormatNote *FormatNote `json:"format_note,omitempty"`
EXT MediaEXT `json:"ext"`
Protocol Protocol `json:"protocol"`
Acodec *Acodec `json:"acodec,omitempty"`
Vcodec string `json:"vcodec"`
URL string `json:"url"`
Width *int64 `json:"width"`
Height *int64 `json:"height"`
FPS *float64 `json:"fps"`
Rows *int64 `json:"rows,omitempty"`
Columns *int64 `json:"columns,omitempty"`
Fragments []Fragment `json:"fragments,omitempty"`
Resolution Resolution `json:"resolution"`
AspectRatio *float64 `json:"aspect_ratio"`
FilesizeApprox *int64 `json:"filesize_approx"`
HTTPHeaders HTTPHeaders `json:"http_headers"`
AudioEXT MediaEXT `json:"audio_ext"`
VideoEXT MediaEXT `json:"video_ext"`
Vbr *float64 `json:"vbr"`
ABR *float64 `json:"abr"`
Tbr *float64 `json:"tbr"`
Format string `json:"format"`
FormatIndex interface{} `json:"format_index"`
ManifestURL *string `json:"manifest_url,omitempty"`
Language *Language `json:"language"`
Preference interface{} `json:"preference"`
Quality *float64 `json:"quality,omitempty"`
HasDRM *bool `json:"has_drm,omitempty"`
SourcePreference *int64 `json:"source_preference,omitempty"`
ASR *int64 `json:"asr"`
Filesize *int64 `json:"filesize"`
AudioChannels *int64 `json:"audio_channels"`
LanguagePreference *int64 `json:"language_preference,omitempty"`
DynamicRange *DynamicRange `json:"dynamic_range"`
Container *Container `json:"container,omitempty"`
DownloaderOptions *DownloaderOptions `json:"downloader_options,omitempty"`
}
type DownloaderOptions struct {
HTTPChunkSize int64 `json:"http_chunk_size"`
}
type Fragment struct {
URL string `json:"url"`
Duration float64 `json:"duration"`
}
type HTTPHeaders struct {
UserAgent string `json:"User-Agent"`
Accept Accept `json:"Accept"`
AcceptLanguage AcceptLanguage `json:"Accept-Language"`
SECFetchMode SECFetchMode `json:"Sec-Fetch-Mode"`
}
type RequestedDownload struct {
RequestedFormats []Format `json:"requested_formats"`
Format string `json:"format"`
FormatID string `json:"format_id"`
EXT string `json:"ext"`
Protocol string `json:"protocol"`
FormatNote string `json:"format_note"`
FilesizeApprox int64 `json:"filesize_approx"`
Tbr float64 `json:"tbr"`
Width int64 `json:"width"`
Height int64 `json:"height"`
Resolution Resolution `json:"resolution"`
FPS int64 `json:"fps"`
DynamicRange DynamicRange `json:"dynamic_range"`
Vcodec string `json:"vcodec"`
Vbr float64 `json:"vbr"`
AspectRatio float64 `json:"aspect_ratio"`
Acodec Acodec `json:"acodec"`
ABR float64 `json:"abr"`
ASR int64 `json:"asr"`
AudioChannels int64 `json:"audio_channels"`
FilenameOld string `json:"_filename"`
Filename string `json:"filename"`
WriteDownloadArchive bool `json:"__write_download_archive"`
Language *Language `json:"language,omitempty"`
}
type FilesToMove struct {
}
type Thumbnail struct {
URL string `json:"url"`
Preference *int64 `json:"preference,omitempty"`
ID string `json:"id"`
Height *int64 `json:"height,omitempty"`
Width *int64 `json:"width,omitempty"`
Resolution *string `json:"resolution,omitempty"`
}
type Heatmap struct {
StartTime float64 `json:"start_time"`
EndTime float64 `json:"end_time"`
Value float64 `json:"value"`
}
type Version struct {
Version string `json:"version"`
CurrentGitHead interface{} `json:"current_git_head"`
ReleaseGitHead string `json:"release_git_head"`
Repository string `json:"repository"`
}
type Acodec string
const (
AcodecNone Acodec = "none"
Mp4A402 Acodec = "mp4a.40.2"
Mp4A405 Acodec = "mp4a.40.5"
Opus Acodec = "opus"
)
type AutomaticCaptionEXT string
const (
Json3 AutomaticCaptionEXT = "json3"
Srv1 AutomaticCaptionEXT = "srv1"
Srv2 AutomaticCaptionEXT = "srv2"
Srv3 AutomaticCaptionEXT = "srv3"
Ttml AutomaticCaptionEXT = "ttml"
Vtt AutomaticCaptionEXT = "vtt"
)
type DynamicRange string
const (
SDR DynamicRange = "SDR"
HDR DynamicRange = "HDR"
)
type MediaEXT string
const (
EXTNone MediaEXT = "none"
EXTMhtml MediaEXT = "mhtml"
M4A MediaEXT = "m4a"
Mp4 MediaEXT = "mp4"
Webm MediaEXT = "webm"
)
type Container string
const (
M4ADash Container = "m4a_dash"
Mp4Dash Container = "mp4_dash"
WebmDash Container = "webm_dash"
)
type FormatNote string
const (
Default FormatNote = "Default"
Low FormatNote = "low"
Medium FormatNote = "medium"
Premium FormatNote = "Premium"
Storyboard FormatNote = "storyboard"
The1080P FormatNote = "1080p"
The144P FormatNote = "144p"
The240P FormatNote = "240p"
The360P FormatNote = "360p"
The480P FormatNote = "480p"
The720P FormatNote = "720p"
)
type Accept string
const (
TextHTMLApplicationXHTMLXMLApplicationXMLQ09Q08 Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
)
type AcceptLanguage string
const (
EnUsEnQ05 AcceptLanguage = "en-us,en;q=0.5"
)
type SECFetchMode string
const (
Navigate SECFetchMode = "navigate"
)
type Language string
const (
En Language = "en"
)
type Protocol string
const (
HTTPS Protocol = "https"
M3U8Native Protocol = "m3u8_native"
ProtocolMhtml Protocol = "mhtml"
)
type Resolution string
const (
AudioOnly Resolution = "audio only"
The1280X720 Resolution = "1280x720"
The160X90 Resolution = "160x90"
The1920X1080 Resolution = "1920x1080"
The256X144 Resolution = "256x144"
The320X180 Resolution = "320x180"
The426X240 Resolution = "426x240"
The48X27 Resolution = "48x27"
The640X360 Resolution = "640x360"
The80X45 Resolution = "80x45"
The854X480 Resolution = "854x480"
)

115
server/pkg/ytdlp/playlist.go Normal file
View file

@ -0,0 +1,115 @@
package ytdlp
import (
"bufio"
"context"
"encoding/json"
"errors"
"io"
"os"
"os/exec"
"strings"
"github.com/royalcat/ctxprogress"
"golang.org/x/sync/errgroup"
)
// Progress implements ctxprogress.Progress.
func (p Entry) Progress() (current int, total int) {
return int(p.PlaylistIndex), int(p.PlaylistCount)
}
// func (p PlaylistEntry) Url() string {
// if p.URL != "" {
// return p.URL
// }
// if p.WebpageURL != "" {
// return p.WebpageURL
// }
// if p.OriginalURL != "" {
// return p.OriginalURL
// }
// return ""
// }
func (yt *Client) Playlist(ctx context.Context, url string) ([]Entry, error) {
group, ctx := errgroup.WithContext(ctx)
w, lines, err := lineReader(group)
if err != nil {
return nil, err
}
cmd := exec.CommandContext(ctx, yt.binary, "-j", url)
cmd.Stdout = w
group.Go(func() error {
err := cmd.Run()
if err != nil {
return err
}
return w.Close()
})
playlists := []Entry{}
for line := range lines {
entry := Entry{}
err = json.Unmarshal([]byte(line), &entry)
if err != nil {
return nil, err
}
playlists = append(playlists, entry)
}
return playlists, nil
}
func lineReader(group *errgroup.Group) (io.WriteCloser, <-chan string, error) {
lines := make(chan string)
var r io.Reader
r, w := io.Pipe()
r = io.TeeReader(r, os.Stdout)
group.Go(func() error {
defer close(lines)
reader := bufio.NewReader(r)
for {
line, err := reader.ReadString('\n')
if err != nil {
if err == io.EOF {
return w.Close()
}
return errors.Join(err, w.Close())
}
line = strings.Trim(line, " \r\t")
if line == "" {
continue
}
lines <- line
}
})
return w, lines, nil
}
var _ ctxprogress.Progress = (*Entry)(nil)
var _ ctxprogress.Progress = (*DownloadProgress)(nil)
func parseProgress(line string) (ctxprogress.Progress, error) {
line = strings.Trim(line, " \r\t")
p := &DownloadProgress{}
err := json.Unmarshal([]byte(line), p)
if err != nil {
return nil, err
}
return p, nil
}

27
server/pkg/ytdlp/playlist_test.go Normal file
View file

@ -0,0 +1,27 @@
package ytdlp_test
import (
"context"
"errors"
"testing"
"git.kmsign.ru/royalcat/tstor/server/pkg/ytdlp"
"github.com/stretchr/testify/require"
)
func TestPlaylist(t *testing.T) {
require := require.New(t)
ctx := context.Background()
if deadline, ok := t.Deadline(); ok {
var cancel context.CancelFunc
ctx, cancel = context.WithDeadlineCause(ctx, deadline, errors.New("test deadline done"))
defer cancel()
}
client, err := ytdlp.New()
require.NoError(err)
entries, err := client.Playlist(ctx, "https://www.youtube.com/playlist?list=PLUay9m6GhoyCXdloEa-VYtnVeshaKl4AW")
require.NoError(err)
require.NotEmpty(entries)
require.Len(entries, int(entries[0].PlaylistCount))
}