small refactor*
This commit is contained in:
parent
b6b541e050
commit
24a4d30275
232 changed files with 2164 additions and 1906 deletions
plugins/ytdlp
111
plugins/ytdlp/controller.go
Normal file
111
plugins/ytdlp/controller.go
Normal file
|
@ -0,0 +1,111 @@
|
|||
package ytdlp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/server/pkg/ctxbilly"
|
||||
"git.kmsign.ru/royalcat/tstor/server/pkg/kvsingle"
|
||||
"git.kmsign.ru/royalcat/tstor/server/pkg/rlog"
|
||||
"git.kmsign.ru/royalcat/tstor/server/pkg/ytdlp"
|
||||
"git.kmsign.ru/royalcat/tstor/server/src/tasks"
|
||||
"github.com/royalcat/ctxio"
|
||||
"github.com/royalcat/ctxprogress"
|
||||
"github.com/royalcat/kv"
|
||||
)
|
||||
|
||||
type Controller struct {
|
||||
datafs ctxbilly.Filesystem
|
||||
source Source
|
||||
client *ytdlp.Client
|
||||
cachedinfo *kvsingle.Value[string, ytdlp.Info]
|
||||
}
|
||||
|
||||
func newYtdlpController(datafs ctxbilly.Filesystem, source Source, client *ytdlp.Client) *Controller {
|
||||
return &Controller{
|
||||
datafs: datafs,
|
||||
source: source,
|
||||
client: client,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) Source() Source {
|
||||
return c.source
|
||||
}
|
||||
|
||||
const sizeApprox = 1024 * 1024 * 1024
|
||||
|
||||
func (c *Controller) Update(ctx context.Context, updater tasks.Updater) error {
|
||||
log := updater.Logger()
|
||||
ctxprogress.New(ctx)
|
||||
ctxprogress.Set(ctx, ctxprogress.RangeProgress{Current: 0, Total: 10})
|
||||
plst, err := c.client.Playlist(ctx, c.source.Url)
|
||||
ctxprogress.Set(ctx, ctxprogress.RangeProgress{Current: 1, Total: 10})
|
||||
ctxprogress.Range(ctx, plst, func(ctx context.Context, _ int, e ytdlp.Entry) bool {
|
||||
if e.OriginalURL == "" {
|
||||
log.Error("no URL in entry", rlog.Error(err))
|
||||
return true
|
||||
}
|
||||
|
||||
info, err := c.Info(ctx)
|
||||
if err != nil {
|
||||
log.Error("error getting info", rlog.Error(err))
|
||||
return true
|
||||
}
|
||||
|
||||
dwl := info.RequestedDownloads[0]
|
||||
|
||||
fileinfo, err := c.datafs.Stat(ctx, dwl.Filename)
|
||||
if err != nil {
|
||||
log.Error("error getting file info", rlog.Error(err))
|
||||
return true
|
||||
}
|
||||
|
||||
if fileinfo.Size()+sizeApprox > dwl.FilesizeApprox && fileinfo.Size()-sizeApprox < dwl.FilesizeApprox {
|
||||
log.Debug("file already downloaded", "filename", dwl.Filename)
|
||||
return true
|
||||
}
|
||||
|
||||
file, err := c.datafs.OpenFile(ctx, dwl.Filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
|
||||
if err != nil {
|
||||
log.Error("error opening destination file", rlog.Error(err))
|
||||
return true
|
||||
}
|
||||
|
||||
err = c.client.Download(ctx, info.OriginalURL, ctxio.IoWriter(ctx, file))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
ctxprogress.Set(ctx, ctxprogress.RangeProgress{Current: 2, Total: 2})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Controller) Info(ctx context.Context) (ytdlp.Info, error) {
|
||||
info, err := c.cachedinfo.Get(ctx)
|
||||
if err == nil {
|
||||
return info, nil
|
||||
}
|
||||
if err != kv.ErrKeyNotFound {
|
||||
return info, err
|
||||
}
|
||||
|
||||
info, err = c.Info(ctx)
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
err = c.cachedinfo.Set(ctx, info)
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (c *Controller) Downloaded() error {
|
||||
return nil
|
||||
}
|
71
plugins/ytdlp/daemon.go
Normal file
71
plugins/ytdlp/daemon.go
Normal file
|
@ -0,0 +1,71 @@
|
|||
package ytdlp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"sync"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/server/pkg/ctxbilly"
|
||||
"git.kmsign.ru/royalcat/tstor/server/pkg/ytdlp"
|
||||
"git.kmsign.ru/royalcat/tstor/server/src/vfs"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/royalcat/ctxio"
|
||||
)
|
||||
|
||||
func NewService(dataDir string) (*Daemon, error) {
|
||||
client, err := ytdlp.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &Daemon{
|
||||
mu: sync.Mutex{},
|
||||
client: client,
|
||||
dataDir: dataDir,
|
||||
controllers: make(map[string]*Controller, 0),
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
type Daemon struct {
|
||||
mu sync.Mutex
|
||||
|
||||
dataDir string
|
||||
client *ytdlp.Client
|
||||
controllers map[string]*Controller
|
||||
}
|
||||
|
||||
func (c *Daemon) addSource(s Source) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
ctl := newYtdlpController(ctxbilly.WrapFileSystem(osfs.New(c.sourceDir(s))), s, c.client)
|
||||
|
||||
c.controllers[s.Name()] = ctl
|
||||
}
|
||||
|
||||
func (c *Daemon) sourceDir(s Source) string {
|
||||
return path.Join(c.dataDir, s.Name())
|
||||
}
|
||||
|
||||
func (c *Daemon) BuildFS(ctx context.Context, sourcePath string, f vfs.File) (vfs.Filesystem, error) {
|
||||
data, err := ctxio.ReadAll(ctx, f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read source file: %w", err)
|
||||
}
|
||||
|
||||
var s Source
|
||||
err = json.Unmarshal(data, &s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.addSource(s)
|
||||
|
||||
downloadFS := ctxbilly.WrapFileSystem(osfs.New(c.sourceDir(s)))
|
||||
|
||||
return newSourceFS(path.Base(f.Name()), downloadFS, c, s), nil
|
||||
}
|
76
plugins/ytdlp/fs.go
Normal file
76
plugins/ytdlp/fs.go
Normal file
|
@ -0,0 +1,76 @@
|
|||
package ytdlp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/server/pkg/ctxbilly"
|
||||
"git.kmsign.ru/royalcat/tstor/server/src/vfs"
|
||||
)
|
||||
|
||||
type SourceFS struct {
|
||||
service *Daemon
|
||||
source Source
|
||||
|
||||
fs ctxbilly.Filesystem
|
||||
|
||||
vfs.DefaultFS
|
||||
}
|
||||
|
||||
var _ vfs.Filesystem = (*SourceFS)(nil)
|
||||
|
||||
func newSourceFS(name string, fs ctxbilly.Filesystem, service *Daemon, source Source) *SourceFS {
|
||||
return &SourceFS{
|
||||
fs: fs,
|
||||
service: service,
|
||||
source: source,
|
||||
DefaultFS: vfs.DefaultFS(name),
|
||||
}
|
||||
}
|
||||
|
||||
// Open implements vfs.Filesystem.
|
||||
func (s *SourceFS) Open(ctx context.Context, filename string) (vfs.File, error) {
|
||||
info, err := s.fs.Stat(ctx, filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f, err := s.fs.OpenFile(ctx, filename, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return vfs.NewCtxBillyFile(info, f), nil
|
||||
}
|
||||
|
||||
// ReadDir implements vfs.Filesystem.
|
||||
func (s *SourceFS) ReadDir(ctx context.Context, path string) ([]fs.DirEntry, error) {
|
||||
infos, err := s.fs.ReadDir(ctx, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entries := make([]fs.DirEntry, 0, len(infos))
|
||||
for _, info := range infos {
|
||||
entries = append(entries, vfs.NewFileInfo(info.Name(), info.Size(), time.Time{}))
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// Stat implements vfs.Filesystem.
|
||||
func (s *SourceFS) Stat(ctx context.Context, filename string) (fs.FileInfo, error) {
|
||||
return s.fs.Stat(ctx, filename)
|
||||
}
|
||||
|
||||
// Unlink implements vfs.Filesystem.
|
||||
func (s *SourceFS) Unlink(ctx context.Context, filename string) error {
|
||||
return vfs.ErrNotImplemented
|
||||
}
|
||||
|
||||
// Rename implements vfs.Filesystem.
|
||||
func (s *SourceFS) Rename(ctx context.Context, oldpath string, newpath string) error {
|
||||
return vfs.ErrNotImplemented
|
||||
}
|
37
plugins/ytdlp/tasks.go
Normal file
37
plugins/ytdlp/tasks.go
Normal file
|
@ -0,0 +1,37 @@
|
|||
package ytdlp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/server/src/tasks"
|
||||
)
|
||||
|
||||
const executorName = "ytdlp"
|
||||
|
||||
type DownloadTask struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
var _ tasks.Task = (*DownloadTask)(nil)
|
||||
|
||||
// Executor implements tasks.Task.
|
||||
func (d *DownloadTask) Executor() string {
|
||||
return executorName
|
||||
}
|
||||
|
||||
var _ tasks.TaskExecutor = (*Daemon)(nil)
|
||||
|
||||
// ExecutorName implements tasks.TaskExecutor.
|
||||
func (c *Daemon) ExecutorName() string {
|
||||
return executorName
|
||||
}
|
||||
|
||||
func (c *Daemon) RunTask(ctx context.Context, upd tasks.Updater, task tasks.Task) error {
|
||||
switch t := task.(type) {
|
||||
case *DownloadTask:
|
||||
return c.controllers[t.Name].Update(ctx, upd)
|
||||
default:
|
||||
return fmt.Errorf("unknown task type: %T", task)
|
||||
}
|
||||
}
|
29
plugins/ytdlp/ytdlp.go
Normal file
29
plugins/ytdlp/ytdlp.go
Normal file
|
@ -0,0 +1,29 @@
|
|||
package ytdlp
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
Url string `json:"url"`
|
||||
}
|
||||
|
||||
var hasher = sha1.New()
|
||||
|
||||
var prefixCutset = [...]string{
|
||||
"https://", "http://", "www.",
|
||||
}
|
||||
|
||||
func urlHash(url string) string {
|
||||
for _, v := range prefixCutset {
|
||||
url = strings.TrimPrefix(url, v)
|
||||
}
|
||||
|
||||
return base64.URLEncoding.EncodeToString(hasher.Sum([]byte(url)))
|
||||
}
|
||||
|
||||
func (s *Source) Name() string {
|
||||
return urlHash(s.Url)
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue