small refactor*

This commit is contained in:
royalcat 2025-03-22 08:49:14 +04:00
parent b6b541e050
commit 24a4d30275
232 changed files with 2164 additions and 1906 deletions

111
plugins/ytdlp/controller.go Normal file
View file

@ -0,0 +1,111 @@
package ytdlp
import (
"context"
"os"
"git.kmsign.ru/royalcat/tstor/server/pkg/ctxbilly"
"git.kmsign.ru/royalcat/tstor/server/pkg/kvsingle"
"git.kmsign.ru/royalcat/tstor/server/pkg/rlog"
"git.kmsign.ru/royalcat/tstor/server/pkg/ytdlp"
"git.kmsign.ru/royalcat/tstor/server/src/tasks"
"github.com/royalcat/ctxio"
"github.com/royalcat/ctxprogress"
"github.com/royalcat/kv"
)
type Controller struct {
datafs ctxbilly.Filesystem
source Source
client *ytdlp.Client
cachedinfo *kvsingle.Value[string, ytdlp.Info]
}
func newYtdlpController(datafs ctxbilly.Filesystem, source Source, client *ytdlp.Client) *Controller {
return &Controller{
datafs: datafs,
source: source,
client: client,
}
}
func (c *Controller) Source() Source {
return c.source
}
const sizeApprox = 1024 * 1024 * 1024
func (c *Controller) Update(ctx context.Context, updater tasks.Updater) error {
log := updater.Logger()
ctxprogress.New(ctx)
ctxprogress.Set(ctx, ctxprogress.RangeProgress{Current: 0, Total: 10})
plst, err := c.client.Playlist(ctx, c.source.Url)
ctxprogress.Set(ctx, ctxprogress.RangeProgress{Current: 1, Total: 10})
ctxprogress.Range(ctx, plst, func(ctx context.Context, _ int, e ytdlp.Entry) bool {
if e.OriginalURL == "" {
log.Error("no URL in entry", rlog.Error(err))
return true
}
info, err := c.Info(ctx)
if err != nil {
log.Error("error getting info", rlog.Error(err))
return true
}
dwl := info.RequestedDownloads[0]
fileinfo, err := c.datafs.Stat(ctx, dwl.Filename)
if err != nil {
log.Error("error getting file info", rlog.Error(err))
return true
}
if fileinfo.Size()+sizeApprox > dwl.FilesizeApprox && fileinfo.Size()-sizeApprox < dwl.FilesizeApprox {
log.Debug("file already downloaded", "filename", dwl.Filename)
return true
}
file, err := c.datafs.OpenFile(ctx, dwl.Filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
log.Error("error opening destination file", rlog.Error(err))
return true
}
err = c.client.Download(ctx, info.OriginalURL, ctxio.IoWriter(ctx, file))
if err != nil {
return false
}
return true
})
ctxprogress.Set(ctx, ctxprogress.RangeProgress{Current: 2, Total: 2})
if err != nil {
return err
}
return nil
}
func (c *Controller) Info(ctx context.Context) (ytdlp.Info, error) {
info, err := c.cachedinfo.Get(ctx)
if err == nil {
return info, nil
}
if err != kv.ErrKeyNotFound {
return info, err
}
info, err = c.Info(ctx)
if err != nil {
return info, err
}
err = c.cachedinfo.Set(ctx, info)
if err != nil {
return info, err
}
return info, nil
}
func (c *Controller) Downloaded() error {
return nil
}

71
plugins/ytdlp/daemon.go Normal file
View file

@ -0,0 +1,71 @@
package ytdlp
import (
"context"
"encoding/json"
"fmt"
"path"
"sync"
"git.kmsign.ru/royalcat/tstor/server/pkg/ctxbilly"
"git.kmsign.ru/royalcat/tstor/server/pkg/ytdlp"
"git.kmsign.ru/royalcat/tstor/server/src/vfs"
"github.com/go-git/go-billy/v5/osfs"
"github.com/royalcat/ctxio"
)
func NewService(dataDir string) (*Daemon, error) {
client, err := ytdlp.New()
if err != nil {
return nil, err
}
s := &Daemon{
mu: sync.Mutex{},
client: client,
dataDir: dataDir,
controllers: make(map[string]*Controller, 0),
}
return s, nil
}
type Daemon struct {
mu sync.Mutex
dataDir string
client *ytdlp.Client
controllers map[string]*Controller
}
func (c *Daemon) addSource(s Source) {
c.mu.Lock()
defer c.mu.Unlock()
ctl := newYtdlpController(ctxbilly.WrapFileSystem(osfs.New(c.sourceDir(s))), s, c.client)
c.controllers[s.Name()] = ctl
}
func (c *Daemon) sourceDir(s Source) string {
return path.Join(c.dataDir, s.Name())
}
func (c *Daemon) BuildFS(ctx context.Context, sourcePath string, f vfs.File) (vfs.Filesystem, error) {
data, err := ctxio.ReadAll(ctx, f)
if err != nil {
return nil, fmt.Errorf("failed to read source file: %w", err)
}
var s Source
err = json.Unmarshal(data, &s)
if err != nil {
return nil, err
}
c.addSource(s)
downloadFS := ctxbilly.WrapFileSystem(osfs.New(c.sourceDir(s)))
return newSourceFS(path.Base(f.Name()), downloadFS, c, s), nil
}

76
plugins/ytdlp/fs.go Normal file
View file

@ -0,0 +1,76 @@
package ytdlp
import (
"context"
"io/fs"
"os"
"time"
"git.kmsign.ru/royalcat/tstor/server/pkg/ctxbilly"
"git.kmsign.ru/royalcat/tstor/server/src/vfs"
)
type SourceFS struct {
service *Daemon
source Source
fs ctxbilly.Filesystem
vfs.DefaultFS
}
var _ vfs.Filesystem = (*SourceFS)(nil)
func newSourceFS(name string, fs ctxbilly.Filesystem, service *Daemon, source Source) *SourceFS {
return &SourceFS{
fs: fs,
service: service,
source: source,
DefaultFS: vfs.DefaultFS(name),
}
}
// Open implements vfs.Filesystem.
func (s *SourceFS) Open(ctx context.Context, filename string) (vfs.File, error) {
info, err := s.fs.Stat(ctx, filename)
if err != nil {
return nil, err
}
f, err := s.fs.OpenFile(ctx, filename, os.O_RDONLY, 0)
if err != nil {
return nil, err
}
return vfs.NewCtxBillyFile(info, f), nil
}
// ReadDir implements vfs.Filesystem.
func (s *SourceFS) ReadDir(ctx context.Context, path string) ([]fs.DirEntry, error) {
infos, err := s.fs.ReadDir(ctx, path)
if err != nil {
return nil, err
}
entries := make([]fs.DirEntry, 0, len(infos))
for _, info := range infos {
entries = append(entries, vfs.NewFileInfo(info.Name(), info.Size(), time.Time{}))
}
return entries, nil
}
// Stat implements vfs.Filesystem.
func (s *SourceFS) Stat(ctx context.Context, filename string) (fs.FileInfo, error) {
return s.fs.Stat(ctx, filename)
}
// Unlink implements vfs.Filesystem.
func (s *SourceFS) Unlink(ctx context.Context, filename string) error {
return vfs.ErrNotImplemented
}
// Rename implements vfs.Filesystem.
func (s *SourceFS) Rename(ctx context.Context, oldpath string, newpath string) error {
return vfs.ErrNotImplemented
}

37
plugins/ytdlp/tasks.go Normal file
View file

@ -0,0 +1,37 @@
package ytdlp
import (
"context"
"fmt"
"git.kmsign.ru/royalcat/tstor/server/src/tasks"
)
const executorName = "ytdlp"
type DownloadTask struct {
Name string
}
var _ tasks.Task = (*DownloadTask)(nil)
// Executor implements tasks.Task.
func (d *DownloadTask) Executor() string {
return executorName
}
var _ tasks.TaskExecutor = (*Daemon)(nil)
// ExecutorName implements tasks.TaskExecutor.
func (c *Daemon) ExecutorName() string {
return executorName
}
func (c *Daemon) RunTask(ctx context.Context, upd tasks.Updater, task tasks.Task) error {
switch t := task.(type) {
case *DownloadTask:
return c.controllers[t.Name].Update(ctx, upd)
default:
return fmt.Errorf("unknown task type: %T", task)
}
}

29
plugins/ytdlp/ytdlp.go Normal file
View file

@ -0,0 +1,29 @@
package ytdlp
import (
"crypto/sha1"
"encoding/base64"
"strings"
)
type Source struct {
Url string `json:"url"`
}
var hasher = sha1.New()
var prefixCutset = [...]string{
"https://", "http://", "www.",
}
func urlHash(url string) string {
for _, v := range prefixCutset {
url = strings.TrimPrefix(url, v)
}
return base64.URLEncoding.EncodeToString(hasher.Sum([]byte(url)))
}
func (s *Source) Name() string {
return urlHash(s.Url)
}