refactor
This commit is contained in:
parent
d056ac1167
commit
bd75492b02
81 changed files with 822 additions and 1098 deletions
src
delivery
export
fuse
httpfs
nfs
webdav
host
iio
sources
source.gostorage.go
torrent
client.gocontroller.gofile_mappings.gofileitem.gofs.gofs_test.goid.goinfobytes.gokv.gopiece_completion.gopiece_storage.goqueue.goservice.gosetup.gostats.gostats_store.gostorage.go
ytdlp
vfs
|
@ -7,7 +7,7 @@ import (
|
|||
"net/http"
|
||||
"os"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
"github.com/anacrolix/missinggo/v2/filecache"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
|
|
@ -3,8 +3,8 @@ package model
|
|||
import (
|
||||
"context"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
)
|
||||
|
||||
type FsElem interface {
|
||||
|
|
|
@ -3,7 +3,7 @@ package model
|
|||
import (
|
||||
"context"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
atorrent "github.com/anacrolix/torrent"
|
||||
)
|
||||
|
||||
|
|
|
@ -5,8 +5,8 @@ package model
|
|||
import (
|
||||
"time"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
torrent1 "github.com/anacrolix/torrent"
|
||||
)
|
||||
|
||||
|
|
|
@ -14,7 +14,7 @@ import (
|
|||
"git.kmsign.ru/royalcat/tstor/pkg/uuid"
|
||||
graph "git.kmsign.ru/royalcat/tstor/src/delivery/graphql"
|
||||
"git.kmsign.ru/royalcat/tstor/src/delivery/graphql/model"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
"github.com/99designs/gqlgen/graphql"
|
||||
aih "github.com/anacrolix/torrent/types/infohash"
|
||||
)
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
|
||||
graph "git.kmsign.ru/royalcat/tstor/src/delivery/graphql"
|
||||
"git.kmsign.ru/royalcat/tstor/src/delivery/graphql/model"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
)
|
||||
|
||||
// Torrents is the resolver for the torrents field.
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
package resolver
|
||||
|
||||
import (
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/go-git/go-billy/v5"
|
||||
)
|
||||
|
||||
|
|
|
@ -7,8 +7,8 @@ import (
|
|||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/rlog"
|
||||
"git.kmsign.ru/royalcat/tstor/src/config"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/anacrolix/missinggo/v2/filecache"
|
||||
echopprof "github.com/labstack/echo-contrib/pprof"
|
||||
"github.com/labstack/echo/v4"
|
||||
|
|
|
@ -8,8 +8,8 @@ import (
|
|||
"git.kmsign.ru/royalcat/tstor/pkg/rlog"
|
||||
graph "git.kmsign.ru/royalcat/tstor/src/delivery/graphql"
|
||||
"git.kmsign.ru/royalcat/tstor/src/delivery/graphql/resolver"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/99designs/gqlgen/graphql"
|
||||
"github.com/99designs/gqlgen/graphql/handler"
|
||||
"github.com/99designs/gqlgen/graphql/handler/extension"
|
||||
|
|
|
@ -8,7 +8,7 @@ import (
|
|||
"path/filepath"
|
||||
"runtime"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/billziss-gh/cgofuse/fuse"
|
||||
)
|
||||
|
||||
|
|
|
@ -5,7 +5,7 @@ package fuse
|
|||
import (
|
||||
"fmt"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
)
|
||||
|
||||
type Handler struct{}
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
"os"
|
||||
"sync"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/billziss-gh/cgofuse/fuse"
|
||||
)
|
||||
|
||||
|
|
|
@ -9,7 +9,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
|
@ -8,8 +8,8 @@ import (
|
|||
"os"
|
||||
"sync"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ctxio"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ioutils"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
@ -93,7 +93,7 @@ func newHTTPFile(ctx context.Context, f vfs.File, dirContent []os.FileInfo) *htt
|
|||
return &httpFile{
|
||||
f: f,
|
||||
dirContent: dirContent,
|
||||
ReadSeekCloser: ctxio.IoReadSeekCloserWrapper(ctx, f, f.Size()),
|
||||
ReadSeekCloser: ioutils.IoReadSeekCloserWrapper(ctx, f, f.Size()),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -6,8 +6,8 @@ import (
|
|||
|
||||
nfs "git.kmsign.ru/royalcat/tstor/pkg/go-nfs"
|
||||
nfshelper "git.kmsign.ru/royalcat/tstor/pkg/go-nfs/helpers"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/log"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
)
|
||||
|
||||
func NewNFSv3Handler(fs vfs.Filesystem) (nfs.Handler, error) {
|
||||
|
|
|
@ -3,7 +3,7 @@ package nfs
|
|||
// import (
|
||||
// "io/fs"
|
||||
|
||||
// "git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
// "git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
// nfsfs "github.com/smallfz/libnfs-go/fs"
|
||||
// )
|
||||
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
"git.kmsign.ru/royalcat/tstor/pkg/ctxbilly"
|
||||
nfs "git.kmsign.ru/royalcat/tstor/pkg/go-nfs"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/go-git/go-billy/v5"
|
||||
)
|
||||
|
||||
|
@ -199,6 +199,11 @@ func (f *billyFile) Write(ctx context.Context, p []byte) (n int, err error) {
|
|||
return 0, billyErr(nil, vfs.ErrNotImplemented, f.log)
|
||||
}
|
||||
|
||||
// WriteAt implements ctxbilly.File.
|
||||
func (f *billyFile) WriteAt(ctx context.Context, p []byte, off int64) (n int, err error) {
|
||||
return 0, billyErr(nil, vfs.ErrNotImplemented, f.log)
|
||||
}
|
||||
|
||||
// Lock implements billy.File.
|
||||
func (*billyFile) Lock() error {
|
||||
return nil // TODO
|
||||
|
|
|
@ -9,7 +9,7 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"golang.org/x/net/webdav"
|
||||
)
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@ import (
|
|||
"os"
|
||||
"testing"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/webdav"
|
||||
)
|
||||
|
|
|
@ -4,7 +4,7 @@ import (
|
|||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"golang.org/x/net/webdav"
|
||||
)
|
||||
|
||||
|
|
|
@ -5,7 +5,7 @@ import (
|
|||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"golang.org/x/net/webdav"
|
||||
)
|
||||
|
||||
|
|
|
@ -1,51 +0,0 @@
|
|||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/lrstanley/go-ytdlp"
|
||||
)
|
||||
|
||||
type SourceUpdater struct {
|
||||
sources []VirtDirSource
|
||||
}
|
||||
|
||||
type SourcedDirSource string
|
||||
|
||||
const (
|
||||
SourcedDirYtDlp SourcedDirSource = "yt-dlp-playlist"
|
||||
)
|
||||
|
||||
type VirtDirSource interface {
|
||||
Source() SourcedDirSource
|
||||
}
|
||||
|
||||
var _ VirtDirSource = (*SourcedDirYtDlpPlaylist)(nil)
|
||||
|
||||
type SourcedDirYtDlpPlaylist struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
func (SourcedDirYtDlpPlaylist) Source() SourcedDirSource {
|
||||
return SourcedDirYtDlp
|
||||
}
|
||||
|
||||
type SDController struct {
|
||||
sources []VirtDirSource
|
||||
}
|
||||
|
||||
func (sd *SourcedDirYtDlpPlaylist) Update(ctx context.Context) error {
|
||||
_, err := ytdlp.Install(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dl := ytdlp.New().PrintJSON()
|
||||
|
||||
_, err = dl.Run(ctx, sd.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
|
@ -1,19 +0,0 @@
|
|||
package host
|
||||
|
||||
import (
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
)
|
||||
|
||||
func NewHostedFS(sourceFS vfs.Filesystem, tsrv *torrent.Service) vfs.Filesystem {
|
||||
factories := map[string]vfs.FsFactory{
|
||||
".torrent": tsrv.NewTorrentFs,
|
||||
}
|
||||
|
||||
// add default torrent factory for root filesystem
|
||||
for k, v := range vfs.ArchiveFactories {
|
||||
factories[k] = v
|
||||
}
|
||||
|
||||
return vfs.NewResolveFS(sourceFS, factories)
|
||||
}
|
|
@ -1,3 +0,0 @@
|
|||
package vfs
|
||||
|
||||
const sorcedDirExt = ".tsvd"
|
|
@ -5,8 +5,8 @@ import (
|
|||
"io"
|
||||
"testing"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ctxio"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ioutils"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -20,7 +20,7 @@ func TestSeekerWrapper(t *testing.T) {
|
|||
|
||||
mf := vfs.NewMemoryFile("text.txt", testData)
|
||||
|
||||
r := ctxio.IoReadSeekCloserWrapper(ctx, mf, mf.Size())
|
||||
r := ioutils.IoReadSeekCloserWrapper(ctx, mf, mf.Size())
|
||||
defer r.Close()
|
||||
|
||||
n, err := r.Seek(6, io.SeekStart)
|
||||
|
|
53
src/sources/source.go
Normal file
53
src/sources/source.go
Normal file
|
@ -0,0 +1,53 @@
|
|||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
type UpdateTask interface{}
|
||||
|
||||
type Source interface {
|
||||
Name() string // unique name within source type
|
||||
SourceType() string
|
||||
Fetch(ctx context.Context, task UpdateTask, dir string) error
|
||||
}
|
||||
|
||||
var sourceTypesRegistry = map[string]reflect.Type{}
|
||||
|
||||
// func RegisterSource[T Source]() {
|
||||
// var s T
|
||||
// t := reflect.TypeOf(s)
|
||||
// if t.Kind() == reflect.Ptr {
|
||||
// RegisterSource[T]()
|
||||
// return
|
||||
// }
|
||||
|
||||
// sourceTypesRegistry[s.SourceType()] = t
|
||||
// }
|
||||
|
||||
type sourceType struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
func parseSource(data []byte) (Source, error) {
|
||||
var sourceType sourceType
|
||||
err := json.Unmarshal(data, &sourceType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
st, ok := sourceTypesRegistry[sourceType.Type]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("source type %s not registred", sourceType.Type)
|
||||
}
|
||||
|
||||
s := reflect.New(st).Interface().(Source)
|
||||
err = json.Unmarshal(data, &s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
21
src/sources/storage.go
Normal file
21
src/sources/storage.go
Normal file
|
@ -0,0 +1,21 @@
|
|||
package sources
|
||||
|
||||
import (
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/torrent"
|
||||
"git.kmsign.ru/royalcat/tstor/src/sources/ytdlp"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
)
|
||||
|
||||
func NewHostedFS(sourceFS vfs.Filesystem, tsrv *torrent.Service, ytdlpsrv *ytdlp.Service) vfs.Filesystem {
|
||||
factories := map[string]vfs.FsFactory{
|
||||
".torrent": tsrv.NewTorrentFs,
|
||||
".ts-ytdlp": ytdlpsrv.BuildFS,
|
||||
}
|
||||
|
||||
// add default torrent factory for root filesystem
|
||||
for k, v := range vfs.ArchiveFactories {
|
||||
factories[k] = v
|
||||
}
|
||||
|
||||
return vfs.NewResolveFS(sourceFS, factories)
|
||||
}
|
|
@ -12,7 +12,7 @@ import (
|
|||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/anacrolix/torrent"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
|
@ -1,4 +1,4 @@
|
|||
package tkv
|
||||
package torrent
|
||||
|
||||
import (
|
||||
"path"
|
||||
|
@ -8,7 +8,7 @@ import (
|
|||
"go.opentelemetry.io/otel/attribute"
|
||||
)
|
||||
|
||||
func New[K kv.Bytes, V any](dbdir, name string) (store kv.Store[K, V], err error) {
|
||||
func NewKV[K kv.Bytes, V any](dbdir, name string) (store kv.Store[K, V], err error) {
|
||||
dir := path.Join(dbdir, name)
|
||||
store, err = kv.NewBadgerKV[K, V](dir)
|
||||
if err != nil {
|
|
@ -13,11 +13,10 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ctxio"
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/rlog"
|
||||
"git.kmsign.ru/royalcat/tstor/src/config"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/tkv"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/royalcat/ctxio"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
@ -32,7 +31,7 @@ import (
|
|||
"github.com/royalcat/kv"
|
||||
)
|
||||
|
||||
var tracer = otel.Tracer("git.kmsign.ru/royalcat/tstor/host/torrent")
|
||||
var tracer = otel.Tracer("git.kmsign.ru/royalcat/tstor/sources/torrent")
|
||||
|
||||
type DirAquire struct {
|
||||
Name string
|
||||
|
@ -99,7 +98,7 @@ func NewService(sourceFs billy.Filesystem, conf config.TorrentClient) (*Service,
|
|||
}
|
||||
client.AddDhtNodes(conf.DHTNodes)
|
||||
|
||||
s.dirsAquire, err = tkv.New[string, DirAquire](conf.MetadataFolder, "dir-acquire")
|
||||
s.dirsAquire, err = NewKV[string, DirAquire](conf.MetadataFolder, "dir-acquire")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
73
src/sources/ytdlp/controller.go
Normal file
73
src/sources/ytdlp/controller.go
Normal file
|
@ -0,0 +1,73 @@
|
|||
package ytdlp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"sync"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ctxbilly"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/royalcat/ctxio"
|
||||
)
|
||||
|
||||
func NewService(dataDir string) *Service {
|
||||
return &Service{
|
||||
dataDir: dataDir,
|
||||
sources: make(map[string]ytdlpSource, 0),
|
||||
}
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
mu sync.Mutex
|
||||
|
||||
dataDir string
|
||||
sources map[string]ytdlpSource
|
||||
}
|
||||
|
||||
func (c *Service) AddSource(s ytdlpSource) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.sources[s.Name()] = s
|
||||
}
|
||||
|
||||
func (c *Service) sourceDir(s ytdlpSource) string {
|
||||
return path.Join(c.dataDir, s.Name())
|
||||
}
|
||||
|
||||
func (c *Service) Update(ctx context.Context) error {
|
||||
for name, s := range c.sources {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
dir := c.sourceDir(s)
|
||||
err := s.Download(ctx, nil, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch source %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Service) BuildFS(ctx context.Context, f vfs.File) (vfs.Filesystem, error) {
|
||||
data, err := ctxio.ReadAll(ctx, f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read source file: %w", err)
|
||||
}
|
||||
|
||||
var s ytdlpSource
|
||||
err = json.Unmarshal(data, &s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.AddSource(s)
|
||||
|
||||
downloadFS := ctxbilly.WrapFileSystem(osfs.New(c.sourceDir(s)))
|
||||
|
||||
return newSourceFS(s.Name(), downloadFS, c, s), nil
|
||||
}
|
69
src/sources/ytdlp/fs.go
Normal file
69
src/sources/ytdlp/fs.go
Normal file
|
@ -0,0 +1,69 @@
|
|||
package ytdlp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ctxbilly"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
)
|
||||
|
||||
type SourceFS struct {
|
||||
service *Service
|
||||
source ytdlpSource
|
||||
|
||||
fs ctxbilly.Filesystem
|
||||
|
||||
vfs.DefaultFS
|
||||
}
|
||||
|
||||
var _ vfs.Filesystem = (*SourceFS)(nil)
|
||||
|
||||
func newSourceFS(name string, fs ctxbilly.Filesystem, service *Service, source ytdlpSource) *SourceFS {
|
||||
return &SourceFS{
|
||||
fs: fs,
|
||||
service: service,
|
||||
source: source,
|
||||
DefaultFS: vfs.DefaultFS(name),
|
||||
}
|
||||
}
|
||||
|
||||
// Open implements vfs.Filesystem.
|
||||
func (s *SourceFS) Open(ctx context.Context, filename string) (vfs.File, error) {
|
||||
info, err := s.fs.Stat(ctx, filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f, err := s.fs.Open(ctx, filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return vfs.NewCtxBillyFile(info, f), nil
|
||||
}
|
||||
|
||||
// ReadDir implements vfs.Filesystem.
|
||||
func (s *SourceFS) ReadDir(ctx context.Context, path string) ([]fs.DirEntry, error) {
|
||||
infos, err := s.fs.ReadDir(ctx, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entries := make([]fs.DirEntry, 0, len(infos))
|
||||
for _, info := range infos {
|
||||
entries = append(entries, vfs.NewFileInfo(info.Name(), info.Size()))
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// Stat implements vfs.Filesystem.
|
||||
func (s *SourceFS) Stat(ctx context.Context, filename string) (fs.FileInfo, error) {
|
||||
return s.fs.Stat(ctx, filename)
|
||||
}
|
||||
|
||||
// Unlink implements vfs.Filesystem.
|
||||
func (s *SourceFS) Unlink(ctx context.Context, filename string) error {
|
||||
return vfs.ErrNotImplemented
|
||||
}
|
7
src/sources/ytdlp/task.go
Normal file
7
src/sources/ytdlp/task.go
Normal file
|
@ -0,0 +1,7 @@
|
|||
package ytdlp
|
||||
|
||||
import "io"
|
||||
|
||||
type TaskUpdater interface {
|
||||
Output() io.Writer
|
||||
}
|
43
src/sources/ytdlp/ytdlp.go
Normal file
43
src/sources/ytdlp/ytdlp.go
Normal file
|
@ -0,0 +1,43 @@
|
|||
package ytdlp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ytdlp"
|
||||
"github.com/royalcat/ctxprogress"
|
||||
)
|
||||
|
||||
type ytdlpSource struct {
|
||||
Url string `json:"url"`
|
||||
}
|
||||
|
||||
var hasher = sha1.New()
|
||||
|
||||
func (s *ytdlpSource) Name() string {
|
||||
return string(hasher.Sum([]byte(s.Url)))
|
||||
}
|
||||
|
||||
func (s *ytdlpSource) Download(ctx context.Context, task TaskUpdater, dir string) error {
|
||||
client, err := ytdlp.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctxprogress.New(ctx)
|
||||
ctxprogress.Set(ctx, ctxprogress.RangeProgress{Current: 0, Total: 2})
|
||||
plst, err := client.Playlist(ctx, s.Url)
|
||||
ctxprogress.Set(ctx, ctxprogress.RangeProgress{Current: 1, Total: 2})
|
||||
ctxprogress.Range(ctx, plst, func(ctx context.Context, _ int, e ytdlp.PlaylistEntry) bool {
|
||||
err = client.Download(ctx, e.Url(), dir)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
ctxprogress.Set(ctx, ctxprogress.RangeProgress{Current: 2, Total: 2})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
|
@ -11,9 +11,10 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ctxio"
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ioutils"
|
||||
"github.com/bodgit/sevenzip"
|
||||
"github.com/nwaples/rardecode/v2"
|
||||
"github.com/royalcat/ctxio"
|
||||
)
|
||||
|
||||
var ArchiveFactories = map[string]FsFactory{
|
||||
|
@ -138,7 +139,7 @@ func (afs *ArchiveFS) Stat(ctx context.Context, filename string) (fs.FileInfo, e
|
|||
|
||||
for p, _ := range afs.files {
|
||||
if strings.HasPrefix(p, filename) {
|
||||
return newDirInfo(path.Base(filename)), nil
|
||||
return NewDirInfo(path.Base(filename)), nil
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -173,7 +174,7 @@ func NewArchiveFile(name string, size int64, af archiveFileReaderFactory) *archi
|
|||
size: size,
|
||||
af: af,
|
||||
|
||||
buffer: ctxio.NewFileBuffer(nil),
|
||||
buffer: ioutils.NewFileBuffer(nil),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -188,7 +189,7 @@ type archiveFile struct {
|
|||
|
||||
offset int64
|
||||
readen int64
|
||||
buffer *ctxio.FileBuffer
|
||||
buffer *ioutils.FileBuffer
|
||||
}
|
||||
|
||||
// Name implements File.
|
||||
|
@ -350,7 +351,7 @@ func SevenZipLoader(ctx context.Context, ctxreader ctxio.ReaderAt, size int64) (
|
|||
var _ archiveLoader = RarLoader
|
||||
|
||||
func RarLoader(ctx context.Context, ctxreader ctxio.ReaderAt, size int64) (map[string]*archiveFile, error) {
|
||||
reader := ctxio.WrapIoReadSeeker(ctx, ctxreader, size)
|
||||
reader := ioutils.WrapIoReadSeeker(ctx, ctxreader, size)
|
||||
|
||||
r, err := rardecode.NewReader(reader)
|
||||
if err != nil {
|
||||
|
@ -369,7 +370,7 @@ func RarLoader(ctx context.Context, ctxreader ctxio.ReaderAt, size int64) (map[s
|
|||
|
||||
name := header.Name
|
||||
af := func(ctx context.Context) (io.ReadCloser, error) {
|
||||
reader := ctxio.WrapIoReadSeeker(ctx, ctxreader, size)
|
||||
reader := ioutils.WrapIoReadSeeker(ctx, ctxreader, size)
|
||||
r, err := rardecode.NewReader(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
|
@ -7,8 +7,8 @@ import (
|
|||
"io"
|
||||
"testing"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ctxio"
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/royalcat/ctxio"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -46,10 +46,7 @@ func (c *CtxBillyFs) Open(ctx context.Context, filename string) (File, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CtxBillyFile{
|
||||
info: info,
|
||||
file: bf,
|
||||
}, nil
|
||||
return NewCtxBillyFile(info, bf), nil
|
||||
}
|
||||
|
||||
// ReadDir implements Filesystem.
|
||||
|
@ -98,6 +95,13 @@ func (c *CtxBillyFs) Unlink(ctx context.Context, filename string) error {
|
|||
return fs.ErrInvalid
|
||||
}
|
||||
|
||||
func NewCtxBillyFile(info fs.FileInfo, bf ctxbilly.File) *CtxBillyFile {
|
||||
return &CtxBillyFile{
|
||||
info: info,
|
||||
file: bf,
|
||||
}
|
||||
}
|
||||
|
||||
var _ File = (*CtxBillyFile)(nil)
|
||||
|
||||
type CtxBillyFile struct {
|
27
src/vfs/default.go
Normal file
27
src/vfs/default.go
Normal file
|
@ -0,0 +1,27 @@
|
|||
package vfs
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
type DefaultFS string
|
||||
|
||||
// Info implements Filesystem.
|
||||
func (d DefaultFS) Info() (fs.FileInfo, error) {
|
||||
return NewDirInfo(string(d)), nil
|
||||
}
|
||||
|
||||
// IsDir implements Filesystem.
|
||||
func (d DefaultFS) IsDir() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Name implements Filesystem.
|
||||
func (d DefaultFS) Name() string {
|
||||
return string(d)
|
||||
}
|
||||
|
||||
// Type implements Filesystem.
|
||||
func (d *DefaultFS) Type() fs.FileMode {
|
||||
return fs.ModeDir
|
||||
}
|
|
@ -25,7 +25,7 @@ func (d *dirFile) Close(ctx context.Context) error {
|
|||
|
||||
// Info implements File.
|
||||
func (d *dirFile) Info() (fs.FileInfo, error) {
|
||||
return newDirInfo(d.name), nil
|
||||
return NewDirInfo(d.name), nil
|
||||
}
|
||||
|
||||
// IsDir implements File.
|
|
@ -24,16 +24,6 @@ func (d *DummyFs) Mode() fs.FileMode {
|
|||
return fs.ModeDir
|
||||
}
|
||||
|
||||
// Size implements Filesystem.
|
||||
func (d *DummyFs) Size() int64 {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// Sys implements Filesystem.
|
||||
func (d *DummyFs) Sys() any {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// FsName implements Filesystem.
|
||||
func (d *DummyFs) FsName() string {
|
||||
return "dummyfs"
|
||||
|
@ -65,7 +55,7 @@ func (d *DummyFs) ReadDir(ctx context.Context, path string) ([]fs.DirEntry, erro
|
|||
|
||||
// Info implements Filesystem.
|
||||
func (d *DummyFs) Info() (fs.FileInfo, error) {
|
||||
return newDirInfo(d.name), nil
|
||||
return NewDirInfo(d.name), nil
|
||||
}
|
||||
|
||||
// IsDir implements Filesystem.
|
|
@ -7,7 +7,7 @@ import (
|
|||
"path"
|
||||
"time"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/pkg/ctxio"
|
||||
"github.com/royalcat/ctxio"
|
||||
"go.opentelemetry.io/otel"
|
||||
)
|
||||
|
||||
|
@ -24,7 +24,7 @@ type File interface {
|
|||
|
||||
var ErrNotImplemented = errors.New("not implemented")
|
||||
|
||||
var tracer = otel.Tracer("git.kmsign.ru/royalcat/tstor/src/host/vfs")
|
||||
var tracer = otel.Tracer("git.kmsign.ru/royalcat/tstor/src/vfs")
|
||||
|
||||
type Filesystem interface {
|
||||
// Open opens the named file for reading. If successful, methods on the
|
||||
|
@ -55,7 +55,7 @@ type fileInfo struct {
|
|||
var _ fs.FileInfo = &fileInfo{}
|
||||
var _ fs.DirEntry = &fileInfo{}
|
||||
|
||||
func newDirInfo(name string) *fileInfo {
|
||||
func NewDirInfo(name string) *fileInfo {
|
||||
return &fileInfo{
|
||||
name: path.Base(name),
|
||||
size: 0,
|
|
@ -29,7 +29,7 @@ func TestDirInfo(t *testing.T) {
|
|||
|
||||
require := require.New(t)
|
||||
|
||||
fi := newDirInfo("abc/name")
|
||||
fi := NewDirInfo("abc/name")
|
||||
|
||||
require.True(fi.IsDir())
|
||||
require.Equal("name", fi.Name())
|
|
@ -42,7 +42,7 @@ func (fs *MemoryFs) FsName() string {
|
|||
|
||||
// Info implements Filesystem.
|
||||
func (fs *MemoryFs) Info() (fs.FileInfo, error) {
|
||||
return newDirInfo(fs.name), nil
|
||||
return NewDirInfo(fs.name), nil
|
||||
}
|
||||
|
||||
// IsDir implements Filesystem.
|
|
@ -17,7 +17,7 @@ var _ Filesystem = (*OsFS)(nil)
|
|||
// Stat implements Filesystem.
|
||||
func (fs *OsFS) Stat(ctx context.Context, filename string) (fs.FileInfo, error) {
|
||||
if path.Clean(filename) == Separator {
|
||||
return newDirInfo(Separator), nil
|
||||
return NewDirInfo(Separator), nil
|
||||
}
|
||||
|
||||
info, err := os.Stat(path.Join(fs.hostDir, filename))
|
||||
|
@ -48,7 +48,7 @@ func (o *OsFS) ReadDir(ctx context.Context, dir string) ([]fs.DirEntry, error) {
|
|||
|
||||
// Info implements Filesystem.
|
||||
func (fs *OsFS) Info() (fs.FileInfo, error) {
|
||||
return newDirInfo(fs.Name()), nil
|
||||
return NewDirInfo(fs.Name()), nil
|
||||
}
|
||||
|
||||
// IsDir implements Filesystem.
|
|
@ -5,7 +5,7 @@ import (
|
|||
"os"
|
||||
"testing"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -349,7 +349,7 @@ func ListDirFromFiles[F File](m map[string]F, name string) ([]fs.DirEntry, error
|
|||
if len(parts) == 1 {
|
||||
out = append(out, NewFileInfo(parts[0], f.Size()))
|
||||
} else {
|
||||
out = append(out, newDirInfo(parts[0]))
|
||||
out = append(out, NewDirInfo(parts[0]))
|
||||
}
|
||||
|
||||
}
|
|
@ -6,7 +6,7 @@ import (
|
|||
"context"
|
||||
"testing"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
|
||||
"git.kmsign.ru/royalcat/tstor/src/vfs"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
Loading…
Add table
Add a link
Reference in a new issue