royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages1 Wiki Activity Actions

docker deploy

Browse source
This commit is contained in:
royalcat 2023-10-18 12:52:48 +03:00
parent d30ef6cc9c
commit ec83e3b08b
16 changed files with 236 additions and 335 deletions

4
src/host/torrent/service.go
View file

@ -47,6 +47,10 @@ func (s *Service) NewTorrentFs(f vfs.File) (vfs.Filesystem, error) {
}
<-t.GotInfo()
t.AllowDataDownload()
for _, f := range t.Files() {
f.SetPriority(torrent.PiecePriorityReadahead)
}
return vfs.NewTorrentFs(t, s.readTimeout), nil
}

61
src/host/vfs/resolver.go
View file

@ -2,6 +2,7 @@ package vfs
import (
"fmt"
"io/fs"
"strings"
"sync"
)
@ -88,11 +89,11 @@ PARTS_LOOP:
}
if nestOn == -1 {
return name, nil, "", nil
return clean(name), nil, "", nil
}
fsPath = Clean(strings.Join(parts[:nestOn], Separator))
nestedFsPath = Clean(strings.Join(parts[nestOn:], Separator))
fsPath = clean(strings.Join(parts[:nestOn], Separator))
nestedFsPath = clean(strings.Join(parts[nestOn:], Separator))
// we dont need lock until now
// it must be before fsmap read to exclude race condition:
@ -119,28 +120,42 @@ PARTS_LOOP:
}
// func (r *resolver) resolveFile(name string, fs Filesystem) (File, error) {
// fsPath, nestedFs, nestedFsPath, err := r.resolvePath(name, fs)
// if err != nil {
// return nil, err
// }
var ErrNotExist = fs.ErrNotExist
// if nestedFs == nil {
// return fs.Open(fsPath)
// }
func getFile[F File](m map[string]F, name string) (File, error) {
name = clean(name)
if name == Separator {
return &Dir{}, nil
}
// return nestedFs.Open(nestedFsPath)
// }
f, ok := m[name]
if ok {
return f, nil
}
// func (r *resolver) resolveDir(name string, fs Filesystem) (map[string]File, error) {
// fsPath, nestedFs, nestedFsPath, err := r.resolvePath(name, fs)
// if err != nil {
// return nil, err
// }
for p := range m {
if strings.HasPrefix(p, name) {
return &Dir{}, nil
}
}
// if nestedFs == nil {
// return fs.ReadDir(fsPath)
// }
return nil, ErrNotExist
}
// return nestedFs.ReadDir(nestedFsPath)
// }
func listFilesInDir[F File](m map[string]F, name string) (map[string]File, error) {
name = clean(name)
out := map[string]File{}
for p, f := range m {
if strings.HasPrefix(p, name) {
parts := strings.Split(trimRelPath(p, name), Separator)
if len(parts) == 1 {
out[parts[0]] = f
} else {
out[parts[0]] = &Dir{}
}
}
}
return out, nil
}

232
src/host/vfs/resolver_test.go
View file

@ -2,6 +2,9 @@ package vfs
import (
"os"
"testing"
"github.com/stretchr/testify/require"
)
type Dummy struct {
@ -49,144 +52,121 @@ func (d *DummyFs) ReadDir(path string) (map[string]File, error) {
var _ Filesystem = &DummyFs{}
// func TestDefaultFactories(t *testing.T) {
// t.Parallel()
func TestResolver(t *testing.T) {
t.Parallel()
resolver := newResolver(ArchiveFactories)
t.Run("nested fs", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// require := require.New(t)
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("/f1.rar/f2.rar", func(path string) (File, error) {
require.Equal("/f1.rar", path)
return &Dummy{}, nil
})
require.Nil(err)
require.Equal("/f1.rar", fsPath)
require.Equal("/f2.rar", nestedFsPath)
require.IsType(&archive{}, nestedFs)
})
t.Run("root", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// require.Contains(defaultFactories, ".zip")
// require.Contains(defaultFactories, ".rar")
// require.Contains(defaultFactories, ".7z")
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("/", func(path string) (File, error) {
require.Equal("/", path)
return &Dummy{}, nil
})
require.Nil(err)
require.Nil(nestedFs)
require.Equal("/", fsPath)
require.Equal("", nestedFsPath)
})
// fs, err := defaultFactories[".zip"](&Dummy{}, nil)
// require.NoError(err)
// require.NotNil(fs)
t.Run("root dirty", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// fs, err = defaultFactories[".rar"](&Dummy{}, nil)
// require.NoError(err)
// require.NotNil(fs)
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("//.//", func(path string) (File, error) {
require.Equal("/", path)
return &Dummy{}, nil
})
require.Nil(err)
require.Nil(nestedFs)
require.Equal("/", fsPath)
require.Equal("", nestedFsPath)
})
t.Run("fs dirty", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// fs, err = defaultFactories[".7z"](&Dummy{}, nil)
// require.NoError(err)
// require.NotNil(fs)
// }
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("//.//f1.rar", func(path string) (File, error) {
require.Equal("/f1.rar", path)
return &Dummy{}, nil
})
require.Nil(err)
require.Equal("/f1.rar", fsPath)
require.Equal("/", nestedFsPath)
require.IsType(&archive{}, nestedFs)
})
t.Run("inside folder", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// func TestStorageAddFs(t *testing.T) {
// t.Parallel()
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("//test1/f1.rar", func(path string) (File, error) {
require.Equal("/test1/f1.rar", path)
return &Dummy{}, nil
})
require.Nil(err)
require.IsType(&archive{}, nestedFs)
require.Equal("/test1/f1.rar", fsPath)
require.Equal("/", nestedFsPath)
})
}
// require := require.New(t)
func TestArchiveFactories(t *testing.T) {
t.Parallel()
// s := newStorage(dummyFactories)
require := require.New(t)
// err := s.AddFS(&DummyFs{}, "/test")
// require.NoError(err)
require.Contains(ArchiveFactories, ".zip")
require.Contains(ArchiveFactories, ".rar")
require.Contains(ArchiveFactories, ".7z")
// f, err := s.Get("/test/dir/here/file1.txt")
// require.NoError(err)
// require.NotNil(f)
fs, err := ArchiveFactories[".zip"](&Dummy{})
require.NoError(err)
require.NotNil(fs)
// err = s.AddFS(&DummyFs{}, "/test")
// require.Error(err)
// }
fs, err = ArchiveFactories[".rar"](&Dummy{})
require.NoError(err)
require.NotNil(fs)
// func TestStorageWindowsPath(t *testing.T) {
// t.Parallel()
fs, err = ArchiveFactories[".7z"](&Dummy{})
require.NoError(err)
require.NotNil(fs)
}
// require := require.New(t)
func TestFiles(t *testing.T) {
t.Parallel()
require := require.New(t)
// s := newStorage(dummyFactories)
files := map[string]*Dummy{
"/test/file.txt": &Dummy{},
}
{
file, err := getFile(files, "/test")
require.Nil(err)
require.Equal(&Dir{}, file)
}
{
file, err := getFile(files, "/test/file.txt")
require.Nil(err)
require.Equal(&Dummy{}, file)
}
// err := s.Add(&Dummy{}, "\\path\\to\\dummy\\file.txt")
// require.NoError(err)
// file, err := s.Get("\\path\\to\\dummy\\file.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// file, err = s.Get("/path/to/dummy/file.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// }
// var dummyFactories = map[string]vfs.FsFactory{
// ".test": func(f vfs.File, factories map[string]vfs.FsFactory) (vfs.Filesystem, error) {
// return &DummyFs{}, nil
// },
// }
// func TestStorage(t *testing.T) {
// t.Parallel()
// require := require.New(t)
// s := newStorage(dummyFactories)
// err := s.Add(&Dummy{}, "/path/to/dummy/file.txt")
// require.NoError(err)
// err = s.Add(&Dummy{}, "/path/to/dummy/file2.txt")
// require.NoError(err)
// contains := s.Has("/path")
// require.True(contains)
// contains = s.Has("/path/to/dummy/")
// require.True(contains)
// file, err := s.Get("/path/to/dummy/file.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// file, err = s.Get("/path/to/dummy/file3.txt")
// require.Error(err)
// require.Nil(file)
// files, err := s.Children("/path/to/dummy/")
// require.NoError(err)
// require.Len(files, 2)
// require.Contains(files, "file.txt")
// require.Contains(files, "file2.txt")
// err = s.Add(&Dummy{}, "/path/to/dummy/folder/file.txt")
// require.NoError(err)
// files, err = s.Children("/path/to/dummy/")
// require.NoError(err)
// require.Len(files, 3)
// require.Contains(files, "file.txt")
// require.Contains(files, "file2.txt")
// require.Contains(files, "folder")
// err = s.Add(&Dummy{}, "path/file4.txt")
// require.NoError(err)
// require.True(s.Has("/path/file4.txt"))
// files, err = s.Children("/")
// require.NoError(err)
// require.Len(files, 1)
// err = s.Add(&Dummy{}, "/path/special_file.test")
// require.NoError(err)
// file, err = s.Get("/path/special_file.test/dir/here/file1.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// files, err = s.Children("/path/special_file.test")
// require.NoError(err)
// require.NotNil(files)
// files, err = s.Children("/path/special_file.test/dir/here")
// require.NoError(err)
// require.Len(files, 2)
// err = s.Add(&Dummy{}, "/path/to/__special__path/file3.txt")
// require.NoError(err)
// file, err = s.Get("/path/to/__special__path/file3.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// s.Clear()
// }
{
out, err := listFilesInDir(files, "/test")
require.Nil(err)
require.Contains(out, "file.txt")
require.Equal(&Dummy{}, out["file.txt"])
}
}

34
src/host/vfs/torrent.go
View file

@ -33,7 +33,11 @@ func (fs *TorrentFs) files() map[string]*torrentFile {
files := make(map[string]*torrentFile)
<-fs.t.GotInfo()
for _, file := range fs.t.Files() {
p := Clean(file.Path())
if file.Priority() == torrent.PiecePriorityNone {
continue
}
p := clean(file.Path())
files[p] = &torrentFile{
readerFunc: file.NewReader,
len: file.Length(),
@ -73,6 +77,12 @@ func (fs *TorrentFs) ReadDir(name string) (map[string]File, error) {
return listFilesInDir(fs.files(), fsPath)
}
func (fs *TorrentFs) Unlink(name string) error {
file := fs.t.Files()[0]
file.SetPriority(torrent.PiecePriorityNone)
return nil
}
type reader interface {
iio.Reader
missinggo.ReadContexter
@ -111,18 +121,11 @@ func readAtLeast(r missinggo.ReadContexter, timeout int, buf []byte, min int) (n
for n < min && err == nil {
var nn int
ctx, cancel := context.WithCancel(context.Background())
timer := time.AfterFunc(
time.Duration(timeout)*time.Second,
func() {
cancel()
},
)
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
defer cancel()
nn, err = r.ReadContext(ctx, buf[n:])
n += nn
timer.Stop()
}
if n >= min {
err = nil
@ -175,15 +178,8 @@ func (d *torrentFile) Close() error {
func (d *torrentFile) Read(p []byte) (n int, err error) {
d.load()
ctx, cancel := context.WithCancel(context.Background())
timer := time.AfterFunc(
time.Duration(d.timeout)*time.Second,
func() {
cancel()
},
)
defer timer.Stop()
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(d.timeout)*time.Second)
defer cancel()
return d.reader.ReadContext(ctx, p)
}

2
src/host/vfs/torrent_test.go
View file

@ -85,6 +85,8 @@ func TestMain(m *testing.M) {
// }
func TestReadAtTorrent(t *testing.T) {
t.Parallel()
require := require.New(t)
to, err := Cli.AddMagnet(testMagnet)

43
src/host/vfs/utils.go
View file

@ -1,55 +1,14 @@
package vfs
import (
"io/fs"
"path"
"strings"
)
var ErrNotExist = fs.ErrNotExist
func getFile[F File](m map[string]F, name string) (File, error) {
name = Clean(name)
if name == Separator {
return &Dir{}, nil
}
f, ok := m[name]
if ok {
return f, nil
}
for p := range m {
if strings.HasPrefix(p, name) {
return &Dir{}, nil
}
}
return nil, ErrNotExist
}
func listFilesInDir[F File](m map[string]F, name string) (map[string]File, error) {
name = Clean(name)
out := map[string]File{}
for p, f := range m {
if strings.HasPrefix(p, name) {
parts := strings.Split(trimRelPath(p, name), Separator)
if len(parts) == 1 {
out[parts[0]] = f
} else {
out[parts[0]] = &Dir{}
}
}
}
return out, nil
}
func trimRelPath(p, t string) string {
return strings.Trim(strings.TrimPrefix(p, t), "/")
}
func Clean(p string) string {
func clean(p string) string {
return path.Clean(Separator + strings.ReplaceAll(p, "\\", "/"))
}

4
src/mounts/fuse/handler.go
View file

@ -1,3 +1,5 @@
//go:build cgo
package fuse
import (
@ -38,7 +40,7 @@ func (s *Handler) Mount(vfs vfs.Filesystem) error {
}
}
host := fuse.NewFileSystemHost(NewFS(vfs))
host := fuse.NewFileSystemHost(newFuseFS(vfs))
// TODO improve error handling here
go func() {

23
src/mounts/fuse/handler_nocgo.go Normal file
View file

@ -0,0 +1,23 @@
//go:build !cgo
package fuse
import (
"fmt"
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
)
type Handler struct{}
func NewHandler(fuseAllowOther bool, path string) *Handler {
return &Handler{}
}
func (s *Handler) Mount(vfs vfs.Filesystem) error {
return fmt.Errorf("tstor was build without fuse support")
}
func (s *Handler) Unmount() {
}

24
src/mounts/fuse/mount.go
View file

@ -1,3 +1,5 @@
//go:build cgo
package fuse
import (
@ -14,22 +16,22 @@ import (
"github.com/rs/zerolog/log"
)
type FS struct {
type fuseFS struct {
fuse.FileSystemBase
fh *fileHandler
log zerolog.Logger
}
func NewFS(fs vfs.Filesystem) fuse.FileSystemInterface {
func newFuseFS(fs vfs.Filesystem) fuse.FileSystemInterface {
l := log.Logger.With().Str("component", "fuse").Logger()
return &FS{
return &fuseFS{
fh: &fileHandler{fs: fs},
log: l,
}
}
func (fs *FS) Open(path string, flags int) (errc int, fh uint64) {
func (fs *fuseFS) Open(path string, flags int) (errc int, fh uint64) {
fh, err := fs.fh.OpenHolder(path)
if os.IsNotExist(err) {
fs.log.Debug().Str("path", path).Msg("file does not exists")
@ -46,15 +48,15 @@ func (fs *FS) Open(path string, flags int) (errc int, fh uint64) {
// Unlink removes a file.
// The FileSystemBase implementation returns -ENOSYS.
func (fs *FS) Unlink(path string) int {
func (fs *fuseFS) Unlink(path string) int {
return -fuse.ENOSYS
}
func (fs *FS) Opendir(path string) (errc int, fh uint64) {
func (fs *fuseFS) Opendir(path string) (errc int, fh uint64) {
return fs.Open(path, 0)
}
func (fs *FS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
func (fs *fuseFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
if path == "/" {
stat.Mode = fuse.S_IFDIR | 0555
return 0
@ -81,7 +83,7 @@ func (fs *FS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
return 0
}
func (fs *FS) Read(path string, dest []byte, off int64, fh uint64) int {
func (fs *fuseFS) Read(path string, dest []byte, off int64, fh uint64) int {
file, err := fs.fh.GetFile(path, fh)
if os.IsNotExist(err) {
fs.log.Error().Err(err).Str("path", path).Msg("file not found on READ operation")
@ -110,7 +112,7 @@ func (fs *FS) Read(path string, dest []byte, off int64, fh uint64) int {
return n
}
func (fs *FS) Release(path string, fh uint64) int {
func (fs *fuseFS) Release(path string, fh uint64) int {
if err := fs.fh.Remove(fh); err != nil {
fs.log.Error().Err(err).Str("path", path).Msg("error getting holder when releasing file")
return -fuse.EIO
@ -119,11 +121,11 @@ func (fs *FS) Release(path string, fh uint64) int {
return 0
}
func (fs *FS) Releasedir(path string, fh uint64) int {
func (fs *fuseFS) Releasedir(path string, fh uint64) int {
return fs.Release(path, fh)
}
func (fs *FS) Readdir(path string,
func (fs *fuseFS) Readdir(path string,
fill func(name string, stat *fuse.Stat_t, ofst int64) bool,
ofst int64,
fh uint64) (errc int) {

2
src/mounts/fuse/mount_test.go
View file

@ -1,3 +1,5 @@
//go:build cgo
package fuse
import (