royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

rework 2 (working)
Some checks failed
mkdocs / mkdocs (push) Failing after 2s
Details
CodeQL / Analyze (go) (push) Failing after 2s
Details
CodeQL / Analyze (javascript) (push) Failing after 2s
Details
docker / build-docker (inux/arm/v6) (push) Failing after 14s
Details
docker / build-docker (linux/386) (push) Failing after 13s
Details
docker / build-docker (linux/amd64) (push) Has started running
Details
docker / build-docker (linux/arm/v7) (push) Failing after 14s
Details
docker / build-docker (linux/arm64) (push) Has been cancelled
Details

Browse source
This commit is contained in:
royalcat 2023-10-16 12:18:40 +03:00
parent 5a77fa5e9c
commit d30ef6cc9c
48 changed files with 1340 additions and 1747 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

114
cmd/tstor/main.go
View file

@ -11,18 +11,18 @@ import (
"time" "time"
"git.kmsign.ru/royalcat/tstor/src/config" "git.kmsign.ru/royalcat/tstor/src/config"
"git.kmsign.ru/royalcat/tstor/src/host"
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
"github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/storage"
"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
"git.kmsign.ru/royalcat/tstor/src/fs"
"git.kmsign.ru/royalcat/tstor/src/http" "git.kmsign.ru/royalcat/tstor/src/http"
dlog "git.kmsign.ru/royalcat/tstor/src/log" dlog "git.kmsign.ru/royalcat/tstor/src/log"
"git.kmsign.ru/royalcat/tstor/src/mounts/fuse" "git.kmsign.ru/royalcat/tstor/src/mounts/fuse"
"git.kmsign.ru/royalcat/tstor/src/mounts/httpfs" "git.kmsign.ru/royalcat/tstor/src/mounts/httpfs"
"git.kmsign.ru/royalcat/tstor/src/mounts/webdav" "git.kmsign.ru/royalcat/tstor/src/mounts/webdav"
"git.kmsign.ru/royalcat/tstor/src/torrent"
"git.kmsign.ru/royalcat/tstor/src/torrent/loader"
) )
const ( const (
@ -75,8 +75,8 @@ func setupStorage(tcfg config.TorrentClient) (storage.ClientImplCloser, storage.
return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err) return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err)
} }
// TODO implement cache dir and storage capacity // TODO implement cache/storage switching
// cacheDir := filepath.Join(tcfg.MetadataFolder, "cache") // cacheDir := filepath.Join(tcfg.DataFolder, "cache")
// if err := os.MkdirAll(cacheDir, 0744); err != nil { // if err := os.MkdirAll(cacheDir, 0744); err != nil {
// return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err) // return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err)
// } // }
@ -84,8 +84,11 @@ func setupStorage(tcfg config.TorrentClient) (storage.ClientImplCloser, storage.
// if err != nil { // if err != nil {
// return nil, nil, fmt.Errorf("error creating cache: %w", err) // return nil, nil, fmt.Errorf("error creating cache: %w", err)
// } // }
// log.Info().Msg(fmt.Sprintf("setting cache size to %d MB", tcfg.GlobalCacheSize)) // log.Info().Msg(fmt.Sprintf("setting cache size to %d MB", 1024))
// fc.SetCapacity(tcfg.GlobalCacheSize * 1024 * 1024) // fc.SetCapacity(1024 * 1024 * 1024)
// rp := storage.NewResourcePieces(fc.AsResourceProvider())
// st := &stc{rp}
filesDir := filepath.Join(tcfg.DataFolder, "files") filesDir := filepath.Join(tcfg.DataFolder, "files")
if err := os.MkdirAll(pcp, 0744); err != nil { if err := os.MkdirAll(pcp, 0744); err != nil {
@ -97,6 +100,14 @@ func setupStorage(tcfg config.TorrentClient) (storage.ClientImplCloser, storage.
return st, pc, nil return st, pc, nil
} }
type stc struct {
storage.ClientImpl
}
func (s *stc) Close() error {
return nil
}
func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error { func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
conf, err := config.Load(configPath) conf, err := config.Load(configPath)
if err != nil { if err != nil {
@ -119,7 +130,7 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
return fmt.Errorf("error creating node ID: %w", err) return fmt.Errorf("error creating node ID: %w", err)
} }
st, pc, err := setupStorage(conf.TorrentClient) st, _, err := setupStorage(conf.TorrentClient)
if err != nil { if err != nil {
return err return err
} }
@ -128,26 +139,14 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
if err != nil { if err != nil {
return fmt.Errorf("error starting torrent client: %w", err) return fmt.Errorf("error starting torrent client: %w", err)
} }
c.AddDhtNodes(conf.TorrentClient.DHTNodes)
var servers []*torrent.Server ts := torrent.NewService(c, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout)
for _, s := range conf.TorrentClient.Servers {
server := torrent.NewServer(c, pc, &s) if err := os.MkdirAll(conf.DataFolder, 0744); err != nil {
servers = append(servers, server) return fmt.Errorf("error creating data folder: %w", err)
if err := server.Start(); err != nil {
return fmt.Errorf("error starting server: %w", err)
}
} }
cfs := host.NewStorage(conf.DataFolder, ts)
cl := loader.NewConfig(conf.TorrentClient.Routes)
fl := loader.NewFolder(conf.TorrentClient.Routes)
ss := torrent.NewStats()
dbl, err := loader.NewDB(filepath.Join(conf.TorrentClient.MetadataFolder, "magnetdb"))
if err != nil {
return fmt.Errorf("error starting magnet database: %w", err)
}
ts := torrent.NewService([]loader.Loader{cl, fl}, dbl, ss, c, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout)
var mh *fuse.Handler var mh *fuse.Handler
if conf.Mounts.Fuse.Enabled { if conf.Mounts.Fuse.Enabled {
@ -161,15 +160,13 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
<-sigChan <-sigChan
log.Info().Msg("closing servers...") log.Info().Msg("closing servers...")
for _, s := range servers { // for _, s := range servers {
if err := s.Close(); err != nil { // if err := s.Close(); err != nil {
log.Warn().Err(err).Msg("problem closing server") // log.Warn().Err(err).Msg("problem closing server")
} // }
} // }
log.Info().Msg("closing items database...") log.Info().Msg("closing items database...")
fis.Close() fis.Close()
log.Info().Msg("closing magnet database...")
dbl.Close()
log.Info().Msg("closing torrent client...") log.Info().Msg("closing torrent client...")
c.Close() c.Close()
if mh != nil { if mh != nil {
@ -181,51 +178,46 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
os.Exit(1) os.Exit(1)
}() }()
fss, err := ts.Load()
if err != nil {
return fmt.Errorf("error when loading torrents: %w", err)
}
go func() { go func() {
if mh == nil { if mh == nil {
return return
} }
if err := mh.Mount(fss); err != nil { if err := mh.Mount(cfs); err != nil {
log.Info().Err(err).Msg("error mounting filesystems") log.Info().Err(err).Msg("error mounting filesystems")
} }
}() }()
go func() { if conf.Mounts.WebDAV.Enabled {
if conf.Mounts.WebDAV.Enabled { go func() {
port = webDAVPort if err := webdav.NewWebDAVServer(cfs, conf.Mounts.WebDAV.Port, conf.Mounts.WebDAV.User, conf.Mounts.WebDAV.Pass); err != nil {
if port == 0 {
port = conf.Mounts.WebDAV.Port
}
cfs, err := fs.NewContainerFs(fss)
if err != nil {
log.Error().Err(err).Msg("error adding files to webDAV")
return
}
if err := webdav.NewWebDAVServer(cfs, port, conf.Mounts.WebDAV.User, conf.Mounts.WebDAV.Pass); err != nil {
log.Error().Err(err).Msg("error starting webDAV") log.Error().Err(err).Msg("error starting webDAV")
} }
}
log.Warn().Msg("webDAV configuration not found!") log.Warn().Msg("webDAV configuration not found!")
}() }()
}
if conf.Mounts.HttpFs.Enabled {
go func() {
httpfs := httpfs.NewHTTPFS(cfs)
cfs, err := fs.NewContainerFs(fss) r := gin.New()
if err != nil {
return fmt.Errorf("error when loading torrents: %w", err) r.GET("*filepath", func(c *gin.Context) {
path := c.Param("filepath")
c.FileFromFS(path, httpfs)
})
log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port)).Msg("starting HTTPFS")
if err := r.Run(fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port)); err != nil {
log.Error().Err(err).Msg("error starting HTTPFS")
}
}()
} }
httpfs := httpfs.NewHTTPFS(cfs)
logFilename := filepath.Join(conf.Log.Path, dlog.FileName) logFilename := filepath.Join(conf.Log.Path, dlog.FileName)
err = http.New(nil, ss, ts, conf, servers, httpfs, logFilename, conf) err = http.New(nil, nil, ts, logFilename, conf)
log.Error().Err(err).Msg("error initializing HTTP server") log.Error().Err(err).Msg("error initializing HTTP server")
return err return err
} }

4
go.mod
View file

@ -9,9 +9,7 @@ require (
github.com/anacrolix/torrent v1.52.6-0.20230929044811-45c91b322ad1 github.com/anacrolix/torrent v1.52.6-0.20230929044811-45c91b322ad1
github.com/billziss-gh/cgofuse v1.5.0 github.com/billziss-gh/cgofuse v1.5.0
github.com/bodgit/sevenzip v1.4.3 github.com/bodgit/sevenzip v1.4.3
github.com/dgraph-io/badger/v3 v3.2103.5
github.com/dgraph-io/badger/v4 v4.2.0 github.com/dgraph-io/badger/v4 v4.2.0
github.com/fsnotify/fsnotify v1.6.0
github.com/gin-gonic/gin v1.9.1 github.com/gin-gonic/gin v1.9.1
github.com/knadh/koanf/parsers/yaml v0.1.0 github.com/knadh/koanf/parsers/yaml v0.1.0
github.com/knadh/koanf/providers/env v0.1.0 github.com/knadh/koanf/providers/env v0.1.0
@ -52,7 +50,6 @@ require (
github.com/bodgit/windows v1.0.1 // indirect github.com/bodgit/windows v1.0.1 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
github.com/bytedance/sonic v1.9.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
@ -61,6 +58,7 @@ require (
github.com/dustin/go-humanize v1.0.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/fatih/structs v1.1.0 // indirect github.com/fatih/structs v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect

33
go.sum
View file

@ -21,8 +21,6 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG
filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI=
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
@ -100,7 +98,6 @@ github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNa
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
@ -128,8 +125,6 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@ -140,19 +135,13 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg=
github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw=
github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs=
github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak=
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
@ -262,7 +251,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@ -290,7 +278,6 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@ -301,7 +288,6 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq
github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@ -313,7 +299,6 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@ -344,7 +329,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@ -353,8 +337,6 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
@ -384,7 +366,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
@ -472,7 +453,6 @@ github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
@ -487,15 +467,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@ -521,7 +492,6 @@ github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW
github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
@ -530,7 +500,6 @@ github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -556,7 +525,6 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -655,7 +623,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

6
src/config/default.go
View file

@ -1,6 +1,7 @@
package config package config
var defaultConfig = Config{ var defaultConfig = Config{
DataFolder: "./data",
WebUi: WebUi{ WebUi: WebUi{
Port: 4444, Port: 4444,
IP: "0.0.0.0", IP: "0.0.0.0",
@ -22,8 +23,9 @@ var defaultConfig = Config{
}, },
TorrentClient: TorrentClient{ TorrentClient: TorrentClient{
DataFolder: "data", DataFolder: "./torrent/data",
MetadataFolder: "metadata", MetadataFolder: "./torrent/metadata",
DHTNodes: []string{},
// GlobalCacheSize: 2048, // GlobalCacheSize: 2048,

5
src/config/model.go
View file

@ -6,6 +6,8 @@ type Config struct {
TorrentClient TorrentClient `koanf:"torrent"` TorrentClient TorrentClient `koanf:"torrent"`
Mounts Mounts `koanf:"mounts"` Mounts Mounts `koanf:"mounts"`
Log Log `koanf:"log"` Log Log `koanf:"log"`
DataFolder string `koanf:"dataFolder"`
} }
type WebUi struct { type WebUi struct {
@ -25,7 +27,8 @@ type TorrentClient struct {
ReadTimeout int `koanf:"read_timeout,omitempty"` ReadTimeout int `koanf:"read_timeout,omitempty"`
AddTimeout int `koanf:"add_timeout,omitempty"` AddTimeout int `koanf:"add_timeout,omitempty"`
DisableIPv6 bool `koanf:"disable_ipv6,omitempty"` DHTNodes []string `koanf:"dhtnodes,omitempty"`
DisableIPv6 bool `koanf:"disable_ipv6,omitempty"`
DataFolder string `koanf:"data_folder,omitempty"` DataFolder string `koanf:"data_folder,omitempty"`
MetadataFolder string `koanf:"metadata_folder,omitempty"` MetadataFolder string `koanf:"metadata_folder,omitempty"`

24
src/fs/container.go
View file

@ -1,24 +0,0 @@
package fs
type ContainerFs struct {
s *storage
}
func NewContainerFs(fss map[string]Filesystem) (*ContainerFs, error) {
s := newStorage(SupportedFactories)
for p, fs := range fss {
if err := s.AddFS(fs, p); err != nil {
return nil, err
}
}
return &ContainerFs{s: s}, nil
}
func (fs *ContainerFs) Open(filename string) (File, error) {
return fs.s.Get(filename)
}
func (fs *ContainerFs) ReadDir(path string) (map[string]File, error) {
return fs.s.Children(path)
}

28
src/fs/container_test.go
View file

@ -1,28 +0,0 @@
package fs
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestContainer(t *testing.T) {
t.Parallel()
require := require.New(t)
fss := map[string]Filesystem{
"/test": &DummyFs{},
}
c, err := NewContainerFs(fss)
require.NoError(err)
f, err := c.Open("/test/dir/here")
require.NoError(err)
require.NotNil(f)
files, err := c.ReadDir("/")
require.NoError(err)
require.Len(files, 1)
}

39
src/fs/memory_test.go
View file

@ -1,39 +0,0 @@
package fs
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMemory(t *testing.T) {
t.Parallel()
require := require.New(t)
mem := NewMemory()
mem.Storage.Add(NewMemoryFile([]byte("Hello")), "/dir/here")
fss := map[string]Filesystem{
"/test": mem,
}
c, err := NewContainerFs(fss)
require.NoError(err)
f, err := c.Open("/test/dir/here")
require.NoError(err)
require.NotNil(f)
require.Equal(int64(5), f.Size())
require.NoError(f.Close())
files, err := c.ReadDir("/")
require.NoError(err)
require.Len(files, 1)
files, err = c.ReadDir("/test")
require.NoError(err)
require.Len(files, 1)
}

184
src/fs/storage.go
View file

@ -1,184 +0,0 @@
package fs
import (
"os"
"path"
"strings"
)
const separator = "/"
type FsFactory func(f File) (Filesystem, error)
var SupportedFactories = map[string]FsFactory{
".zip": func(f File) (Filesystem, error) {
return NewArchive(f, f.Size(), &Zip{}), nil
},
".rar": func(f File) (Filesystem, error) {
return NewArchive(f, f.Size(), &Rar{}), nil
},
".7z": func(f File) (Filesystem, error) {
return NewArchive(f, f.Size(), &SevenZip{}), nil
},
}
type storage struct {
factories map[string]FsFactory
files map[string]File
filesystems map[string]Filesystem
children map[string]map[string]File
}
func newStorage(factories map[string]FsFactory) *storage {
return &storage{
files: make(map[string]File),
children: make(map[string]map[string]File),
filesystems: make(map[string]Filesystem),
factories: factories,
}
}
func (s *storage) Clear() {
s.files = make(map[string]File)
s.children = make(map[string]map[string]File)
s.filesystems = make(map[string]Filesystem)
s.Add(&Dir{}, "/")
}
func (s *storage) Has(path string) bool {
path = clean(path)
f := s.files[path]
if f != nil {
return true
}
if f, _ := s.getFileFromFs(path); f != nil {
return true
}
return false
}
func (s *storage) AddFS(fs Filesystem, p string) error {
p = clean(p)
if s.Has(p) {
if dir, err := s.Get(p); err == nil {
if !dir.IsDir() {
return os.ErrExist
}
}
return nil
}
s.filesystems[p] = fs
return s.createParent(p, &Dir{})
}
func (s *storage) Add(f File, p string) error {
p = clean(p)
if s.Has(p) {
if dir, err := s.Get(p); err == nil {
if !dir.IsDir() {
return os.ErrExist
}
}
return nil
}
ext := path.Ext(p)
if ffs := s.factories[ext]; ffs != nil {
fs, err := ffs(f)
if err != nil {
return err
}
s.filesystems[p] = fs
} else {
s.files[p] = f
}
return s.createParent(p, f)
}
func (s *storage) createParent(p string, f File) error {
base, filename := path.Split(p)
base = clean(base)
if err := s.Add(&Dir{}, base); err != nil {
return err
}
if _, ok := s.children[base]; !ok {
s.children[base] = make(map[string]File)
}
if filename != "" {
s.children[base][filename] = f
}
return nil
}
func (s *storage) Children(path string) (map[string]File, error) {
path = clean(path)
files, err := s.getDirFromFs(path)
if err == nil {
return files, nil
}
if !os.IsNotExist(err) {
return nil, err
}
l := make(map[string]File)
for n, f := range s.children[path] {
l[n] = f
}
return l, nil
}
func (s *storage) Get(path string) (File, error) {
path = clean(path)
if !s.Has(path) {
return nil, os.ErrNotExist
}
file, ok := s.files[path]
if ok {
return file, nil
}
return s.getFileFromFs(path)
}
func (s *storage) getFileFromFs(p string) (File, error) {
for fsp, fs := range s.filesystems {
if strings.HasPrefix(p, fsp) {
return fs.Open(separator + strings.TrimPrefix(p, fsp))
}
}
return nil, os.ErrNotExist
}
func (s *storage) getDirFromFs(p string) (map[string]File, error) {
for fsp, fs := range s.filesystems {
if strings.HasPrefix(p, fsp) {
path := strings.TrimPrefix(p, fsp)
return fs.ReadDir(path)
}
}
return nil, os.ErrNotExist
}
func clean(p string) string {
return path.Clean(separator + strings.ReplaceAll(p, "\\", "/"))
}

195
src/fs/storage_test.go
View file

@ -1,195 +0,0 @@
package fs
import (
"os"
"testing"
"github.com/stretchr/testify/require"
)
var dummyFactories = map[string]FsFactory{
".test": func(f File) (Filesystem, error) {
return &DummyFs{}, nil
},
}
func TestStorage(t *testing.T) {
t.Parallel()
require := require.New(t)
s := newStorage(dummyFactories)
err := s.Add(&Dummy{}, "/path/to/dummy/file.txt")
require.NoError(err)
err = s.Add(&Dummy{}, "/path/to/dummy/file2.txt")
require.NoError(err)
contains := s.Has("/path")
require.True(contains)
contains = s.Has("/path/to/dummy/")
require.True(contains)
file, err := s.Get("/path/to/dummy/file.txt")
require.NoError(err)
require.Equal(&Dummy{}, file)
file, err = s.Get("/path/to/dummy/file3.txt")
require.Error(err)
require.Nil(file)
files, err := s.Children("/path/to/dummy/")
require.NoError(err)
require.Len(files, 2)
require.Contains(files, "file.txt")
require.Contains(files, "file2.txt")
err = s.Add(&Dummy{}, "/path/to/dummy/folder/file.txt")
require.NoError(err)
files, err = s.Children("/path/to/dummy/")
require.NoError(err)
require.Len(files, 3)
require.Contains(files, "file.txt")
require.Contains(files, "file2.txt")
require.Contains(files, "folder")
err = s.Add(&Dummy{}, "path/file4.txt")
require.NoError(err)
require.True(s.Has("/path/file4.txt"))
files, err = s.Children("/")
require.NoError(err)
require.Len(files, 1)
err = s.Add(&Dummy{}, "/path/special_file.test")
require.NoError(err)
file, err = s.Get("/path/special_file.test/dir/here/file1.txt")
require.NoError(err)
require.Equal(&Dummy{}, file)
files, err = s.Children("/path/special_file.test")
require.NoError(err)
require.NotNil(files)
files, err = s.Children("/path/special_file.test/dir/here")
require.NoError(err)
require.Len(files, 2)
err = s.Add(&Dummy{}, "/path/to/__special__path/file3.txt")
require.NoError(err)
file, err = s.Get("/path/to/__special__path/file3.txt")
require.NoError(err)
require.Equal(&Dummy{}, file)
s.Clear()
}
func TestStorageWindowsPath(t *testing.T) {
t.Parallel()
require := require.New(t)
s := newStorage(dummyFactories)
err := s.Add(&Dummy{}, "\\path\\to\\dummy\\file.txt")
require.NoError(err)
file, err := s.Get("\\path\\to\\dummy\\file.txt")
require.NoError(err)
require.Equal(&Dummy{}, file)
file, err = s.Get("/path/to/dummy/file.txt")
require.NoError(err)
require.Equal(&Dummy{}, file)
}
func TestStorageAddFs(t *testing.T) {
t.Parallel()
require := require.New(t)
s := newStorage(dummyFactories)
err := s.AddFS(&DummyFs{}, "/test")
require.NoError(err)
f, err := s.Get("/test/dir/here/file1.txt")
require.NoError(err)
require.NotNil(f)
err = s.AddFS(&DummyFs{}, "/test")
require.Error(err)
}
func TestSupportedFactories(t *testing.T) {
t.Parallel()
require := require.New(t)
require.Contains(SupportedFactories, ".zip")
require.Contains(SupportedFactories, ".rar")
require.Contains(SupportedFactories, ".7z")
fs, err := SupportedFactories[".zip"](&Dummy{})
require.NoError(err)
require.NotNil(fs)
fs, err = SupportedFactories[".rar"](&Dummy{})
require.NoError(err)
require.NotNil(fs)
fs, err = SupportedFactories[".7z"](&Dummy{})
require.NoError(err)
require.NotNil(fs)
}
var _ Filesystem = &DummyFs{}
type DummyFs struct {
}
func (d *DummyFs) Open(filename string) (File, error) {
return &Dummy{}, nil
}
func (d *DummyFs) ReadDir(path string) (map[string]File, error) {
if path == "/dir/here" {
return map[string]File{
"file1.txt": &Dummy{},
"file2.txt": &Dummy{},
}, nil
}
return nil, os.ErrNotExist
}
var _ File = &Dummy{}
type Dummy struct {
}
func (d *Dummy) Size() int64 {
return 0
}
func (d *Dummy) IsDir() bool {
return false
}
func (d *Dummy) Close() error {
return nil
}
func (d *Dummy) Read(p []byte) (n int, err error) {
return 0, nil
}
func (d *Dummy) ReadAt(p []byte, off int64) (n int, err error) {
return 0, nil
}

120
src/host/storage.go Normal file
View file

@ -0,0 +1,120 @@
package host
import (
"git.kmsign.ru/royalcat/tstor/src/host/torrent"
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
)
type storage struct {
factories map[string]vfs.FsFactory
}
func NewStorage(downPath string, tsrv *torrent.Service) vfs.Filesystem {
factories := map[string]vfs.FsFactory{
".torrent": tsrv.NewTorrentFs,
}
// add default torrent factory for root filesystem
for k, v := range vfs.ArchiveFactories {
factories[k] = v
}
return vfs.NewResolveFS(downPath, factories)
}
// func (s *storage) Clear() {
// s.files = make(map[string]vfs.File)
// }
// func (s *storage) Has(path string) bool {
// path = clean(path)
// f := s.files[path]
// if f != nil {
// return true
// }
// if f, _ := s.getFileFromFs(path); f != nil {
// return true
// }
// return false
// }
// func (s *storage) createParent(p string, f File) error {
// base, filename := path.Split(p)
// base = clean(base)
// if err := s.Add(&Dir{}, base); err != nil {
// return err
// }
// if _, ok := s.children[base]; !ok {
// s.children[base] = make(map[string]File)
// }
// if filename != "" {
// s.children[base][filename] = f
// }
// return nil
// }
// func (s *storage) Children(path string) (map[string]File, error) {
// path = clean(path)
// files, err := s.getDirFromFs(path)
// if err == nil {
// return files, nil
// }
// if !os.IsNotExist(err) {
// return nil, err
// }
// l := make(map[string]File)
// for n, f := range s.children[path] {
// l[n] = f
// }
// return l, nil
// }
// func (s *storage) Get(path string) (File, error) {
// path = clean(path)
// if !s.Has(path) {
// return nil, os.ErrNotExist
// }
// file, ok := s.files[path]
// if ok {
// return file, nil
// }
// return s.getFileFromFs(path)
// }
// func (s *storage) getFileFromFs(p string) (File, error) {
// for fsp, fs := range s.filesystems {
// if strings.HasPrefix(p, fsp) {
// return fs.Open(separator + strings.TrimPrefix(p, fsp))
// }
// }
// return nil, os.ErrNotExist
// }
// func (s *storage) getDirFromFs(p string) (map[string]File, error) {
// for fsp, fs := range s.filesystems {
// if strings.HasPrefix(p, fsp) {
// path := strings.TrimPrefix(p, fsp)
// return fs.ReadDir(path)
// }
// }
// return nil, os.ErrNotExist
// }
// func clean(p string) string {
// return path.Clean(separator + strings.ReplaceAll(p, "\\", "/"))
// }

0
src/torrent/client.go → src/host/torrent/client.go
View file

0
src/torrent/id.go → src/host/torrent/id.go
View file

213
src/host/torrent/service.go Normal file
View file

@ -0,0 +1,213 @@
package torrent
import (
"sync"
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
type Service struct {
c *torrent.Client
// stats *Stats
mu sync.Mutex
log zerolog.Logger
addTimeout, readTimeout int
}
func NewService(c *torrent.Client, addTimeout, readTimeout int) *Service {
l := log.Logger.With().Str("component", "torrent-service").Logger()
return &Service{
log: l,
c: c,
// stats: newStats(), // TODO persistent
addTimeout: addTimeout,
readTimeout: readTimeout,
}
}
var _ vfs.FsFactory = (*Service)(nil).NewTorrentFs
func (s *Service) NewTorrentFs(f vfs.File) (vfs.Filesystem, error) {
defer f.Close()
mi, err := metainfo.Load(f)
if err != nil {
return nil, err
}
t, err := s.c.AddTorrent(mi)
if err != nil {
return nil, err
}
<-t.GotInfo()
t.AllowDataDownload()
return vfs.NewTorrentFs(t, s.readTimeout), nil
}
func (s *Service) Stats() (*Stats, error) {
return &Stats{}, nil
}
// func (s *Service) Load() (map[string]vfs.Filesystem, error) {
// // Load from config
// s.log.Info().Msg("adding torrents from configuration")
// for _, loader := range s.loaders {
// if err := s.load(loader); err != nil {
// return nil, err
// }
// }
// // Load from DB
// s.log.Info().Msg("adding torrents from database")
// return s.fss, s.load(s.db)
// }
// func (s *Service) load(l loader.Loader) error {
// list, err := l.ListMagnets()
// if err != nil {
// return err
// }
// for r, ms := range list {
// s.addRoute(r)
// for _, m := range ms {
// if err := s.addMagnet(r, m); err != nil {
// return err
// }
// }
// }
// list, err = l.ListTorrentPaths()
// if err != nil {
// return err
// }
// for r, ms := range list {
// s.addRoute(r)
// for _, p := range ms {
// if err := s.addTorrentPath(r, p); err != nil {
// return err
// }
// }
// }
// return nil
// }
// func (s *Service) AddMagnet(r, m string) error {
// if err := s.addMagnet(r, m); err != nil {
// return err
// }
// // Add to db
// return s.db.AddMagnet(r, m)
// }
// func (s *Service) addTorrentPath(r, p string) error {
// // Add to client
// t, err := s.c.AddTorrentFromFile(p)
// if err != nil {
// return err
// }
// return s.addTorrent(r, t)
// }
// func (s *Service) addMagnet(r, m string) error {
// // Add to client
// t, err := s.c.AddMagnet(m)
// if err != nil {
// return err
// }
// return s.addTorrent(r, t)
// }
// func (s *Service) addRoute(r string) {
// s.s.AddRoute(r)
// // Add to filesystems
// folder := path.Join("/", r)
// s.mu.Lock()
// defer s.mu.Unlock()
// _, ok := s.fss[folder]
// if !ok {
// s.fss[folder] = vfs.NewTorrentFs(s.readTimeout)
// }
// }
// func (s *Service) addTorrent(r string, t *torrent.Torrent) error {
// // only get info if name is not available
// if t.Info() == nil {
// s.log.Info().Str("hash", t.InfoHash().String()).Msg("getting torrent info")
// select {
// case <-time.After(time.Duration(s.addTimeout) * time.Second):
// s.log.Error().Str("hash", t.InfoHash().String()).Msg("timeout getting torrent info")
// return errors.New("timeout getting torrent info")
// case <-t.GotInfo():
// s.log.Info().Str("hash", t.InfoHash().String()).Msg("obtained torrent info")
// }
// }
// // Add to stats
// s.s.Add(r, t)
// // Add to filesystems
// folder := path.Join("/", r)
// s.mu.Lock()
// defer s.mu.Unlock()
// tfs, ok := s.fss[folder].(*vfs.TorrentFs)
// if !ok {
// return errors.New("error adding torrent to filesystem")
// }
// tfs.AddTorrent(t)
// s.log.Info().Str("name", t.Info().Name).Str("route", r).Msg("torrent added")
// return nil
// }
// func (s *Service) RemoveFromHash(r, h string) error {
// // Remove from db
// deleted, err := s.db.RemoveFromHash(r, h)
// if err != nil {
// return err
// }
// if !deleted {
// return fmt.Errorf("element with hash %v on route %v cannot be removed", h, r)
// }
// // Remove from stats
// s.s.Del(r, h)
// // Remove from fs
// folder := path.Join("/", r)
// tfs, ok := s.fss[folder].(*vfs.TorrentFs)
// if !ok {
// return errors.New("error removing torrent from filesystem")
// }
// tfs.RemoveTorrent(h)
// // Remove from client
// var mh metainfo.Hash
// if err := mh.FromHexString(h); err != nil {
// return err
// }
// t, ok := s.c.Torrent(metainfo.NewHashFromHex(h))
// if ok {
// t.Drop()
// }
// return nil
// }

82
src/torrent/stats.go → src/host/torrent/stats.go
View file

@ -2,7 +2,6 @@ package torrent
import ( import (
"errors" "errors"
"sort"
"sync" "sync"
"time" "time"
@ -73,65 +72,42 @@ type stat struct {
} }
type Stats struct { type Stats struct {
mut sync.Mutex mut sync.Mutex
torrents map[string]*torrent.Torrent torrents map[string]*torrent.Torrent
torrentsByRoute map[string]map[string]*torrent.Torrent previousStats map[string]*stat
previousStats map[string]*stat
gTime time.Time gTime time.Time
} }
func NewStats() *Stats { func newStats() *Stats {
return &Stats{ return &Stats{
gTime: time.Now(), gTime: time.Now(),
torrents: make(map[string]*torrent.Torrent), torrents: make(map[string]*torrent.Torrent),
torrentsByRoute: make(map[string]map[string]*torrent.Torrent), // torrentsByRoute: make(map[string]map[string]*torrent.Torrent),
previousStats: make(map[string]*stat), // previousStats: make(map[string]*stat),
} }
} }
func (s *Stats) AddRoute(route string) { func (s *Stats) Add(path string, t *torrent.Torrent) {
_, ok := s.torrentsByRoute[route]
if !ok {
s.torrentsByRoute[route] = make(map[string]*torrent.Torrent)
}
}
func (s *Stats) Add(route string, t *torrent.Torrent) {
s.mut.Lock() s.mut.Lock()
defer s.mut.Unlock() defer s.mut.Unlock()
h := t.InfoHash().String() s.torrents[path] = t
s.previousStats[path] = &stat{}
s.torrents[h] = t
s.previousStats[h] = &stat{}
_, ok := s.torrentsByRoute[route]
if !ok {
s.torrentsByRoute[route] = make(map[string]*torrent.Torrent)
}
s.torrentsByRoute[route][h] = t
} }
func (s *Stats) Del(route, hash string) { func (s *Stats) Del(path string) {
s.mut.Lock() s.mut.Lock()
defer s.mut.Unlock() defer s.mut.Unlock()
delete(s.torrents, hash) delete(s.torrents, path)
delete(s.previousStats, hash) delete(s.previousStats, path)
ts, ok := s.torrentsByRoute[route]
if !ok {
return
}
delete(ts, hash)
} }
func (s *Stats) Stats(hash string) (*TorrentStats, error) { func (s *Stats) Stats(path string) (*TorrentStats, error) {
s.mut.Lock() s.mut.Lock()
defer s.mut.Unlock() defer s.mut.Unlock()
t, ok := s.torrents[hash] t, ok := s.torrents[path]
if !(ok) { if !(ok) {
return nil, ErrTorrentNotFound return nil, ErrTorrentNotFound
} }
@ -141,32 +117,6 @@ func (s *Stats) Stats(hash string) (*TorrentStats, error) {
return s.stats(now, t, true), nil return s.stats(now, t, true), nil
} }
func (s *Stats) RoutesStats() []*RouteStats {
s.mut.Lock()
defer s.mut.Unlock()
now := time.Now()
var out []*RouteStats
for r, tl := range s.torrentsByRoute {
var tStats []*TorrentStats
for _, t := range tl {
ts := s.stats(now, t, true)
tStats = append(tStats, ts)
}
sort.Sort(byName(tStats))
rs := &RouteStats{
Name: r,
TorrentStats: tStats,
}
out = append(out, rs)
}
return out
}
func (s *Stats) GlobalStats() *GlobalTorrentStats { func (s *Stats) GlobalStats() *GlobalTorrentStats {
s.mut.Lock() s.mut.Lock()
defer s.mut.Unlock() defer s.mut.Unlock()

0
src/torrent/store.go → src/host/torrent/store.go
View file

267
src/fs/archive.go → src/host/vfs/archive.go
View file

@ -1,4 +1,4 @@
package fs package vfs
import ( import (
"archive/zip" "archive/zip"
@ -12,18 +12,129 @@ import (
"github.com/nwaples/rardecode/v2" "github.com/nwaples/rardecode/v2"
) )
var _ loader = &Zip{} var ArchiveFactories = map[string]FsFactory{
".zip": func(f File) (Filesystem, error) {
type Zip struct { return NewArchive(f, f.Size(), ZipLoader), nil
},
".rar": func(f File) (Filesystem, error) {
return NewArchive(f, f.Size(), RarLoader), nil
},
".7z": func(f File) (Filesystem, error) {
return NewArchive(f, f.Size(), SevenZipLoader), nil
},
} }
func (fs *Zip) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile, error) { type ArchiveLoader func(r iio.Reader, size int64) (map[string]*archiveFile, error)
var _ Filesystem = &archive{}
type archive struct {
r iio.Reader
size int64
files func() (map[string]*archiveFile, error)
}
func NewArchive(r iio.Reader, size int64, loader ArchiveLoader) *archive {
return &archive{
r: r,
size: size,
files: sync.OnceValues(func() (map[string]*archiveFile, error) {
return loader(r, size)
}),
}
}
func (a *archive) Open(filename string) (File, error) {
files, err := a.files()
if err != nil {
return nil, err
}
return getFile(files, filename)
}
func (fs *archive) ReadDir(path string) (map[string]File, error) {
files, err := fs.files()
if err != nil {
return nil, err
}
return listFilesInDir(files, path)
}
var _ File = &archiveFile{}
func NewArchiveFile(readerFunc func() (iio.Reader, error), len int64) *archiveFile {
return &archiveFile{
readerFunc: readerFunc,
len: len,
}
}
type archiveFile struct {
readerFunc func() (iio.Reader, error)
reader iio.Reader
len int64
}
func (d *archiveFile) load() error {
if d.reader != nil {
return nil
}
r, err := d.readerFunc()
if err != nil {
return err
}
d.reader = r
return nil
}
func (d *archiveFile) Size() int64 {
return d.len
}
func (d *archiveFile) IsDir() bool {
return false
}
func (d *archiveFile) Close() (err error) {
if d.reader != nil {
err = d.reader.Close()
d.reader = nil
}
return
}
func (d *archiveFile) Read(p []byte) (n int, err error) {
if err := d.load(); err != nil {
return 0, err
}
return d.reader.Read(p)
}
func (d *archiveFile) ReadAt(p []byte, off int64) (n int, err error) {
if err := d.load(); err != nil {
return 0, err
}
return d.reader.ReadAt(p, off)
}
var _ ArchiveLoader = ZipLoader
func ZipLoader(reader iio.Reader, size int64) (map[string]*archiveFile, error) {
zr, err := zip.NewReader(reader, size) zr, err := zip.NewReader(reader, size)
if err != nil { if err != nil {
return nil, err return nil, err
} }
out := make(map[string]*ArchiveFile) out := make(map[string]*archiveFile)
for _, f := range zr.File { for _, f := range zr.File {
f := f f := f
if f.FileInfo().IsDir() { if f.FileInfo().IsDir() {
@ -48,18 +159,15 @@ func (fs *Zip) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile,
return out, nil return out, nil
} }
var _ loader = &SevenZip{} var _ ArchiveLoader = SevenZipLoader
type SevenZip struct { func SevenZipLoader(reader iio.Reader, size int64) (map[string]*archiveFile, error) {
}
func (fs *SevenZip) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile, error) {
r, err := sevenzip.NewReader(reader, size) r, err := sevenzip.NewReader(reader, size)
if err != nil { if err != nil {
return nil, err return nil, err
} }
out := make(map[string]*ArchiveFile) out := make(map[string]*archiveFile)
for _, f := range r.File { for _, f := range r.File {
f := f f := f
if f.FileInfo().IsDir() { if f.FileInfo().IsDir() {
@ -84,18 +192,15 @@ func (fs *SevenZip) getFiles(reader iio.Reader, size int64) (map[string]*Archive
return out, nil return out, nil
} }
var _ loader = &Rar{} var _ ArchiveLoader = RarLoader
type Rar struct { func RarLoader(reader iio.Reader, size int64) (map[string]*archiveFile, error) {
}
func (fs *Rar) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile, error) {
r, err := rardecode.NewReader(iio.NewSeekerWrapper(reader, size)) r, err := rardecode.NewReader(iio.NewSeekerWrapper(reader, size))
if err != nil { if err != nil {
return nil, err return nil, err
} }
out := make(map[string]*ArchiveFile) out := make(map[string]*archiveFile)
for { for {
header, err := r.Next() header, err := r.Next()
if err == io.EOF { if err == io.EOF {
@ -118,129 +223,3 @@ func (fs *Rar) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile,
return out, nil return out, nil
} }
type loader interface {
getFiles(r iio.Reader, size int64) (map[string]*ArchiveFile, error)
}
var _ Filesystem = &archive{}
type archive struct {
r iio.Reader
s *storage
size int64
once sync.Once
l loader
}
func NewArchive(r iio.Reader, size int64, l loader) *archive {
return &archive{
r: r,
s: newStorage(nil),
size: size,
l: l,
}
}
func (fs *archive) loadOnce() error {
var errOut error
fs.once.Do(func() {
files, err := fs.l.getFiles(fs.r, fs.size)
if err != nil {
errOut = err
return
}
for name, file := range files {
if err := fs.s.Add(file, name); err != nil {
errOut = err
return
}
}
})
return errOut
}
func (fs *archive) Open(filename string) (File, error) {
if filename == string(os.PathSeparator) {
return &Dir{}, nil
}
if err := fs.loadOnce(); err != nil {
return nil, err
}
return fs.s.Get(filename)
}
func (fs *archive) ReadDir(path string) (map[string]File, error) {
if err := fs.loadOnce(); err != nil {
return nil, err
}
return fs.s.Children(path)
}
var _ File = &ArchiveFile{}
func NewArchiveFile(readerFunc func() (iio.Reader, error), len int64) *ArchiveFile {
return &ArchiveFile{
readerFunc: readerFunc,
len: len,
}
}
type ArchiveFile struct {
readerFunc func() (iio.Reader, error)
reader iio.Reader
len int64
}
func (d *ArchiveFile) load() error {
if d.reader != nil {
return nil
}
r, err := d.readerFunc()
if err != nil {
return err
}
d.reader = r
return nil
}
func (d *ArchiveFile) Size() int64 {
return d.len
}
func (d *ArchiveFile) IsDir() bool {
return false
}
func (d *ArchiveFile) Close() (err error) {
if d.reader != nil {
err = d.reader.Close()
d.reader = nil
}
return
}
func (d *ArchiveFile) Read(p []byte) (n int, err error) {
if err := d.load(); err != nil {
return 0, err
}
return d.reader.Read(p)
}
func (d *ArchiveFile) ReadAt(p []byte, off int64) (n int, err error) {
if err := d.load(); err != nil {
return 0, err
}
return d.reader.ReadAt(p, off)
}

4
src/fs/archive_test.go → src/host/vfs/archive_test.go
View file

@ -1,4 +1,4 @@
package fs package vfs
import ( import (
"archive/zip" "archive/zip"
@ -18,7 +18,7 @@ func TestZipFilesystem(t *testing.T) {
zReader, len := createTestZip(require) zReader, len := createTestZip(require)
zfs := NewArchive(zReader, len, &Zip{}) zfs := NewArchive(zReader, len, ZipLoader)
files, err := zfs.ReadDir("/path/to/test/file") files, err := zfs.ReadDir("/path/to/test/file")
require.NoError(err) require.NoError(err)

2
src/fs/dir.go → src/host/vfs/dir.go
View file

@ -1,4 +1,4 @@
package fs package vfs
var _ File = &Dir{} var _ File = &Dir{}

2
src/fs/fs.go → src/host/vfs/fs.go
View file

@ -1,4 +1,4 @@
package fs package vfs
import ( import (
"os" "os"

2
src/fs/fs_test.go → src/host/vfs/fs_test.go
View file

@ -1,4 +1,4 @@
package fs package vfs
import ( import (
"io/fs" "io/fs"

22
src/fs/memory.go → src/host/vfs/memory.go
View file

@ -1,27 +1,27 @@
package fs package vfs
import ( import (
"bytes" "bytes"
) )
var _ Filesystem = &Memory{} var _ Filesystem = &MemoryFs{}
type Memory struct { type MemoryFs struct {
Storage *storage files map[string]*MemoryFile
} }
func NewMemory() *Memory { func NewMemoryFS(files map[string]*MemoryFile) *MemoryFs {
return &Memory{ return &MemoryFs{
Storage: newStorage(nil), files: files,
} }
} }
func (fs *Memory) Open(filename string) (File, error) { func (m *MemoryFs) Open(filename string) (File, error) {
return fs.Storage.Get(filename) return getFile(m.files, filename)
} }
func (fs *Memory) ReadDir(path string) (map[string]File, error) { func (fs *MemoryFs) ReadDir(path string) (map[string]File, error) {
return fs.Storage.Children(path) return listFilesInDir(fs.files, path)
} }
var _ File = &MemoryFile{} var _ File = &MemoryFile{}

46
src/host/vfs/memory_test.go Normal file
View file

@ -0,0 +1,46 @@
package vfs
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMemory(t *testing.T) {
t.Parallel()
require := require.New(t)
testData := "Hello"
c := NewMemoryFS(map[string]*MemoryFile{
"/dir/here": NewMemoryFile([]byte(testData)),
})
// fss := map[string]Filesystem{
// "/test": mem,
// }
// c, err := NewContainerFs(fss)
// require.NoError(err)
f, err := c.Open("/dir/here")
require.NoError(err)
require.NotNil(f)
require.Equal(int64(5), f.Size())
require.NoError(f.Close())
data := make([]byte, 5)
n, err := f.Read(data)
require.NoError(err)
require.Equal(n, 5)
require.Equal(string(data), testData)
files, err := c.ReadDir("/")
require.NoError(err)
require.Len(files, 1)
files, err = c.ReadDir("/dir")
require.NoError(err)
require.Len(files, 1)
}

167
src/host/vfs/os.go Normal file
View file

@ -0,0 +1,167 @@
package vfs
import (
"io/fs"
"os"
"path"
"sync"
)
type OsFS struct {
hostDir string
}
// Open implements Filesystem.
func (fs *OsFS) Open(filename string) (File, error) {
if path.Clean(filename) == Separator {
return &Dir{}, nil
}
osfile, err := os.Open(path.Join(fs.hostDir, filename))
if err != nil {
return nil, err
}
return NewOsFile(osfile), nil
}
// ReadDir implements Filesystem.
func (o *OsFS) ReadDir(dir string) (map[string]File, error) {
dir = path.Join(o.hostDir, dir)
entries, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
out := map[string]File{}
for _, e := range entries {
if e.IsDir() {
out[e.Name()] = &Dir{}
} else {
out[e.Name()] = NewLazyOsFile(path.Join(dir, e.Name()))
}
}
return out, nil
}
func NewOsFs(osDir string) *OsFS {
return &OsFS{
hostDir: osDir,
}
}
var _ Filesystem = &OsFS{}
type OsFile struct {
f *os.File
}
func NewOsFile(f *os.File) *OsFile {
return &OsFile{f: f}
}
var _ File = &OsFile{}
// Close implements File.
func (f *OsFile) Close() error {
return f.f.Close()
}
// Read implements File.
func (f *OsFile) Read(p []byte) (n int, err error) {
return f.f.Read(p)
}
// ReadAt implements File.
func (f *OsFile) ReadAt(p []byte, off int64) (n int, err error) {
return f.f.ReadAt(p, off)
}
func (f *OsFile) Stat() (fs.FileInfo, error) {
return f.f.Stat()
}
// Size implements File.
func (f *OsFile) Size() int64 {
stat, err := f.Stat()
if err != nil {
return 0
}
return stat.Size()
}
// IsDir implements File.
func (f *OsFile) IsDir() bool {
stat, err := f.Stat()
if err != nil {
return false
}
return stat.IsDir()
}
type LazyOsFile struct {
m sync.Mutex
path string
file *os.File
}
func NewLazyOsFile(path string) *LazyOsFile {
return &LazyOsFile{path: path}
}
var _ File = &OsFile{}
func (f *LazyOsFile) open() error {
f.m.Lock()
defer f.m.Unlock()
if f.file != nil {
return nil
}
osFile, err := os.Open(f.path)
if err != nil {
return err
}
f.file = osFile
return nil
}
// Close implements File.
func (f *LazyOsFile) Close() error {
return f.file.Close()
}
// Read implements File.
func (f *LazyOsFile) Read(p []byte) (n int, err error) {
return f.file.Read(p)
}
// ReadAt implements File.
func (f *LazyOsFile) ReadAt(p []byte, off int64) (n int, err error) {
return f.file.ReadAt(p, off)
}
func (f *LazyOsFile) Stat() (fs.FileInfo, error) {
if f.file == nil {
return os.Stat(f.path)
} else {
return f.file.Stat()
}
}
// Size implements File.
func (f *LazyOsFile) Size() int64 {
stat, err := f.Stat()
if err != nil {
return 0
}
return stat.Size()
}
// IsDir implements File.
func (f *LazyOsFile) IsDir() bool {
stat, err := f.Stat()
if err != nil {
return false
}
return stat.IsDir()
}

146
src/host/vfs/resolver.go Normal file
View file

@ -0,0 +1,146 @@
package vfs
import (
"fmt"
"strings"
"sync"
)
type ResolveFS struct {
osDir string
osFS *OsFS
resolver *resolver
}
func NewResolveFS(osDir string, factories map[string]FsFactory) *ResolveFS {
return &ResolveFS{
osDir: osDir,
osFS: NewOsFs(osDir),
resolver: newResolver(factories),
}
}
// Open implements Filesystem.
func (r *ResolveFS) Open(filename string) (File, error) {
fsPath, nestedFs, nestedFsPath, err := r.resolver.resolvePath(filename, r.osFS.Open)
if err != nil {
return nil, err
}
if nestedFs != nil {
return nestedFs.Open(nestedFsPath)
}
return r.osFS.Open(fsPath)
}
// ReadDir implements Filesystem.
func (r *ResolveFS) ReadDir(dir string) (map[string]File, error) {
fsPath, nestedFs, nestedFsPath, err := r.resolver.resolvePath(dir, r.osFS.Open)
if err != nil {
return nil, err
}
if nestedFs != nil {
return nestedFs.ReadDir(nestedFsPath)
}
return r.osFS.ReadDir(fsPath)
}
var _ Filesystem = &ResolveFS{}
type FsFactory func(f File) (Filesystem, error)
const Separator = "/"
func newResolver(factories map[string]FsFactory) *resolver {
return &resolver{
factories: factories,
fsmap: map[string]Filesystem{},
}
}
type resolver struct {
m sync.Mutex
factories map[string]FsFactory
fsmap map[string]Filesystem // filesystem cache
// TODO: add fsmap clean
}
type openFile func(path string) (File, error)
// open requeue raw open, without resolver call
func (r *resolver) resolvePath(name string, rawOpen openFile) (fsPath string, nestedFs Filesystem, nestedFsPath string, err error) {
name = strings.TrimPrefix(name, Separator)
parts := strings.Split(name, Separator)
nestOn := -1
var nestFactory FsFactory
PARTS_LOOP:
for i, part := range parts {
for ext, factory := range r.factories {
if strings.HasSuffix(part, ext) {
nestOn = i + 1
nestFactory = factory
break PARTS_LOOP
}
}
}
if nestOn == -1 {
return name, nil, "", nil
}
fsPath = Clean(strings.Join(parts[:nestOn], Separator))
nestedFsPath = Clean(strings.Join(parts[nestOn:], Separator))
// we dont need lock until now
// it must be before fsmap read to exclude race condition:
// read -> write
// read -> write
r.m.Lock()
defer r.m.Unlock()
if nestedFs, ok := r.fsmap[fsPath]; ok {
return fsPath, nestedFs, nestedFsPath, nil
} else {
fsFile, err := rawOpen(fsPath)
if err != nil {
return "", nil, "", fmt.Errorf("error opening filesystem file: %s with error: %w", fsPath, err)
}
nestedFs, err := nestFactory(fsFile)
if err != nil {
return "", nil, "", fmt.Errorf("error creating filesystem from file: %s with error: %w", fsPath, err)
}
r.fsmap[fsPath] = nestedFs
return fsPath, nestedFs, nestedFsPath, nil
}
}
// func (r *resolver) resolveFile(name string, fs Filesystem) (File, error) {
// fsPath, nestedFs, nestedFsPath, err := r.resolvePath(name, fs)
// if err != nil {
// return nil, err
// }
// if nestedFs == nil {
// return fs.Open(fsPath)
// }
// return nestedFs.Open(nestedFsPath)
// }
// func (r *resolver) resolveDir(name string, fs Filesystem) (map[string]File, error) {
// fsPath, nestedFs, nestedFsPath, err := r.resolvePath(name, fs)
// if err != nil {
// return nil, err
// }
// if nestedFs == nil {
// return fs.ReadDir(fsPath)
// }
// return nestedFs.ReadDir(nestedFsPath)
// }

192
src/host/vfs/resolver_test.go Normal file
View file

@ -0,0 +1,192 @@
package vfs
import (
"os"
)
type Dummy struct {
}
func (d *Dummy) Size() int64 {
return 0
}
func (d *Dummy) IsDir() bool {
return false
}
func (d *Dummy) Close() error {
return nil
}
func (d *Dummy) Read(p []byte) (n int, err error) {
return 0, nil
}
func (d *Dummy) ReadAt(p []byte, off int64) (n int, err error) {
return 0, nil
}
var _ File = &Dummy{}
type DummyFs struct {
}
func (d *DummyFs) Open(filename string) (File, error) {
return &Dummy{}, nil
}
func (d *DummyFs) ReadDir(path string) (map[string]File, error) {
if path == "/dir/here" {
return map[string]File{
"file1.txt": &Dummy{},
"file2.txt": &Dummy{},
}, nil
}
return nil, os.ErrNotExist
}
var _ Filesystem = &DummyFs{}
// func TestDefaultFactories(t *testing.T) {
// t.Parallel()
// require := require.New(t)
// require.Contains(defaultFactories, ".zip")
// require.Contains(defaultFactories, ".rar")
// require.Contains(defaultFactories, ".7z")
// fs, err := defaultFactories[".zip"](&Dummy{}, nil)
// require.NoError(err)
// require.NotNil(fs)
// fs, err = defaultFactories[".rar"](&Dummy{}, nil)
// require.NoError(err)
// require.NotNil(fs)
// fs, err = defaultFactories[".7z"](&Dummy{}, nil)
// require.NoError(err)
// require.NotNil(fs)
// }
// func TestStorageAddFs(t *testing.T) {
// t.Parallel()
// require := require.New(t)
// s := newStorage(dummyFactories)
// err := s.AddFS(&DummyFs{}, "/test")
// require.NoError(err)
// f, err := s.Get("/test/dir/here/file1.txt")
// require.NoError(err)
// require.NotNil(f)
// err = s.AddFS(&DummyFs{}, "/test")
// require.Error(err)
// }
// func TestStorageWindowsPath(t *testing.T) {
// t.Parallel()
// require := require.New(t)
// s := newStorage(dummyFactories)
// err := s.Add(&Dummy{}, "\\path\\to\\dummy\\file.txt")
// require.NoError(err)
// file, err := s.Get("\\path\\to\\dummy\\file.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// file, err = s.Get("/path/to/dummy/file.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// }
// var dummyFactories = map[string]vfs.FsFactory{
// ".test": func(f vfs.File, factories map[string]vfs.FsFactory) (vfs.Filesystem, error) {
// return &DummyFs{}, nil
// },
// }
// func TestStorage(t *testing.T) {
// t.Parallel()
// require := require.New(t)
// s := newStorage(dummyFactories)
// err := s.Add(&Dummy{}, "/path/to/dummy/file.txt")
// require.NoError(err)
// err = s.Add(&Dummy{}, "/path/to/dummy/file2.txt")
// require.NoError(err)
// contains := s.Has("/path")
// require.True(contains)
// contains = s.Has("/path/to/dummy/")
// require.True(contains)
// file, err := s.Get("/path/to/dummy/file.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// file, err = s.Get("/path/to/dummy/file3.txt")
// require.Error(err)
// require.Nil(file)
// files, err := s.Children("/path/to/dummy/")
// require.NoError(err)
// require.Len(files, 2)
// require.Contains(files, "file.txt")
// require.Contains(files, "file2.txt")
// err = s.Add(&Dummy{}, "/path/to/dummy/folder/file.txt")
// require.NoError(err)
// files, err = s.Children("/path/to/dummy/")
// require.NoError(err)
// require.Len(files, 3)
// require.Contains(files, "file.txt")
// require.Contains(files, "file2.txt")
// require.Contains(files, "folder")
// err = s.Add(&Dummy{}, "path/file4.txt")
// require.NoError(err)
// require.True(s.Has("/path/file4.txt"))
// files, err = s.Children("/")
// require.NoError(err)
// require.Len(files, 1)
// err = s.Add(&Dummy{}, "/path/special_file.test")
// require.NoError(err)
// file, err = s.Get("/path/special_file.test/dir/here/file1.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// files, err = s.Children("/path/special_file.test")
// require.NoError(err)
// require.NotNil(files)
// files, err = s.Children("/path/special_file.test/dir/here")
// require.NoError(err)
// require.Len(files, 2)
// err = s.Add(&Dummy{}, "/path/to/__special__path/file3.txt")
// require.NoError(err)
// file, err = s.Get("/path/to/__special__path/file3.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// s.Clear()
// }

99
src/fs/torrent.go → src/host/vfs/torrent.go
View file

@ -1,4 +1,4 @@
package fs package vfs
import ( import (
"context" "context"
@ -11,71 +11,66 @@ import (
"github.com/anacrolix/torrent" "github.com/anacrolix/torrent"
) )
var _ Filesystem = &Torrent{} var _ Filesystem = &TorrentFs{}
type Torrent struct { type TorrentFs struct {
mu sync.RWMutex mu sync.RWMutex
ts map[string]*torrent.Torrent t *torrent.Torrent
s *storage
loaded bool
readTimeout int readTimeout int
resolver *resolver
} }
func NewTorrent(readTimeout int) *Torrent { func NewTorrentFs(t *torrent.Torrent, readTimeout int) *TorrentFs {
return &Torrent{ return &TorrentFs{
s: newStorage(SupportedFactories), t: t,
ts: make(map[string]*torrent.Torrent),
readTimeout: readTimeout, readTimeout: readTimeout,
resolver: newResolver(ArchiveFactories),
} }
} }
func (fs *Torrent) AddTorrent(t *torrent.Torrent) { func (fs *TorrentFs) files() map[string]*torrentFile {
fs.mu.Lock() files := make(map[string]*torrentFile)
defer fs.mu.Unlock() <-fs.t.GotInfo()
fs.loaded = false for _, file := range fs.t.Files() {
fs.ts[t.InfoHash().HexString()] = t p := Clean(file.Path())
} files[p] = &torrentFile{
readerFunc: file.NewReader,
func (fs *Torrent) RemoveTorrent(h string) { len: file.Length(),
fs.mu.Lock() timeout: fs.readTimeout,
defer fs.mu.Unlock()
fs.s.Clear()
fs.loaded = false
delete(fs.ts, h)
}
func (fs *Torrent) load() {
if fs.loaded {
return
}
fs.mu.RLock()
defer fs.mu.RUnlock()
for _, t := range fs.ts {
<-t.GotInfo()
for _, file := range t.Files() {
fs.s.Add(&torrentFile{
readerFunc: file.NewReader,
len: file.Length(),
timeout: fs.readTimeout,
}, file.Path())
} }
} }
fs.loaded = true return files
} }
func (fs *Torrent) Open(filename string) (File, error) { func (fs *TorrentFs) rawOpen(path string) (File, error) {
fs.load() file, err := getFile(fs.files(), path)
return fs.s.Get(filename) return file, err
} }
func (fs *Torrent) ReadDir(path string) (map[string]File, error) { func (fs *TorrentFs) Open(filename string) (File, error) {
fs.load() fsPath, nestedFs, nestedFsPath, err := fs.resolver.resolvePath(filename, fs.rawOpen)
return fs.s.Children(path) if err != nil {
return nil, err
}
if nestedFs != nil {
return nestedFs.Open(nestedFsPath)
}
return fs.rawOpen(fsPath)
}
func (fs *TorrentFs) ReadDir(name string) (map[string]File, error) {
fsPath, nestedFs, nestedFsPath, err := fs.resolver.resolvePath(name, fs.rawOpen)
if err != nil {
return nil, err
}
if nestedFs != nil {
return nestedFs.ReadDir(nestedFsPath)
}
return listFilesInDir(fs.files(), fsPath)
} }
type reader interface { type reader interface {
@ -93,7 +88,9 @@ type readAtWrapper struct {
} }
func newReadAtWrapper(r torrent.Reader, timeout int) reader { func newReadAtWrapper(r torrent.Reader, timeout int) reader {
return &readAtWrapper{Reader: r, timeout: timeout} w := &readAtWrapper{Reader: r, timeout: timeout}
w.SetResponsive()
return w
} }
func (rw *readAtWrapper) ReadAt(p []byte, off int64) (int, error) { func (rw *readAtWrapper) ReadAt(p []byte, off int64) (int, error) {

76
src/fs/torrent_test.go → src/host/vfs/torrent_test.go
View file

@ -1,4 +1,4 @@
package fs package vfs
import ( import (
"os" "os"
@ -34,55 +34,55 @@ func TestMain(m *testing.M) {
os.Exit(exitVal) os.Exit(exitVal)
} }
func TestTorrentFilesystem(t *testing.T) { // func TestTorrentFilesystem(t *testing.T) {
require := require.New(t) // require := require.New(t)
to, err := Cli.AddMagnet(testMagnet) // to, err := Cli.AddMagnet(testMagnet)
require.NoError(err) // require.NoError(err)
tfs := NewTorrent(600) // tfs := NewTorrentFs(600)
tfs.AddTorrent(to) // tfs.AddTorrent(to)
files, err := tfs.ReadDir("/") // files, err := tfs.ReadDir("/")
require.NoError(err) // require.NoError(err)
require.Len(files, 1) // require.Len(files, 1)
require.Contains(files, "The WIRED CD - Rip. Sample. Mash. Share") // require.Contains(files, "The WIRED CD - Rip. Sample. Mash. Share")
files, err = tfs.ReadDir("/The WIRED CD - Rip. Sample. Mash. Share") // files, err = tfs.ReadDir("/The WIRED CD - Rip. Sample. Mash. Share")
require.NoError(err) // require.NoError(err)
require.Len(files, 18) // require.Len(files, 18)
f, err := tfs.Open("/The WIRED CD - Rip. Sample. Mash. Share/not_existing_file.txt") // f, err := tfs.Open("/The WIRED CD - Rip. Sample. Mash. Share/not_existing_file.txt")
require.Equal(os.ErrNotExist, err) // require.Equal(os.ErrNotExist, err)
require.Nil(f) // require.Nil(f)
f, err = tfs.Open("/The WIRED CD - Rip. Sample. Mash. Share/01 - Beastie Boys - Now Get Busy.mp3") // f, err = tfs.Open("/The WIRED CD - Rip. Sample. Mash. Share/01 - Beastie Boys - Now Get Busy.mp3")
require.NoError(err) // require.NoError(err)
require.NotNil(f) // require.NotNil(f)
require.Equal(f.Size(), int64(1964275)) // require.Equal(f.Size(), int64(1964275))
b := make([]byte, 10) // b := make([]byte, 10)
n, err := f.Read(b) // n, err := f.Read(b)
require.NoError(err) // require.NoError(err)
require.Equal(10, n) // require.Equal(10, n)
require.Equal([]byte{0x49, 0x44, 0x33, 0x3, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x76}, b) // require.Equal([]byte{0x49, 0x44, 0x33, 0x3, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x76}, b)
n, err = f.ReadAt(b, 10) // n, err = f.ReadAt(b, 10)
require.NoError(err) // require.NoError(err)
require.Equal(10, n) // require.Equal(10, n)
n, err = f.ReadAt(b, 10000) // n, err = f.ReadAt(b, 10000)
require.NoError(err) // require.NoError(err)
require.Equal(10, n) // require.Equal(10, n)
tfs.RemoveTorrent(to.InfoHash().String()) // tfs.RemoveTorrent(to.InfoHash().String())
files, err = tfs.ReadDir("/") // files, err = tfs.ReadDir("/")
require.NoError(err) // require.NoError(err)
require.Len(files, 0) // require.Len(files, 0)
require.NoError(f.Close()) // require.NoError(f.Close())
} // }
func TestReadAtTorrent(t *testing.T) { func TestReadAtTorrent(t *testing.T) {
require := require.New(t) require := require.New(t)

55
src/host/vfs/utils.go Normal file
View file

@ -0,0 +1,55 @@
package vfs
import (
"io/fs"
"path"
"strings"
)
var ErrNotExist = fs.ErrNotExist
func getFile[F File](m map[string]F, name string) (File, error) {
name = Clean(name)
if name == Separator {
return &Dir{}, nil
}
f, ok := m[name]
if ok {
return f, nil
}
for p := range m {
if strings.HasPrefix(p, name) {
return &Dir{}, nil
}
}
return nil, ErrNotExist
}
func listFilesInDir[F File](m map[string]F, name string) (map[string]File, error) {
name = Clean(name)
out := map[string]File{}
for p, f := range m {
if strings.HasPrefix(p, name) {
parts := strings.Split(trimRelPath(p, name), Separator)
if len(parts) == 1 {
out[parts[0]] = f
} else {
out[parts[0]] = &Dir{}
}
}
}
return out, nil
}
func trimRelPath(p, t string) string {
return strings.Trim(strings.TrimPrefix(p, t), "/")
}
func Clean(p string) string {
return path.Clean(Separator + strings.ReplaceAll(p, "\\", "/"))
}

87
src/http/api.go
View file

@ -6,9 +6,8 @@ import (
"math" "math"
"net/http" "net/http"
"os" "os"
"sort"
"git.kmsign.ru/royalcat/tstor/src/torrent" "git.kmsign.ru/royalcat/tstor/src/host/torrent"
"github.com/anacrolix/missinggo/v2/filecache" "github.com/anacrolix/missinggo/v2/filecache"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
) )
@ -30,56 +29,56 @@ var apiStatusHandler = func(fc *filecache.Cache, ss *torrent.Stats) gin.HandlerF
} }
} }
var apiServersHandler = func(ss []*torrent.Server) gin.HandlerFunc { // var apiServersHandler = func(ss []*torrent.Server) gin.HandlerFunc {
return func(ctx *gin.Context) { // return func(ctx *gin.Context) {
var infos []*torrent.ServerInfo // var infos []*torrent.ServerInfo
for _, s := range ss { // for _, s := range ss {
infos = append(infos, s.Info()) // infos = append(infos, s.Info())
} // }
ctx.JSON(http.StatusOK, infos) // ctx.JSON(http.StatusOK, infos)
} // }
} // }
var apiRoutesHandler = func(ss *torrent.Stats) gin.HandlerFunc { // var apiRoutesHandler = func(ss *torrent.Stats) gin.HandlerFunc {
return func(ctx *gin.Context) { // return func(ctx *gin.Context) {
s := ss.RoutesStats() // s := ss.RoutesStats()
sort.Sort(torrent.ByName(s)) // sort.Sort(torrent.ByName(s))
ctx.JSON(http.StatusOK, s) // ctx.JSON(http.StatusOK, s)
} // }
} // }
var apiAddTorrentHandler = func(s *torrent.Service) gin.HandlerFunc { // var apiAddTorrentHandler = func(s *torrent.Service) gin.HandlerFunc {
return func(ctx *gin.Context) { // return func(ctx *gin.Context) {
route := ctx.Param("route") // route := ctx.Param("route")
var json RouteAdd // var json RouteAdd
if err := ctx.ShouldBindJSON(&json); err != nil { // if err := ctx.ShouldBindJSON(&json); err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) // ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return // return
} // }
if err := s.AddMagnet(route, json.Magnet); err != nil { // if err := s.AddMagnet(route, json.Magnet); err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) // ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return // return
} // }
ctx.JSON(http.StatusOK, nil) // ctx.JSON(http.StatusOK, nil)
} // }
} // }
var apiDelTorrentHandler = func(s *torrent.Service) gin.HandlerFunc { // var apiDelTorrentHandler = func(s *torrent.Service) gin.HandlerFunc {
return func(ctx *gin.Context) { // return func(ctx *gin.Context) {
route := ctx.Param("route") // route := ctx.Param("route")
hash := ctx.Param("torrent_hash") // hash := ctx.Param("torrent_hash")
if err := s.RemoveFromHash(route, hash); err != nil { // if err := s.RemoveFromHash(route, hash); err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) // ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return // return
} // }
ctx.JSON(http.StatusOK, nil) // ctx.JSON(http.StatusOK, nil)
} // }
} // }
var apiLogHandler = func(path string) gin.HandlerFunc { var apiLogHandler = func(path string) gin.HandlerFunc {
return func(ctx *gin.Context) { return func(ctx *gin.Context) {

22
src/http/http.go
View file

@ -6,14 +6,14 @@ import (
"git.kmsign.ru/royalcat/tstor" "git.kmsign.ru/royalcat/tstor"
"git.kmsign.ru/royalcat/tstor/src/config" "git.kmsign.ru/royalcat/tstor/src/config"
"git.kmsign.ru/royalcat/tstor/src/torrent" "git.kmsign.ru/royalcat/tstor/src/host/torrent"
"github.com/anacrolix/missinggo/v2/filecache" "github.com/anacrolix/missinggo/v2/filecache"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/shurcooL/httpfs/html/vfstemplate" "github.com/shurcooL/httpfs/html/vfstemplate"
) )
func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.Config, tss []*torrent.Server, fs http.FileSystem, logPath string, cfg *config.Config) error { func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, logPath string, cfg *config.Config) error {
gin.SetMode(gin.ReleaseMode) gin.SetMode(gin.ReleaseMode)
r := gin.New() r := gin.New()
r.Use(gin.Recovery()) r.Use(gin.Recovery())
@ -24,14 +24,6 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.
c.FileFromFS(c.Request.URL.Path, http.FS(tstor.Assets)) c.FileFromFS(c.Request.URL.Path, http.FS(tstor.Assets))
}) })
if cfg.Mounts.HttpFs.Enabled {
log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d/fs", cfg.Mounts.HttpFs.Port)).Msg("starting HTTPFS")
r.GET("/fs/*filepath", func(c *gin.Context) {
path := c.Param("filepath")
c.FileFromFS(path, fs)
})
}
t, err := vfstemplate.ParseGlob(http.FS(tstor.Templates), nil, "/templates/*") t, err := vfstemplate.ParseGlob(http.FS(tstor.Templates), nil, "/templates/*")
if err != nil { if err != nil {
return fmt.Errorf("error parsing html: %w", err) return fmt.Errorf("error parsing html: %w", err)
@ -40,7 +32,7 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.
r.SetHTMLTemplate(t) r.SetHTMLTemplate(t)
r.GET("/", indexHandler) r.GET("/", indexHandler)
r.GET("/routes", routesHandler(ss)) // r.GET("/routes", routesHandler(ss))
r.GET("/logs", logsHandler) r.GET("/logs", logsHandler)
r.GET("/servers", serversFoldersHandler()) r.GET("/servers", serversFoldersHandler())
@ -48,11 +40,11 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.
{ {
api.GET("/log", apiLogHandler(logPath)) api.GET("/log", apiLogHandler(logPath))
api.GET("/status", apiStatusHandler(fc, ss)) api.GET("/status", apiStatusHandler(fc, ss))
api.GET("/servers", apiServersHandler(tss)) // api.GET("/servers", apiServersHandler(tss))
api.GET("/routes", apiRoutesHandler(ss)) // api.GET("/routes", apiRoutesHandler(ss))
api.POST("/routes/:route/torrent", apiAddTorrentHandler(s)) // api.POST("/routes/:route/torrent", apiAddTorrentHandler(s))
api.DELETE("/routes/:route/torrent/:torrent_hash", apiDelTorrentHandler(s)) // api.DELETE("/routes/:route/torrent/:torrent_hash", apiDelTorrentHandler(s))
} }

11
src/http/web.go
View file

@ -3,7 +3,6 @@ package http
import ( import (
"net/http" "net/http"
"git.kmsign.ru/royalcat/tstor/src/torrent"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
) )
@ -11,11 +10,11 @@ var indexHandler = func(c *gin.Context) {
c.HTML(http.StatusOK, "index.html", nil) c.HTML(http.StatusOK, "index.html", nil)
} }
var routesHandler = func(ss *torrent.Stats) gin.HandlerFunc { // var routesHandler = func(ss *torrent.Stats) gin.HandlerFunc {
return func(c *gin.Context) { // return func(c *gin.Context) {
c.HTML(http.StatusOK, "routes.html", ss.RoutesStats()) // c.HTML(http.StatusOK, "routes.html", ss.RoutesStats())
} // }
} // }
var logsHandler = func(c *gin.Context) { var logsHandler = func(c *gin.Context) {
c.HTML(http.StatusOK, "logs.html", nil) c.HTML(http.StatusOK, "logs.html", nil)

4
src/iio/wrapper_test.go
View file

@ -4,7 +4,7 @@ import (
"io" "io"
"testing" "testing"
"git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"git.kmsign.ru/royalcat/tstor/src/iio" "git.kmsign.ru/royalcat/tstor/src/iio"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -16,7 +16,7 @@ func TestSeekerWrapper(t *testing.T) {
require := require.New(t) require := require.New(t)
mf := fs.NewMemoryFile(testData) mf := vfs.NewMemoryFile(testData)
r := iio.NewSeekerWrapper(mf, mf.Size()) r := iio.NewSeekerWrapper(mf, mf.Size())
defer r.Close() defer r.Close()

11
src/mounts/fuse/handler.go
View file

@ -5,7 +5,7 @@ import (
"path/filepath" "path/filepath"
"runtime" "runtime"
"git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"github.com/billziss-gh/cgofuse/fuse" "github.com/billziss-gh/cgofuse/fuse"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
@ -25,7 +25,7 @@ func NewHandler(fuseAllowOther bool, path string) *Handler {
} }
} }
func (s *Handler) Mount(fss map[string]fs.Filesystem) error { func (s *Handler) Mount(vfs vfs.Filesystem) error {
folder := s.path folder := s.path
// On windows, the folder must don't exist // On windows, the folder must don't exist
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
@ -38,12 +38,7 @@ func (s *Handler) Mount(fss map[string]fs.Filesystem) error {
} }
} }
cfs, err := fs.NewContainerFs(fss) host := fuse.NewFileSystemHost(NewFS(vfs))
if err != nil {
return err
}
host := fuse.NewFileSystemHost(NewFS(cfs))
// TODO improve error handling here // TODO improve error handling here
go func() { go func() {

14
src/mounts/fuse/mount.go
View file

@ -7,7 +7,7 @@ import (
"os" "os"
"sync" "sync"
"git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"github.com/billziss-gh/cgofuse/fuse" "github.com/billziss-gh/cgofuse/fuse"
"github.com/rs/zerolog" "github.com/rs/zerolog"
@ -21,7 +21,7 @@ type FS struct {
log zerolog.Logger log zerolog.Logger
} }
func NewFS(fs fs.Filesystem) fuse.FileSystemInterface { func NewFS(fs vfs.Filesystem) fuse.FileSystemInterface {
l := log.Logger.With().Str("component", "fuse").Logger() l := log.Logger.With().Str("component", "fuse").Logger()
return &FS{ return &FS{
fh: &fileHandler{fs: fs}, fh: &fileHandler{fs: fs},
@ -154,11 +154,11 @@ var ErrBadHolderIndex = errors.New("holder index too big")
type fileHandler struct { type fileHandler struct {
mu sync.RWMutex mu sync.RWMutex
opened []fs.File opened []vfs.File
fs fs.Filesystem fs vfs.Filesystem
} }
func (fh *fileHandler) GetFile(path string, fhi uint64) (fs.File, error) { func (fh *fileHandler) GetFile(path string, fhi uint64) (vfs.File, error) {
fh.mu.RLock() fh.mu.RLock()
defer fh.mu.RUnlock() defer fh.mu.RUnlock()
@ -204,7 +204,7 @@ func (fh *fileHandler) OpenHolder(path string) (uint64, error) {
return uint64(len(fh.opened) - 1), nil return uint64(len(fh.opened) - 1), nil
} }
func (fh *fileHandler) get(fhi uint64) (fs.File, error) { func (fh *fileHandler) get(fhi uint64) (vfs.File, error) {
if int(fhi) >= len(fh.opened) { if int(fhi) >= len(fh.opened) {
return nil, ErrBadHolderIndex return nil, ErrBadHolderIndex
} }
@ -240,7 +240,7 @@ func (fh *fileHandler) Remove(fhi uint64) error {
return nil return nil
} }
func (fh *fileHandler) lookupFile(path string) (fs.File, error) { func (fh *fileHandler) lookupFile(path string) (vfs.File, error) {
file, err := fh.fs.Open(path) file, err := fh.fs.Open(path)
if err != nil { if err != nil {
return nil, err return nil, err

20
src/mounts/fuse/mount_test.go
View file

@ -7,7 +7,7 @@ import (
"testing" "testing"
"time" "time"
"git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -22,12 +22,11 @@ func TestHandler(t *testing.T) {
h := NewHandler(false, p) h := NewHandler(false, p)
mem := fs.NewMemory() mem := vfs.NewMemoryFS(map[string]*vfs.MemoryFile{
"/test.txt": vfs.NewMemoryFile([]byte("test")),
})
err := mem.Storage.Add(fs.NewMemoryFile([]byte("test")), "/test.txt") err := h.Mount(mem)
require.NoError(err)
err = h.Mount(map[string]fs.Filesystem{"/mem": mem})
require.NoError(err) require.NoError(err)
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
@ -50,12 +49,11 @@ func TestHandlerDriveLetter(t *testing.T) {
h := NewHandler(false, p) h := NewHandler(false, p)
mem := fs.NewMemory() mem := vfs.NewMemoryFS(map[string]*vfs.MemoryFile{
"/test.txt": vfs.NewMemoryFile([]byte("test")),
})
err := mem.Storage.Add(fs.NewMemoryFile([]byte("test")), "/test.txt") err := h.Mount(mem)
require.NoError(err)
err = h.Mount(map[string]fs.Filesystem{"/mem": mem})
require.NoError(err) require.NoError(err)
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)

12
src/mounts/httpfs/httpfs.go
View file

@ -7,17 +7,17 @@ import (
"os" "os"
"sync" "sync"
dfs "git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"git.kmsign.ru/royalcat/tstor/src/iio" "git.kmsign.ru/royalcat/tstor/src/iio"
) )
var _ http.FileSystem = &HTTPFS{} var _ http.FileSystem = &HTTPFS{}
type HTTPFS struct { type HTTPFS struct {
fs dfs.Filesystem fs vfs.Filesystem
} }
func NewHTTPFS(fs dfs.Filesystem) *HTTPFS { func NewHTTPFS(fs vfs.Filesystem) *HTTPFS {
return &HTTPFS{fs: fs} return &HTTPFS{fs: fs}
} }
@ -27,7 +27,7 @@ func (fs *HTTPFS) Open(name string) (http.File, error) {
return nil, err return nil, err
} }
fi := dfs.NewFileInfo(name, f.Size(), f.IsDir()) fi := vfs.NewFileInfo(name, f.Size(), f.IsDir())
// TODO make this lazy // TODO make this lazy
fis, err := fs.filesToFileInfo(name) fis, err := fs.filesToFileInfo(name)
@ -46,7 +46,7 @@ func (fs *HTTPFS) filesToFileInfo(path string) ([]fs.FileInfo, error) {
var out []os.FileInfo var out []os.FileInfo
for n, f := range files { for n, f := range files {
out = append(out, dfs.NewFileInfo(n, f.Size(), f.IsDir())) out = append(out, vfs.NewFileInfo(n, f.Size(), f.IsDir()))
} }
return out, nil return out, nil
@ -65,7 +65,7 @@ type httpFile struct {
fi fs.FileInfo fi fs.FileInfo
} }
func newHTTPFile(f dfs.File, fis []fs.FileInfo, fi fs.FileInfo) *httpFile { func newHTTPFile(f vfs.File, fis []fs.FileInfo, fi fs.FileInfo) *httpFile {
return &httpFile{ return &httpFile{
dirContent: fis, dirContent: fis,
fi: fi, fi: fi,

10
src/mounts/webdav/fs.go
View file

@ -8,7 +8,7 @@ import (
"sync" "sync"
"time" "time"
"git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"git.kmsign.ru/royalcat/tstor/src/iio" "git.kmsign.ru/royalcat/tstor/src/iio"
"golang.org/x/net/webdav" "golang.org/x/net/webdav"
) )
@ -16,10 +16,10 @@ import (
var _ webdav.FileSystem = &WebDAV{} var _ webdav.FileSystem = &WebDAV{}
type WebDAV struct { type WebDAV struct {
fs fs.Filesystem fs vfs.Filesystem
} }
func newFS(fs fs.Filesystem) *WebDAV { func newFS(fs vfs.Filesystem) *WebDAV {
return &WebDAV{fs: fs} return &WebDAV{fs: fs}
} }
@ -59,7 +59,7 @@ func (wd *WebDAV) Rename(ctx context.Context, oldName, newName string) error {
return webdav.ErrNotImplemented return webdav.ErrNotImplemented
} }
func (wd *WebDAV) lookupFile(path string) (fs.File, error) { func (wd *WebDAV) lookupFile(path string) (vfs.File, error) {
return wd.fs.Open(path) return wd.fs.Open(path)
} }
@ -93,7 +93,7 @@ type webDAVFile struct {
dirContent []os.FileInfo dirContent []os.FileInfo
} }
func newFile(name string, f fs.File, df func() ([]os.FileInfo, error)) *webDAVFile { func newFile(name string, f vfs.File, df func() ([]os.FileInfo, error)) *webDAVFile {
return &webDAVFile{ return &webDAVFile{
fi: newFileInfo(name, f.Size(), f.IsDir()), fi: newFileInfo(name, f.Size(), f.IsDir()),
dirFunc: df, dirFunc: df,

16
src/mounts/webdav/fs_test.go
View file

@ -6,7 +6,7 @@ import (
"os" "os"
"testing" "testing"
"git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"golang.org/x/net/webdav" "golang.org/x/net/webdav"
) )
@ -16,10 +16,9 @@ func TestWebDAVFilesystem(t *testing.T) {
require := require.New(t) require := require.New(t)
mfs := fs.NewMemory() mfs := vfs.NewMemoryFS(map[string]*vfs.MemoryFile{
mf := fs.NewMemoryFile([]byte("test file content.")) "/folder/file.txt": vfs.NewMemoryFile([]byte("test file content.")),
err := mfs.Storage.Add(mf, "/folder/file.txt") })
require.NoError(err)
wfs := newFS(mfs) wfs := newFS(mfs)
@ -67,10 +66,9 @@ func TestErrNotImplemented(t *testing.T) {
require := require.New(t) require := require.New(t)
mfs := fs.NewMemory() mfs := vfs.NewMemoryFS(map[string]*vfs.MemoryFile{
mf := fs.NewMemoryFile([]byte("test file content.")) "/folder/file.txt": vfs.NewMemoryFile([]byte("test file content.")),
err := mfs.Storage.Add(mf, "/folder/file.txt") })
require.NoError(err)
wfs := newFS(mfs) wfs := newFS(mfs)

4
src/mounts/webdav/handler.go
View file

@ -3,12 +3,12 @@ package webdav
import ( import (
"net/http" "net/http"
"git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"golang.org/x/net/webdav" "golang.org/x/net/webdav"
) )
func newHandler(fs fs.Filesystem) *webdav.Handler { func newHandler(fs vfs.Filesystem) *webdav.Handler {
l := log.Logger.With().Str("component", "webDAV").Logger() l := log.Logger.With().Str("component", "webDAV").Logger()
return &webdav.Handler{ return &webdav.Handler{
Prefix: "/", Prefix: "/",

4
src/mounts/webdav/http.go
View file

@ -4,11 +4,11 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
func NewWebDAVServer(fs fs.Filesystem, port int, user, pass string) error { func NewWebDAVServer(fs vfs.Filesystem, port int, user, pass string) error {
log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", port)).Msg("starting webDAV server") log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", port)).Msg("starting webDAV server")
srv := newHandler(fs) srv := newHandler(fs)

45
src/torrent/loader/config.go
View file

@ -1,45 +0,0 @@
package loader
import "git.kmsign.ru/royalcat/tstor/src/config"
var _ Loader = &Config{}
type Config struct {
c []config.Route
}
func NewConfig(r []config.Route) *Config {
return &Config{
c: r,
}
}
func (l *Config) ListMagnets() (map[string][]string, error) {
out := make(map[string][]string)
for _, r := range l.c {
for _, t := range r.Torrents {
if t.MagnetURI == "" {
continue
}
out[r.Name] = append(out[r.Name], t.MagnetURI)
}
}
return out, nil
}
func (l *Config) ListTorrentPaths() (map[string][]string, error) {
out := make(map[string][]string)
for _, r := range l.c {
for _, t := range r.Torrents {
if t.TorrentPath == "" {
continue
}
out[r.Name] = append(out[r.Name], t.TorrentPath)
}
}
return out, nil
}

112
src/torrent/loader/db.go
View file

@ -1,112 +0,0 @@
package loader
import (
"path"
dlog "git.kmsign.ru/royalcat/tstor/src/log"
"github.com/anacrolix/torrent/metainfo"
"github.com/dgraph-io/badger/v3"
"github.com/rs/zerolog/log"
)
var _ LoaderAdder = &DB{}
const routeRootKey = "/route/"
type DB struct {
db *badger.DB
}
func NewDB(path string) (*DB, error) {
l := log.Logger.With().Str("component", "torrent-store").Logger()
opts := badger.DefaultOptions(path).
WithLogger(&dlog.Badger{L: l}).
WithValueLogFileSize(1<<26 - 1)
db, err := badger.Open(opts)
if err != nil {
return nil, err
}
err = db.RunValueLogGC(0.5)
if err != nil && err != badger.ErrNoRewrite {
return nil, err
}
return &DB{
db: db,
}, nil
}
func (l *DB) AddMagnet(r, m string) error {
err := l.db.Update(func(txn *badger.Txn) error {
spec, err := metainfo.ParseMagnetUri(m)
if err != nil {
return err
}
ih := spec.InfoHash.HexString()
rp := path.Join(routeRootKey, ih, r)
return txn.Set([]byte(rp), []byte(m))
})
if err != nil {
return err
}
return l.db.Sync()
}
func (l *DB) RemoveFromHash(r, h string) (bool, error) {
tx := l.db.NewTransaction(true)
defer tx.Discard()
var mh metainfo.Hash
if err := mh.FromHexString(h); err != nil {
return false, err
}
rp := path.Join(routeRootKey, h, r)
if _, err := tx.Get([]byte(rp)); err != nil {
return false, nil
}
if err := tx.Delete([]byte(rp)); err != nil {
return false, err
}
return true, tx.Commit()
}
func (l *DB) ListMagnets() (map[string][]string, error) {
tx := l.db.NewTransaction(false)
defer tx.Discard()
it := tx.NewIterator(badger.DefaultIteratorOptions)
defer it.Close()
prefix := []byte(routeRootKey)
out := make(map[string][]string)
for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
_, r := path.Split(string(it.Item().Key()))
i := it.Item()
if err := i.Value(func(v []byte) error {
out[r] = append(out[r], string(v))
return nil
}); err != nil {
return nil, err
}
}
return out, nil
}
func (l *DB) ListTorrentPaths() (map[string][]string, error) {
return nil, nil
}
func (l *DB) Close() error {
return l.db.Close()
}

62
src/torrent/loader/db_test.go
View file

@ -1,62 +0,0 @@
package loader
import (
"os"
"testing"
"github.com/anacrolix/torrent/storage"
"github.com/stretchr/testify/require"
)
const m1 = "magnet:?xt=urn:btih:c9e15763f722f23e98a29decdfae341b98d53056"
func TestDB(t *testing.T) {
require := require.New(t)
tmpService, err := os.MkdirTemp("", "service")
require.NoError(err)
tmpStorage, err := os.MkdirTemp("", "storage")
require.NoError(err)
cs := storage.NewFile(tmpStorage)
defer cs.Close()
s, err := NewDB(tmpService)
require.NoError(err)
defer s.Close()
err = s.AddMagnet("route1", "WRONG MAGNET")
require.Error(err)
err = s.AddMagnet("route1", m1)
require.NoError(err)
err = s.AddMagnet("route2", m1)
require.NoError(err)
l, err := s.ListMagnets()
require.NoError(err)
require.Len(l, 2)
require.Len(l["route1"], 1)
require.Equal(l["route1"][0], m1)
require.Len(l["route2"], 1)
require.Equal(l["route2"][0], m1)
removed, err := s.RemoveFromHash("other", "c9e15763f722f23e98a29decdfae341b98d53056")
require.NoError(err)
require.False(removed)
removed, err = s.RemoveFromHash("route1", "c9e15763f722f23e98a29decdfae341b98d53056")
require.NoError(err)
require.True(removed)
l, err = s.ListMagnets()
require.NoError(err)
require.Len(l, 1)
require.Len(l["route2"], 1)
require.Equal(l["route2"][0], m1)
require.NoError(s.Close())
require.NoError(cs.Close())
}

55
src/torrent/loader/folder.go
View file

@ -1,55 +0,0 @@
package loader
import (
"io/fs"
"path"
"path/filepath"
"git.kmsign.ru/royalcat/tstor/src/config"
)
var _ Loader = &Folder{}
type Folder struct {
c []config.Route
}
func NewFolder(r []config.Route) *Folder {
return &Folder{
c: r,
}
}
func (f *Folder) ListMagnets() (map[string][]string, error) {
return nil, nil
}
func (f *Folder) ListTorrentPaths() (map[string][]string, error) {
out := make(map[string][]string)
for _, r := range f.c {
if r.TorrentFolder == "" {
continue
}
err := filepath.WalkDir(r.TorrentFolder, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if path.Ext(p) == ".torrent" {
out[r.Name] = append(out[r.Name], p)
}
return nil
})
if err != nil {
return nil, err
}
}
return out, nil
}

13
src/torrent/loader/loader.go
View file

@ -1,13 +0,0 @@
package loader
type Loader interface {
ListMagnets() (map[string][]string, error)
ListTorrentPaths() (map[string][]string, error)
}
type LoaderAdder interface {
Loader
RemoveFromHash(r, h string) (bool, error)
AddMagnet(r, m string) error
}

255
src/torrent/server.go
View file

@ -1,255 +0,0 @@
package torrent
import (
"fmt"
"os"
"path/filepath"
"sync"
"sync/atomic"
"time"
"git.kmsign.ru/royalcat/tstor/src/config"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/bencode"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/storage"
"github.com/fsnotify/fsnotify"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
type ServerState int
const (
UNKNOWN ServerState = iota
SEEDING
READING
UPDATING
STOPPED
ERROR
)
func (ss ServerState) String() string {
return [...]string{"Unknown", "Seeding", "Reading", "Updating", "Stopped", "Error"}[ss]
}
type ServerInfo struct {
Magnet string `json:"magnetUri"`
UpdatedAt int64 `json:"updatedAt"`
Name string `json:"name"`
Folder string `json:"folder"`
State string `json:"state"`
Peers int `json:"peers"`
Seeds int `json:"seeds"`
}
type Server struct {
cfg *config.Server
log zerolog.Logger
fw *fsnotify.Watcher
eventsCount uint64
c *torrent.Client
pc storage.PieceCompletion
mu sync.RWMutex
t *torrent.Torrent
si ServerInfo
}
func NewServer(c *torrent.Client, pc storage.PieceCompletion, cfg *config.Server) *Server {
l := log.Logger.With().Str("component", "server").Str("name", cfg.Name).Logger()
return &Server{
cfg: cfg,
log: l,
c: c,
pc: pc,
}
}
func (s *Server) Start() error {
s.log.Info().Msg("starting new server folder")
w, err := fsnotify.NewWatcher()
if err != nil {
return err
}
if err := os.MkdirAll(s.cfg.Path, 0744); err != nil {
return fmt.Errorf("error creating server folder: %s. Error: %w", s.cfg.Path, err)
}
if err := filepath.Walk(s.cfg.Path,
func(path string, info os.FileInfo, err error) error {
if info.Mode().IsDir() {
s.log.Debug().Str("folder", path).Msg("adding new folder")
return w.Add(path)
}
return nil
}); err != nil {
return err
}
s.fw = w
go func() {
if err := s.makeMagnet(); err != nil {
s.updateState(ERROR)
s.log.Error().Err(err).Msg("error generating magnet on start")
}
s.watch()
}()
go func() {
for {
select {
case event, ok := <-w.Events:
if !ok {
return
}
s.log.Info().Str("file", event.Name).Str("op", event.Op.String()).Msg("file changed inside server folder")
atomic.AddUint64(&s.eventsCount, 1)
case err, ok := <-w.Errors:
if !ok {
return
}
s.updateState(STOPPED)
s.log.Error().Err(err).Msg("error watching server folder")
}
}
}()
s.log.Info().Msg("server folder started")
return nil
}
func (s *Server) watch() {
s.log.Info().Msg("starting watcher")
for range time.Tick(time.Second * 5) {
if s.eventsCount == 0 {
continue
}
ec := s.eventsCount
if err := s.makeMagnet(); err != nil {
s.updateState(ERROR)
s.log.Error().Err(err).Msg("error generating magnet")
}
atomic.AddUint64(&s.eventsCount, -ec)
}
}
func (s *Server) makeMagnet() error {
s.log.Info().Msg("starting serving new torrent")
info := metainfo.Info{
PieceLength: 2 << 18,
}
s.updateState(READING)
if err := info.BuildFromFilePath(s.cfg.Path); err != nil {
return err
}
s.updateState(UPDATING)
if len(info.Files) == 0 {
s.mu.Lock()
s.si.Magnet = ""
s.si.Folder = s.cfg.Path
s.si.Name = s.cfg.Name
s.si.UpdatedAt = time.Now().Unix()
s.mu.Unlock()
s.log.Info().Msg("not creating magnet from empty folder")
s.updateState(STOPPED)
return nil
}
mi := metainfo.MetaInfo{
InfoBytes: bencode.MustMarshal(info),
}
ih := mi.HashInfoBytes()
to, _ := s.c.AddTorrentOpt(torrent.AddTorrentOpts{
InfoHash: ih,
Storage: storage.NewFileOpts(storage.NewFileClientOpts{
ClientBaseDir: s.cfg.Path,
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
return filepath.Join(opts.File.Path...)
},
TorrentDirMaker: nil,
PieceCompletion: s.pc,
}),
})
tks := s.trackers()
err := to.MergeSpec(&torrent.TorrentSpec{
InfoBytes: mi.InfoBytes,
Trackers: [][]string{tks},
})
if err != nil {
return err
}
m := metainfo.Magnet{
InfoHash: ih,
DisplayName: s.cfg.Name,
Trackers: tks,
}
s.mu.Lock()
s.t = to
s.si.Magnet = m.String()
s.si.Folder = s.cfg.Path
s.si.Name = s.cfg.Name
s.si.UpdatedAt = time.Now().Unix()
s.mu.Unlock()
s.updateState(SEEDING)
s.log.Info().Str("hash", ih.HexString()).Msg("new torrent is ready")
return nil
}
func (s *Server) updateState(ss ServerState) {
s.mu.Lock()
s.si.State = ss.String()
s.mu.Unlock()
}
func (s *Server) trackers() []string {
// TODO load trackers from URL too
return s.cfg.Trackers
}
func (s *Server) Close() error {
if s.fw == nil {
return nil
}
return s.fw.Close()
}
func (s *Server) Info() *ServerInfo {
s.mu.RLock()
defer s.mu.RUnlock()
if s.t != nil {
st := s.t.Stats()
s.si.Peers = st.TotalPeers
s.si.Seeds = st.ConnectedSeeders
}
return &s.si
}

203
src/torrent/service.go
View file

@ -1,203 +0,0 @@
package torrent
import (
"errors"
"fmt"
"path"
"sync"
"time"
"git.kmsign.ru/royalcat/tstor/src/fs"
"git.kmsign.ru/royalcat/tstor/src/torrent/loader"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
type Service struct {
c *torrent.Client
s *Stats
mu sync.Mutex
fss map[string]fs.Filesystem
loaders []loader.Loader
db loader.LoaderAdder
log zerolog.Logger
addTimeout, readTimeout int
}
func NewService(loaders []loader.Loader, db loader.LoaderAdder, stats *Stats, c *torrent.Client, addTimeout, readTimeout int) *Service {
l := log.Logger.With().Str("component", "torrent-service").Logger()
return &Service{
log: l,
s: stats,
c: c,
fss: make(map[string]fs.Filesystem),
loaders: loaders,
db: db,
addTimeout: addTimeout,
readTimeout: readTimeout,
}
}
func (s *Service) Load() (map[string]fs.Filesystem, error) {
// Load from config
s.log.Info().Msg("adding torrents from configuration")
for _, loader := range s.loaders {
if err := s.load(loader); err != nil {
return nil, err
}
}
// Load from DB
s.log.Info().Msg("adding torrents from database")
return s.fss, s.load(s.db)
}
func (s *Service) load(l loader.Loader) error {
list, err := l.ListMagnets()
if err != nil {
return err
}
for r, ms := range list {
s.addRoute(r)
for _, m := range ms {
if err := s.addMagnet(r, m); err != nil {
return err
}
}
}
list, err = l.ListTorrentPaths()
if err != nil {
return err
}
for r, ms := range list {
s.addRoute(r)
for _, p := range ms {
if err := s.addTorrentPath(r, p); err != nil {
return err
}
}
}
return nil
}
func (s *Service) AddMagnet(r, m string) error {
if err := s.addMagnet(r, m); err != nil {
return err
}
// Add to db
return s.db.AddMagnet(r, m)
}
func (s *Service) addTorrentPath(r, p string) error {
// Add to client
t, err := s.c.AddTorrentFromFile(p)
if err != nil {
return err
}
return s.addTorrent(r, t)
}
func (s *Service) addMagnet(r, m string) error {
// Add to client
t, err := s.c.AddMagnet(m)
if err != nil {
return err
}
return s.addTorrent(r, t)
}
func (s *Service) addRoute(r string) {
s.s.AddRoute(r)
// Add to filesystems
folder := path.Join("/", r)
s.mu.Lock()
defer s.mu.Unlock()
_, ok := s.fss[folder]
if !ok {
s.fss[folder] = fs.NewTorrent(s.readTimeout)
}
}
func (s *Service) addTorrent(r string, t *torrent.Torrent) error {
// only get info if name is not available
if t.Info() == nil {
s.log.Info().Str("hash", t.InfoHash().String()).Msg("getting torrent info")
select {
case <-time.After(time.Duration(s.addTimeout) * time.Second):
s.log.Error().Str("hash", t.InfoHash().String()).Msg("timeout getting torrent info")
return errors.New("timeout getting torrent info")
case <-t.GotInfo():
s.log.Info().Str("hash", t.InfoHash().String()).Msg("obtained torrent info")
}
}
// Add to stats
s.s.Add(r, t)
// Add to filesystems
folder := path.Join("/", r)
s.mu.Lock()
defer s.mu.Unlock()
tfs, ok := s.fss[folder].(*fs.Torrent)
if !ok {
return errors.New("error adding torrent to filesystem")
}
tfs.AddTorrent(t)
s.log.Info().Str("name", t.Info().Name).Str("route", r).Msg("torrent added")
return nil
}
func (s *Service) RemoveFromHash(r, h string) error {
// Remove from db
deleted, err := s.db.RemoveFromHash(r, h)
if err != nil {
return err
}
if !deleted {
return fmt.Errorf("element with hash %v on route %v cannot be removed", h, r)
}
// Remove from stats
s.s.Del(r, h)
// Remove from fs
folder := path.Join("/", r)
tfs, ok := s.fss[folder].(*fs.Torrent)
if !ok {
return errors.New("error removing torrent from filesystem")
}
tfs.RemoveTorrent(h)
// Remove from client
var mh metainfo.Hash
if err := mh.FromHexString(h); err != nil {
return err
}
t, ok := s.c.Torrent(metainfo.NewHashFromHex(h))
if ok {
t.Drop()
}
return nil
}