From d30ef6cc9cfcce4ead3f88520e5bb90f421b24e6 Mon Sep 17 00:00:00 2001 From: royalcat Date: Mon, 16 Oct 2023 12:18:40 +0300 Subject: [PATCH] rework 2 (working) --- cmd/tstor/main.go | 114 ++++++------ go.mod | 4 +- go.sum | 33 ---- src/config/default.go | 6 +- src/config/model.go | 5 +- src/fs/container.go | 24 --- src/fs/container_test.go | 28 --- src/fs/memory_test.go | 39 ---- src/fs/storage.go | 184 ------------------ src/fs/storage_test.go | 195 ------------------- src/host/storage.go | 120 ++++++++++++ src/{ => host}/torrent/client.go | 0 src/{ => host}/torrent/id.go | 0 src/host/torrent/service.go | 213 +++++++++++++++++++++ src/{ => host}/torrent/stats.go | 82 ++------ src/{ => host}/torrent/store.go | 0 src/{fs => host/vfs}/archive.go | 267 ++++++++++++--------------- src/{fs => host/vfs}/archive_test.go | 4 +- src/{fs => host/vfs}/dir.go | 2 +- src/{fs => host/vfs}/fs.go | 2 +- src/{fs => host/vfs}/fs_test.go | 2 +- src/{fs => host/vfs}/memory.go | 22 +-- src/host/vfs/memory_test.go | 46 +++++ src/host/vfs/os.go | 167 +++++++++++++++++ src/host/vfs/resolver.go | 146 +++++++++++++++ src/host/vfs/resolver_test.go | 192 +++++++++++++++++++ src/{fs => host/vfs}/torrent.go | 99 +++++----- src/{fs => host/vfs}/torrent_test.go | 76 ++++---- src/host/vfs/utils.go | 55 ++++++ src/http/api.go | 87 +++++---- src/http/http.go | 22 +-- src/http/web.go | 11 +- src/iio/wrapper_test.go | 4 +- src/mounts/fuse/handler.go | 11 +- src/mounts/fuse/mount.go | 14 +- src/mounts/fuse/mount_test.go | 20 +- src/mounts/httpfs/httpfs.go | 12 +- src/mounts/webdav/fs.go | 10 +- src/mounts/webdav/fs_test.go | 16 +- src/mounts/webdav/handler.go | 4 +- src/mounts/webdav/http.go | 4 +- src/torrent/loader/config.go | 45 ----- src/torrent/loader/db.go | 112 ----------- src/torrent/loader/db_test.go | 62 ------- src/torrent/loader/folder.go | 55 ------ src/torrent/loader/loader.go | 13 -- src/torrent/server.go | 255 ------------------------- src/torrent/service.go | 203 -------------------- 48 files changed, 1340 insertions(+), 1747 deletions(-) delete mode 100644 src/fs/container.go delete mode 100644 src/fs/container_test.go delete mode 100644 src/fs/memory_test.go delete mode 100644 src/fs/storage.go delete mode 100644 src/fs/storage_test.go create mode 100644 src/host/storage.go rename src/{ => host}/torrent/client.go (100%) rename src/{ => host}/torrent/id.go (100%) create mode 100644 src/host/torrent/service.go rename src/{ => host}/torrent/stats.go (74%) rename src/{ => host}/torrent/store.go (100%) rename src/{fs => host/vfs}/archive.go (58%) rename src/{fs => host/vfs}/archive_test.go (95%) rename src/{fs => host/vfs}/dir.go (96%) rename src/{fs => host/vfs}/fs.go (98%) rename src/{fs => host/vfs}/fs_test.go (97%) rename src/{fs => host/vfs}/memory.go (50%) create mode 100644 src/host/vfs/memory_test.go create mode 100644 src/host/vfs/os.go create mode 100644 src/host/vfs/resolver.go create mode 100644 src/host/vfs/resolver_test.go rename src/{fs => host/vfs}/torrent.go (62%) rename src/{fs => host/vfs}/torrent_test.go (62%) create mode 100644 src/host/vfs/utils.go delete mode 100644 src/torrent/loader/config.go delete mode 100644 src/torrent/loader/db.go delete mode 100644 src/torrent/loader/db_test.go delete mode 100644 src/torrent/loader/folder.go delete mode 100644 src/torrent/loader/loader.go delete mode 100644 src/torrent/server.go delete mode 100644 src/torrent/service.go diff --git a/cmd/tstor/main.go b/cmd/tstor/main.go index 71cc7d9..979118d 100644 --- a/cmd/tstor/main.go +++ b/cmd/tstor/main.go @@ -11,18 +11,18 @@ import ( "time" "git.kmsign.ru/royalcat/tstor/src/config" + "git.kmsign.ru/royalcat/tstor/src/host" + "git.kmsign.ru/royalcat/tstor/src/host/torrent" "github.com/anacrolix/torrent/storage" + "github.com/gin-gonic/gin" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" - "git.kmsign.ru/royalcat/tstor/src/fs" "git.kmsign.ru/royalcat/tstor/src/http" dlog "git.kmsign.ru/royalcat/tstor/src/log" "git.kmsign.ru/royalcat/tstor/src/mounts/fuse" "git.kmsign.ru/royalcat/tstor/src/mounts/httpfs" "git.kmsign.ru/royalcat/tstor/src/mounts/webdav" - "git.kmsign.ru/royalcat/tstor/src/torrent" - "git.kmsign.ru/royalcat/tstor/src/torrent/loader" ) const ( @@ -75,8 +75,8 @@ func setupStorage(tcfg config.TorrentClient) (storage.ClientImplCloser, storage. return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err) } - // TODO implement cache dir and storage capacity - // cacheDir := filepath.Join(tcfg.MetadataFolder, "cache") + // TODO implement cache/storage switching + // cacheDir := filepath.Join(tcfg.DataFolder, "cache") // if err := os.MkdirAll(cacheDir, 0744); err != nil { // return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err) // } @@ -84,8 +84,11 @@ func setupStorage(tcfg config.TorrentClient) (storage.ClientImplCloser, storage. // if err != nil { // return nil, nil, fmt.Errorf("error creating cache: %w", err) // } - // log.Info().Msg(fmt.Sprintf("setting cache size to %d MB", tcfg.GlobalCacheSize)) - // fc.SetCapacity(tcfg.GlobalCacheSize * 1024 * 1024) + // log.Info().Msg(fmt.Sprintf("setting cache size to %d MB", 1024)) + // fc.SetCapacity(1024 * 1024 * 1024) + + // rp := storage.NewResourcePieces(fc.AsResourceProvider()) + // st := &stc{rp} filesDir := filepath.Join(tcfg.DataFolder, "files") if err := os.MkdirAll(pcp, 0744); err != nil { @@ -97,6 +100,14 @@ func setupStorage(tcfg config.TorrentClient) (storage.ClientImplCloser, storage. return st, pc, nil } +type stc struct { + storage.ClientImpl +} + +func (s *stc) Close() error { + return nil +} + func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error { conf, err := config.Load(configPath) if err != nil { @@ -119,7 +130,7 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error { return fmt.Errorf("error creating node ID: %w", err) } - st, pc, err := setupStorage(conf.TorrentClient) + st, _, err := setupStorage(conf.TorrentClient) if err != nil { return err } @@ -128,26 +139,14 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error { if err != nil { return fmt.Errorf("error starting torrent client: %w", err) } + c.AddDhtNodes(conf.TorrentClient.DHTNodes) - var servers []*torrent.Server - for _, s := range conf.TorrentClient.Servers { - server := torrent.NewServer(c, pc, &s) - servers = append(servers, server) - if err := server.Start(); err != nil { - return fmt.Errorf("error starting server: %w", err) - } + ts := torrent.NewService(c, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout) + + if err := os.MkdirAll(conf.DataFolder, 0744); err != nil { + return fmt.Errorf("error creating data folder: %w", err) } - - cl := loader.NewConfig(conf.TorrentClient.Routes) - fl := loader.NewFolder(conf.TorrentClient.Routes) - ss := torrent.NewStats() - - dbl, err := loader.NewDB(filepath.Join(conf.TorrentClient.MetadataFolder, "magnetdb")) - if err != nil { - return fmt.Errorf("error starting magnet database: %w", err) - } - - ts := torrent.NewService([]loader.Loader{cl, fl}, dbl, ss, c, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout) + cfs := host.NewStorage(conf.DataFolder, ts) var mh *fuse.Handler if conf.Mounts.Fuse.Enabled { @@ -161,15 +160,13 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error { <-sigChan log.Info().Msg("closing servers...") - for _, s := range servers { - if err := s.Close(); err != nil { - log.Warn().Err(err).Msg("problem closing server") - } - } + // for _, s := range servers { + // if err := s.Close(); err != nil { + // log.Warn().Err(err).Msg("problem closing server") + // } + // } log.Info().Msg("closing items database...") fis.Close() - log.Info().Msg("closing magnet database...") - dbl.Close() log.Info().Msg("closing torrent client...") c.Close() if mh != nil { @@ -181,51 +178,46 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error { os.Exit(1) }() - fss, err := ts.Load() - if err != nil { - return fmt.Errorf("error when loading torrents: %w", err) - } - go func() { if mh == nil { return } - if err := mh.Mount(fss); err != nil { + if err := mh.Mount(cfs); err != nil { log.Info().Err(err).Msg("error mounting filesystems") } }() - go func() { - if conf.Mounts.WebDAV.Enabled { - port = webDAVPort - if port == 0 { - port = conf.Mounts.WebDAV.Port - } - - cfs, err := fs.NewContainerFs(fss) - if err != nil { - log.Error().Err(err).Msg("error adding files to webDAV") - return - } - - if err := webdav.NewWebDAVServer(cfs, port, conf.Mounts.WebDAV.User, conf.Mounts.WebDAV.Pass); err != nil { + if conf.Mounts.WebDAV.Enabled { + go func() { + if err := webdav.NewWebDAVServer(cfs, conf.Mounts.WebDAV.Port, conf.Mounts.WebDAV.User, conf.Mounts.WebDAV.Pass); err != nil { log.Error().Err(err).Msg("error starting webDAV") } - } - log.Warn().Msg("webDAV configuration not found!") - }() + log.Warn().Msg("webDAV configuration not found!") + }() + } + if conf.Mounts.HttpFs.Enabled { + go func() { + httpfs := httpfs.NewHTTPFS(cfs) - cfs, err := fs.NewContainerFs(fss) - if err != nil { - return fmt.Errorf("error when loading torrents: %w", err) + r := gin.New() + + r.GET("*filepath", func(c *gin.Context) { + path := c.Param("filepath") + c.FileFromFS(path, httpfs) + }) + + log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port)).Msg("starting HTTPFS") + if err := r.Run(fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port)); err != nil { + log.Error().Err(err).Msg("error starting HTTPFS") + } + }() } - httpfs := httpfs.NewHTTPFS(cfs) logFilename := filepath.Join(conf.Log.Path, dlog.FileName) - err = http.New(nil, ss, ts, conf, servers, httpfs, logFilename, conf) + err = http.New(nil, nil, ts, logFilename, conf) log.Error().Err(err).Msg("error initializing HTTP server") return err } diff --git a/go.mod b/go.mod index 5254a2f..421dc8d 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,7 @@ require ( github.com/anacrolix/torrent v1.52.6-0.20230929044811-45c91b322ad1 github.com/billziss-gh/cgofuse v1.5.0 github.com/bodgit/sevenzip v1.4.3 - github.com/dgraph-io/badger/v3 v3.2103.5 github.com/dgraph-io/badger/v4 v4.2.0 - github.com/fsnotify/fsnotify v1.6.0 github.com/gin-gonic/gin v1.9.1 github.com/knadh/koanf/parsers/yaml v0.1.0 github.com/knadh/koanf/providers/env v0.1.0 @@ -52,7 +50,6 @@ require ( github.com/bodgit/windows v1.0.1 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect github.com/bytedance/sonic v1.9.1 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -61,6 +58,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/fatih/structs v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect diff --git a/go.sum b/go.sum index 9bf761f..14db9e5 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,6 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= @@ -100,7 +98,6 @@ github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNa github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= @@ -128,8 +125,6 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1 github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -140,19 +135,13 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= -github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= @@ -262,7 +251,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -290,7 +278,6 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -301,7 +288,6 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -313,7 +299,6 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -344,7 +329,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -353,8 +337,6 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -384,7 +366,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -472,7 +453,6 @@ github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -487,15 +467,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -521,7 +492,6 @@ github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= @@ -530,7 +500,6 @@ github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -556,7 +525,6 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -655,7 +623,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/src/config/default.go b/src/config/default.go index 1be76be..488d633 100644 --- a/src/config/default.go +++ b/src/config/default.go @@ -1,6 +1,7 @@ package config var defaultConfig = Config{ + DataFolder: "./data", WebUi: WebUi{ Port: 4444, IP: "0.0.0.0", @@ -22,8 +23,9 @@ var defaultConfig = Config{ }, TorrentClient: TorrentClient{ - DataFolder: "data", - MetadataFolder: "metadata", + DataFolder: "./torrent/data", + MetadataFolder: "./torrent/metadata", + DHTNodes: []string{}, // GlobalCacheSize: 2048, diff --git a/src/config/model.go b/src/config/model.go index ff10a27..f7c9fd9 100644 --- a/src/config/model.go +++ b/src/config/model.go @@ -6,6 +6,8 @@ type Config struct { TorrentClient TorrentClient `koanf:"torrent"` Mounts Mounts `koanf:"mounts"` Log Log `koanf:"log"` + + DataFolder string `koanf:"dataFolder"` } type WebUi struct { @@ -25,7 +27,8 @@ type TorrentClient struct { ReadTimeout int `koanf:"read_timeout,omitempty"` AddTimeout int `koanf:"add_timeout,omitempty"` - DisableIPv6 bool `koanf:"disable_ipv6,omitempty"` + DHTNodes []string `koanf:"dhtnodes,omitempty"` + DisableIPv6 bool `koanf:"disable_ipv6,omitempty"` DataFolder string `koanf:"data_folder,omitempty"` MetadataFolder string `koanf:"metadata_folder,omitempty"` diff --git a/src/fs/container.go b/src/fs/container.go deleted file mode 100644 index 3a839fe..0000000 --- a/src/fs/container.go +++ /dev/null @@ -1,24 +0,0 @@ -package fs - -type ContainerFs struct { - s *storage -} - -func NewContainerFs(fss map[string]Filesystem) (*ContainerFs, error) { - s := newStorage(SupportedFactories) - for p, fs := range fss { - if err := s.AddFS(fs, p); err != nil { - return nil, err - } - } - - return &ContainerFs{s: s}, nil -} - -func (fs *ContainerFs) Open(filename string) (File, error) { - return fs.s.Get(filename) -} - -func (fs *ContainerFs) ReadDir(path string) (map[string]File, error) { - return fs.s.Children(path) -} diff --git a/src/fs/container_test.go b/src/fs/container_test.go deleted file mode 100644 index ff1913e..0000000 --- a/src/fs/container_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package fs - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestContainer(t *testing.T) { - t.Parallel() - - require := require.New(t) - - fss := map[string]Filesystem{ - "/test": &DummyFs{}, - } - - c, err := NewContainerFs(fss) - require.NoError(err) - - f, err := c.Open("/test/dir/here") - require.NoError(err) - require.NotNil(f) - - files, err := c.ReadDir("/") - require.NoError(err) - require.Len(files, 1) -} diff --git a/src/fs/memory_test.go b/src/fs/memory_test.go deleted file mode 100644 index fc06634..0000000 --- a/src/fs/memory_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package fs - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestMemory(t *testing.T) { - t.Parallel() - - require := require.New(t) - - mem := NewMemory() - - mem.Storage.Add(NewMemoryFile([]byte("Hello")), "/dir/here") - - fss := map[string]Filesystem{ - "/test": mem, - } - - c, err := NewContainerFs(fss) - require.NoError(err) - - f, err := c.Open("/test/dir/here") - require.NoError(err) - require.NotNil(f) - require.Equal(int64(5), f.Size()) - require.NoError(f.Close()) - - files, err := c.ReadDir("/") - require.NoError(err) - require.Len(files, 1) - - files, err = c.ReadDir("/test") - require.NoError(err) - require.Len(files, 1) - -} diff --git a/src/fs/storage.go b/src/fs/storage.go deleted file mode 100644 index 9271b23..0000000 --- a/src/fs/storage.go +++ /dev/null @@ -1,184 +0,0 @@ -package fs - -import ( - "os" - "path" - "strings" -) - -const separator = "/" - -type FsFactory func(f File) (Filesystem, error) - -var SupportedFactories = map[string]FsFactory{ - ".zip": func(f File) (Filesystem, error) { - return NewArchive(f, f.Size(), &Zip{}), nil - }, - ".rar": func(f File) (Filesystem, error) { - return NewArchive(f, f.Size(), &Rar{}), nil - }, - ".7z": func(f File) (Filesystem, error) { - return NewArchive(f, f.Size(), &SevenZip{}), nil - }, -} - -type storage struct { - factories map[string]FsFactory - - files map[string]File - filesystems map[string]Filesystem - children map[string]map[string]File -} - -func newStorage(factories map[string]FsFactory) *storage { - return &storage{ - files: make(map[string]File), - children: make(map[string]map[string]File), - filesystems: make(map[string]Filesystem), - factories: factories, - } -} - -func (s *storage) Clear() { - s.files = make(map[string]File) - s.children = make(map[string]map[string]File) - s.filesystems = make(map[string]Filesystem) - - s.Add(&Dir{}, "/") -} - -func (s *storage) Has(path string) bool { - path = clean(path) - - f := s.files[path] - if f != nil { - return true - } - - if f, _ := s.getFileFromFs(path); f != nil { - return true - } - - return false -} - -func (s *storage) AddFS(fs Filesystem, p string) error { - p = clean(p) - if s.Has(p) { - if dir, err := s.Get(p); err == nil { - if !dir.IsDir() { - return os.ErrExist - } - } - - return nil - } - - s.filesystems[p] = fs - return s.createParent(p, &Dir{}) -} - -func (s *storage) Add(f File, p string) error { - p = clean(p) - if s.Has(p) { - if dir, err := s.Get(p); err == nil { - if !dir.IsDir() { - return os.ErrExist - } - } - - return nil - } - - ext := path.Ext(p) - if ffs := s.factories[ext]; ffs != nil { - fs, err := ffs(f) - if err != nil { - return err - } - - s.filesystems[p] = fs - } else { - s.files[p] = f - } - - return s.createParent(p, f) -} - -func (s *storage) createParent(p string, f File) error { - base, filename := path.Split(p) - base = clean(base) - - if err := s.Add(&Dir{}, base); err != nil { - return err - } - - if _, ok := s.children[base]; !ok { - s.children[base] = make(map[string]File) - } - - if filename != "" { - s.children[base][filename] = f - } - - return nil -} - -func (s *storage) Children(path string) (map[string]File, error) { - path = clean(path) - - files, err := s.getDirFromFs(path) - if err == nil { - return files, nil - } - - if !os.IsNotExist(err) { - return nil, err - } - - l := make(map[string]File) - for n, f := range s.children[path] { - l[n] = f - } - - return l, nil -} - -func (s *storage) Get(path string) (File, error) { - path = clean(path) - if !s.Has(path) { - return nil, os.ErrNotExist - } - - file, ok := s.files[path] - if ok { - return file, nil - } - - return s.getFileFromFs(path) -} - -func (s *storage) getFileFromFs(p string) (File, error) { - for fsp, fs := range s.filesystems { - if strings.HasPrefix(p, fsp) { - return fs.Open(separator + strings.TrimPrefix(p, fsp)) - } - } - - return nil, os.ErrNotExist -} - -func (s *storage) getDirFromFs(p string) (map[string]File, error) { - for fsp, fs := range s.filesystems { - if strings.HasPrefix(p, fsp) { - path := strings.TrimPrefix(p, fsp) - return fs.ReadDir(path) - } - } - - return nil, os.ErrNotExist -} - -func clean(p string) string { - return path.Clean(separator + strings.ReplaceAll(p, "\\", "/")) -} diff --git a/src/fs/storage_test.go b/src/fs/storage_test.go deleted file mode 100644 index 38d4068..0000000 --- a/src/fs/storage_test.go +++ /dev/null @@ -1,195 +0,0 @@ -package fs - -import ( - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -var dummyFactories = map[string]FsFactory{ - ".test": func(f File) (Filesystem, error) { - return &DummyFs{}, nil - }, -} - -func TestStorage(t *testing.T) { - t.Parallel() - - require := require.New(t) - - s := newStorage(dummyFactories) - - err := s.Add(&Dummy{}, "/path/to/dummy/file.txt") - require.NoError(err) - - err = s.Add(&Dummy{}, "/path/to/dummy/file2.txt") - require.NoError(err) - - contains := s.Has("/path") - require.True(contains) - - contains = s.Has("/path/to/dummy/") - require.True(contains) - - file, err := s.Get("/path/to/dummy/file.txt") - require.NoError(err) - require.Equal(&Dummy{}, file) - - file, err = s.Get("/path/to/dummy/file3.txt") - require.Error(err) - require.Nil(file) - - files, err := s.Children("/path/to/dummy/") - require.NoError(err) - require.Len(files, 2) - require.Contains(files, "file.txt") - require.Contains(files, "file2.txt") - - err = s.Add(&Dummy{}, "/path/to/dummy/folder/file.txt") - require.NoError(err) - - files, err = s.Children("/path/to/dummy/") - require.NoError(err) - require.Len(files, 3) - require.Contains(files, "file.txt") - require.Contains(files, "file2.txt") - require.Contains(files, "folder") - - err = s.Add(&Dummy{}, "path/file4.txt") - require.NoError(err) - - require.True(s.Has("/path/file4.txt")) - - files, err = s.Children("/") - require.NoError(err) - require.Len(files, 1) - - err = s.Add(&Dummy{}, "/path/special_file.test") - require.NoError(err) - - file, err = s.Get("/path/special_file.test/dir/here/file1.txt") - require.NoError(err) - require.Equal(&Dummy{}, file) - - files, err = s.Children("/path/special_file.test") - require.NoError(err) - require.NotNil(files) - - files, err = s.Children("/path/special_file.test/dir/here") - require.NoError(err) - require.Len(files, 2) - - err = s.Add(&Dummy{}, "/path/to/__special__path/file3.txt") - require.NoError(err) - - file, err = s.Get("/path/to/__special__path/file3.txt") - require.NoError(err) - require.Equal(&Dummy{}, file) - - s.Clear() -} - -func TestStorageWindowsPath(t *testing.T) { - t.Parallel() - - require := require.New(t) - - s := newStorage(dummyFactories) - - err := s.Add(&Dummy{}, "\\path\\to\\dummy\\file.txt") - require.NoError(err) - - file, err := s.Get("\\path\\to\\dummy\\file.txt") - require.NoError(err) - require.Equal(&Dummy{}, file) - - file, err = s.Get("/path/to/dummy/file.txt") - require.NoError(err) - require.Equal(&Dummy{}, file) -} - -func TestStorageAddFs(t *testing.T) { - t.Parallel() - - require := require.New(t) - - s := newStorage(dummyFactories) - - err := s.AddFS(&DummyFs{}, "/test") - require.NoError(err) - - f, err := s.Get("/test/dir/here/file1.txt") - require.NoError(err) - require.NotNil(f) - - err = s.AddFS(&DummyFs{}, "/test") - require.Error(err) -} - -func TestSupportedFactories(t *testing.T) { - t.Parallel() - - require := require.New(t) - - require.Contains(SupportedFactories, ".zip") - require.Contains(SupportedFactories, ".rar") - require.Contains(SupportedFactories, ".7z") - - fs, err := SupportedFactories[".zip"](&Dummy{}) - require.NoError(err) - require.NotNil(fs) - - fs, err = SupportedFactories[".rar"](&Dummy{}) - require.NoError(err) - require.NotNil(fs) - - fs, err = SupportedFactories[".7z"](&Dummy{}) - require.NoError(err) - require.NotNil(fs) -} - -var _ Filesystem = &DummyFs{} - -type DummyFs struct { -} - -func (d *DummyFs) Open(filename string) (File, error) { - return &Dummy{}, nil -} - -func (d *DummyFs) ReadDir(path string) (map[string]File, error) { - if path == "/dir/here" { - return map[string]File{ - "file1.txt": &Dummy{}, - "file2.txt": &Dummy{}, - }, nil - } - - return nil, os.ErrNotExist -} - -var _ File = &Dummy{} - -type Dummy struct { -} - -func (d *Dummy) Size() int64 { - return 0 -} - -func (d *Dummy) IsDir() bool { - return false -} - -func (d *Dummy) Close() error { - return nil -} - -func (d *Dummy) Read(p []byte) (n int, err error) { - return 0, nil -} - -func (d *Dummy) ReadAt(p []byte, off int64) (n int, err error) { - return 0, nil -} diff --git a/src/host/storage.go b/src/host/storage.go new file mode 100644 index 0000000..8372068 --- /dev/null +++ b/src/host/storage.go @@ -0,0 +1,120 @@ +package host + +import ( + "git.kmsign.ru/royalcat/tstor/src/host/torrent" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" +) + +type storage struct { + factories map[string]vfs.FsFactory +} + +func NewStorage(downPath string, tsrv *torrent.Service) vfs.Filesystem { + factories := map[string]vfs.FsFactory{ + ".torrent": tsrv.NewTorrentFs, + } + + // add default torrent factory for root filesystem + for k, v := range vfs.ArchiveFactories { + factories[k] = v + } + + return vfs.NewResolveFS(downPath, factories) +} + +// func (s *storage) Clear() { +// s.files = make(map[string]vfs.File) +// } + +// func (s *storage) Has(path string) bool { +// path = clean(path) + +// f := s.files[path] +// if f != nil { +// return true +// } + +// if f, _ := s.getFileFromFs(path); f != nil { +// return true +// } + +// return false +// } + +// func (s *storage) createParent(p string, f File) error { +// base, filename := path.Split(p) +// base = clean(base) + +// if err := s.Add(&Dir{}, base); err != nil { +// return err +// } + +// if _, ok := s.children[base]; !ok { +// s.children[base] = make(map[string]File) +// } + +// if filename != "" { +// s.children[base][filename] = f +// } + +// return nil +// } + +// func (s *storage) Children(path string) (map[string]File, error) { +// path = clean(path) + +// files, err := s.getDirFromFs(path) +// if err == nil { +// return files, nil +// } + +// if !os.IsNotExist(err) { +// return nil, err +// } + +// l := make(map[string]File) +// for n, f := range s.children[path] { +// l[n] = f +// } + +// return l, nil +// } + +// func (s *storage) Get(path string) (File, error) { +// path = clean(path) +// if !s.Has(path) { +// return nil, os.ErrNotExist +// } + +// file, ok := s.files[path] +// if ok { +// return file, nil +// } + +// return s.getFileFromFs(path) +// } + +// func (s *storage) getFileFromFs(p string) (File, error) { +// for fsp, fs := range s.filesystems { +// if strings.HasPrefix(p, fsp) { +// return fs.Open(separator + strings.TrimPrefix(p, fsp)) +// } +// } + +// return nil, os.ErrNotExist +// } + +// func (s *storage) getDirFromFs(p string) (map[string]File, error) { +// for fsp, fs := range s.filesystems { +// if strings.HasPrefix(p, fsp) { +// path := strings.TrimPrefix(p, fsp) +// return fs.ReadDir(path) +// } +// } + +// return nil, os.ErrNotExist +// } + +// func clean(p string) string { +// return path.Clean(separator + strings.ReplaceAll(p, "\\", "/")) +// } diff --git a/src/torrent/client.go b/src/host/torrent/client.go similarity index 100% rename from src/torrent/client.go rename to src/host/torrent/client.go diff --git a/src/torrent/id.go b/src/host/torrent/id.go similarity index 100% rename from src/torrent/id.go rename to src/host/torrent/id.go diff --git a/src/host/torrent/service.go b/src/host/torrent/service.go new file mode 100644 index 0000000..de87544 --- /dev/null +++ b/src/host/torrent/service.go @@ -0,0 +1,213 @@ +package torrent + +import ( + "sync" + + "git.kmsign.ru/royalcat/tstor/src/host/vfs" + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/metainfo" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +type Service struct { + c *torrent.Client + + // stats *Stats + + mu sync.Mutex + + log zerolog.Logger + addTimeout, readTimeout int +} + +func NewService(c *torrent.Client, addTimeout, readTimeout int) *Service { + l := log.Logger.With().Str("component", "torrent-service").Logger() + return &Service{ + log: l, + c: c, + // stats: newStats(), // TODO persistent + addTimeout: addTimeout, + readTimeout: readTimeout, + } +} + +var _ vfs.FsFactory = (*Service)(nil).NewTorrentFs + +func (s *Service) NewTorrentFs(f vfs.File) (vfs.Filesystem, error) { + defer f.Close() + + mi, err := metainfo.Load(f) + if err != nil { + return nil, err + } + t, err := s.c.AddTorrent(mi) + if err != nil { + return nil, err + } + <-t.GotInfo() + t.AllowDataDownload() + return vfs.NewTorrentFs(t, s.readTimeout), nil +} + +func (s *Service) Stats() (*Stats, error) { + return &Stats{}, nil +} + +// func (s *Service) Load() (map[string]vfs.Filesystem, error) { +// // Load from config +// s.log.Info().Msg("adding torrents from configuration") +// for _, loader := range s.loaders { +// if err := s.load(loader); err != nil { +// return nil, err +// } +// } + +// // Load from DB +// s.log.Info().Msg("adding torrents from database") +// return s.fss, s.load(s.db) +// } + +// func (s *Service) load(l loader.Loader) error { +// list, err := l.ListMagnets() +// if err != nil { +// return err +// } +// for r, ms := range list { +// s.addRoute(r) +// for _, m := range ms { +// if err := s.addMagnet(r, m); err != nil { +// return err +// } +// } +// } + +// list, err = l.ListTorrentPaths() +// if err != nil { +// return err +// } +// for r, ms := range list { +// s.addRoute(r) +// for _, p := range ms { +// if err := s.addTorrentPath(r, p); err != nil { +// return err +// } +// } +// } + +// return nil +// } + +// func (s *Service) AddMagnet(r, m string) error { +// if err := s.addMagnet(r, m); err != nil { +// return err +// } + +// // Add to db +// return s.db.AddMagnet(r, m) +// } + +// func (s *Service) addTorrentPath(r, p string) error { +// // Add to client +// t, err := s.c.AddTorrentFromFile(p) +// if err != nil { +// return err +// } + +// return s.addTorrent(r, t) +// } + +// func (s *Service) addMagnet(r, m string) error { +// // Add to client +// t, err := s.c.AddMagnet(m) +// if err != nil { +// return err +// } + +// return s.addTorrent(r, t) + +// } + +// func (s *Service) addRoute(r string) { +// s.s.AddRoute(r) + +// // Add to filesystems +// folder := path.Join("/", r) +// s.mu.Lock() +// defer s.mu.Unlock() +// _, ok := s.fss[folder] +// if !ok { +// s.fss[folder] = vfs.NewTorrentFs(s.readTimeout) +// } +// } + +// func (s *Service) addTorrent(r string, t *torrent.Torrent) error { +// // only get info if name is not available +// if t.Info() == nil { +// s.log.Info().Str("hash", t.InfoHash().String()).Msg("getting torrent info") +// select { +// case <-time.After(time.Duration(s.addTimeout) * time.Second): +// s.log.Error().Str("hash", t.InfoHash().String()).Msg("timeout getting torrent info") +// return errors.New("timeout getting torrent info") +// case <-t.GotInfo(): +// s.log.Info().Str("hash", t.InfoHash().String()).Msg("obtained torrent info") +// } + +// } + +// // Add to stats +// s.s.Add(r, t) + +// // Add to filesystems +// folder := path.Join("/", r) +// s.mu.Lock() +// defer s.mu.Unlock() + +// tfs, ok := s.fss[folder].(*vfs.TorrentFs) +// if !ok { +// return errors.New("error adding torrent to filesystem") +// } + +// tfs.AddTorrent(t) +// s.log.Info().Str("name", t.Info().Name).Str("route", r).Msg("torrent added") + +// return nil +// } + +// func (s *Service) RemoveFromHash(r, h string) error { +// // Remove from db +// deleted, err := s.db.RemoveFromHash(r, h) +// if err != nil { +// return err +// } + +// if !deleted { +// return fmt.Errorf("element with hash %v on route %v cannot be removed", h, r) +// } + +// // Remove from stats +// s.s.Del(r, h) + +// // Remove from fs +// folder := path.Join("/", r) + +// tfs, ok := s.fss[folder].(*vfs.TorrentFs) +// if !ok { +// return errors.New("error removing torrent from filesystem") +// } + +// tfs.RemoveTorrent(h) + +// // Remove from client +// var mh metainfo.Hash +// if err := mh.FromHexString(h); err != nil { +// return err +// } + +// t, ok := s.c.Torrent(metainfo.NewHashFromHex(h)) +// if ok { +// t.Drop() +// } + +// return nil +// } diff --git a/src/torrent/stats.go b/src/host/torrent/stats.go similarity index 74% rename from src/torrent/stats.go rename to src/host/torrent/stats.go index 88cfc3e..9b6f956 100644 --- a/src/torrent/stats.go +++ b/src/host/torrent/stats.go @@ -2,7 +2,6 @@ package torrent import ( "errors" - "sort" "sync" "time" @@ -73,65 +72,42 @@ type stat struct { } type Stats struct { - mut sync.Mutex - torrents map[string]*torrent.Torrent - torrentsByRoute map[string]map[string]*torrent.Torrent - previousStats map[string]*stat + mut sync.Mutex + torrents map[string]*torrent.Torrent + previousStats map[string]*stat gTime time.Time } -func NewStats() *Stats { +func newStats() *Stats { return &Stats{ - gTime: time.Now(), - torrents: make(map[string]*torrent.Torrent), - torrentsByRoute: make(map[string]map[string]*torrent.Torrent), - previousStats: make(map[string]*stat), + gTime: time.Now(), + torrents: make(map[string]*torrent.Torrent), + // torrentsByRoute: make(map[string]map[string]*torrent.Torrent), + // previousStats: make(map[string]*stat), } } -func (s *Stats) AddRoute(route string) { - _, ok := s.torrentsByRoute[route] - if !ok { - s.torrentsByRoute[route] = make(map[string]*torrent.Torrent) - } -} - -func (s *Stats) Add(route string, t *torrent.Torrent) { +func (s *Stats) Add(path string, t *torrent.Torrent) { s.mut.Lock() defer s.mut.Unlock() - h := t.InfoHash().String() - - s.torrents[h] = t - s.previousStats[h] = &stat{} - - _, ok := s.torrentsByRoute[route] - if !ok { - s.torrentsByRoute[route] = make(map[string]*torrent.Torrent) - } - - s.torrentsByRoute[route][h] = t + s.torrents[path] = t + s.previousStats[path] = &stat{} } -func (s *Stats) Del(route, hash string) { +func (s *Stats) Del(path string) { s.mut.Lock() defer s.mut.Unlock() - delete(s.torrents, hash) - delete(s.previousStats, hash) - ts, ok := s.torrentsByRoute[route] - if !ok { - return - } - - delete(ts, hash) + delete(s.torrents, path) + delete(s.previousStats, path) } -func (s *Stats) Stats(hash string) (*TorrentStats, error) { +func (s *Stats) Stats(path string) (*TorrentStats, error) { s.mut.Lock() defer s.mut.Unlock() - t, ok := s.torrents[hash] + t, ok := s.torrents[path] if !(ok) { return nil, ErrTorrentNotFound } @@ -141,32 +117,6 @@ func (s *Stats) Stats(hash string) (*TorrentStats, error) { return s.stats(now, t, true), nil } -func (s *Stats) RoutesStats() []*RouteStats { - s.mut.Lock() - defer s.mut.Unlock() - - now := time.Now() - - var out []*RouteStats - for r, tl := range s.torrentsByRoute { - var tStats []*TorrentStats - for _, t := range tl { - ts := s.stats(now, t, true) - tStats = append(tStats, ts) - } - - sort.Sort(byName(tStats)) - - rs := &RouteStats{ - Name: r, - TorrentStats: tStats, - } - out = append(out, rs) - } - - return out -} - func (s *Stats) GlobalStats() *GlobalTorrentStats { s.mut.Lock() defer s.mut.Unlock() diff --git a/src/torrent/store.go b/src/host/torrent/store.go similarity index 100% rename from src/torrent/store.go rename to src/host/torrent/store.go diff --git a/src/fs/archive.go b/src/host/vfs/archive.go similarity index 58% rename from src/fs/archive.go rename to src/host/vfs/archive.go index 30898c3..3963689 100644 --- a/src/fs/archive.go +++ b/src/host/vfs/archive.go @@ -1,4 +1,4 @@ -package fs +package vfs import ( "archive/zip" @@ -12,18 +12,129 @@ import ( "github.com/nwaples/rardecode/v2" ) -var _ loader = &Zip{} - -type Zip struct { +var ArchiveFactories = map[string]FsFactory{ + ".zip": func(f File) (Filesystem, error) { + return NewArchive(f, f.Size(), ZipLoader), nil + }, + ".rar": func(f File) (Filesystem, error) { + return NewArchive(f, f.Size(), RarLoader), nil + }, + ".7z": func(f File) (Filesystem, error) { + return NewArchive(f, f.Size(), SevenZipLoader), nil + }, } -func (fs *Zip) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile, error) { +type ArchiveLoader func(r iio.Reader, size int64) (map[string]*archiveFile, error) + +var _ Filesystem = &archive{} + +type archive struct { + r iio.Reader + + size int64 + + files func() (map[string]*archiveFile, error) +} + +func NewArchive(r iio.Reader, size int64, loader ArchiveLoader) *archive { + return &archive{ + r: r, + size: size, + files: sync.OnceValues(func() (map[string]*archiveFile, error) { + return loader(r, size) + }), + } +} + +func (a *archive) Open(filename string) (File, error) { + files, err := a.files() + if err != nil { + return nil, err + } + + return getFile(files, filename) +} + +func (fs *archive) ReadDir(path string) (map[string]File, error) { + files, err := fs.files() + if err != nil { + return nil, err + } + + return listFilesInDir(files, path) +} + +var _ File = &archiveFile{} + +func NewArchiveFile(readerFunc func() (iio.Reader, error), len int64) *archiveFile { + return &archiveFile{ + readerFunc: readerFunc, + len: len, + } +} + +type archiveFile struct { + readerFunc func() (iio.Reader, error) + reader iio.Reader + len int64 +} + +func (d *archiveFile) load() error { + if d.reader != nil { + return nil + } + r, err := d.readerFunc() + if err != nil { + return err + } + + d.reader = r + + return nil +} + +func (d *archiveFile) Size() int64 { + return d.len +} + +func (d *archiveFile) IsDir() bool { + return false +} + +func (d *archiveFile) Close() (err error) { + if d.reader != nil { + err = d.reader.Close() + d.reader = nil + } + + return +} + +func (d *archiveFile) Read(p []byte) (n int, err error) { + if err := d.load(); err != nil { + return 0, err + } + + return d.reader.Read(p) +} + +func (d *archiveFile) ReadAt(p []byte, off int64) (n int, err error) { + if err := d.load(); err != nil { + return 0, err + } + + return d.reader.ReadAt(p, off) +} + +var _ ArchiveLoader = ZipLoader + +func ZipLoader(reader iio.Reader, size int64) (map[string]*archiveFile, error) { zr, err := zip.NewReader(reader, size) if err != nil { return nil, err } - out := make(map[string]*ArchiveFile) + out := make(map[string]*archiveFile) for _, f := range zr.File { f := f if f.FileInfo().IsDir() { @@ -48,18 +159,15 @@ func (fs *Zip) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile, return out, nil } -var _ loader = &SevenZip{} +var _ ArchiveLoader = SevenZipLoader -type SevenZip struct { -} - -func (fs *SevenZip) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile, error) { +func SevenZipLoader(reader iio.Reader, size int64) (map[string]*archiveFile, error) { r, err := sevenzip.NewReader(reader, size) if err != nil { return nil, err } - out := make(map[string]*ArchiveFile) + out := make(map[string]*archiveFile) for _, f := range r.File { f := f if f.FileInfo().IsDir() { @@ -84,18 +192,15 @@ func (fs *SevenZip) getFiles(reader iio.Reader, size int64) (map[string]*Archive return out, nil } -var _ loader = &Rar{} +var _ ArchiveLoader = RarLoader -type Rar struct { -} - -func (fs *Rar) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile, error) { +func RarLoader(reader iio.Reader, size int64) (map[string]*archiveFile, error) { r, err := rardecode.NewReader(iio.NewSeekerWrapper(reader, size)) if err != nil { return nil, err } - out := make(map[string]*ArchiveFile) + out := make(map[string]*archiveFile) for { header, err := r.Next() if err == io.EOF { @@ -118,129 +223,3 @@ func (fs *Rar) getFiles(reader iio.Reader, size int64) (map[string]*ArchiveFile, return out, nil } - -type loader interface { - getFiles(r iio.Reader, size int64) (map[string]*ArchiveFile, error) -} - -var _ Filesystem = &archive{} - -type archive struct { - r iio.Reader - s *storage - - size int64 - once sync.Once - l loader -} - -func NewArchive(r iio.Reader, size int64, l loader) *archive { - return &archive{ - r: r, - s: newStorage(nil), - size: size, - l: l, - } -} - -func (fs *archive) loadOnce() error { - var errOut error - fs.once.Do(func() { - files, err := fs.l.getFiles(fs.r, fs.size) - if err != nil { - errOut = err - return - } - - for name, file := range files { - if err := fs.s.Add(file, name); err != nil { - errOut = err - return - } - } - }) - - return errOut -} - -func (fs *archive) Open(filename string) (File, error) { - if filename == string(os.PathSeparator) { - return &Dir{}, nil - } - - if err := fs.loadOnce(); err != nil { - return nil, err - } - - return fs.s.Get(filename) -} - -func (fs *archive) ReadDir(path string) (map[string]File, error) { - if err := fs.loadOnce(); err != nil { - return nil, err - } - - return fs.s.Children(path) -} - -var _ File = &ArchiveFile{} - -func NewArchiveFile(readerFunc func() (iio.Reader, error), len int64) *ArchiveFile { - return &ArchiveFile{ - readerFunc: readerFunc, - len: len, - } -} - -type ArchiveFile struct { - readerFunc func() (iio.Reader, error) - reader iio.Reader - len int64 -} - -func (d *ArchiveFile) load() error { - if d.reader != nil { - return nil - } - r, err := d.readerFunc() - if err != nil { - return err - } - - d.reader = r - - return nil -} - -func (d *ArchiveFile) Size() int64 { - return d.len -} - -func (d *ArchiveFile) IsDir() bool { - return false -} - -func (d *ArchiveFile) Close() (err error) { - if d.reader != nil { - err = d.reader.Close() - d.reader = nil - } - - return -} - -func (d *ArchiveFile) Read(p []byte) (n int, err error) { - if err := d.load(); err != nil { - return 0, err - } - - return d.reader.Read(p) -} - -func (d *ArchiveFile) ReadAt(p []byte, off int64) (n int, err error) { - if err := d.load(); err != nil { - return 0, err - } - - return d.reader.ReadAt(p, off) -} diff --git a/src/fs/archive_test.go b/src/host/vfs/archive_test.go similarity index 95% rename from src/fs/archive_test.go rename to src/host/vfs/archive_test.go index 2ad7b84..0383b56 100644 --- a/src/fs/archive_test.go +++ b/src/host/vfs/archive_test.go @@ -1,4 +1,4 @@ -package fs +package vfs import ( "archive/zip" @@ -18,7 +18,7 @@ func TestZipFilesystem(t *testing.T) { zReader, len := createTestZip(require) - zfs := NewArchive(zReader, len, &Zip{}) + zfs := NewArchive(zReader, len, ZipLoader) files, err := zfs.ReadDir("/path/to/test/file") require.NoError(err) diff --git a/src/fs/dir.go b/src/host/vfs/dir.go similarity index 96% rename from src/fs/dir.go rename to src/host/vfs/dir.go index 025c031..3077321 100644 --- a/src/fs/dir.go +++ b/src/host/vfs/dir.go @@ -1,4 +1,4 @@ -package fs +package vfs var _ File = &Dir{} diff --git a/src/fs/fs.go b/src/host/vfs/fs.go similarity index 98% rename from src/fs/fs.go rename to src/host/vfs/fs.go index 9043111..8a73aad 100644 --- a/src/fs/fs.go +++ b/src/host/vfs/fs.go @@ -1,4 +1,4 @@ -package fs +package vfs import ( "os" diff --git a/src/fs/fs_test.go b/src/host/vfs/fs_test.go similarity index 97% rename from src/fs/fs_test.go rename to src/host/vfs/fs_test.go index d6fbb2f..8a79fdd 100644 --- a/src/fs/fs_test.go +++ b/src/host/vfs/fs_test.go @@ -1,4 +1,4 @@ -package fs +package vfs import ( "io/fs" diff --git a/src/fs/memory.go b/src/host/vfs/memory.go similarity index 50% rename from src/fs/memory.go rename to src/host/vfs/memory.go index f2bcb0f..1031a7d 100644 --- a/src/fs/memory.go +++ b/src/host/vfs/memory.go @@ -1,27 +1,27 @@ -package fs +package vfs import ( "bytes" ) -var _ Filesystem = &Memory{} +var _ Filesystem = &MemoryFs{} -type Memory struct { - Storage *storage +type MemoryFs struct { + files map[string]*MemoryFile } -func NewMemory() *Memory { - return &Memory{ - Storage: newStorage(nil), +func NewMemoryFS(files map[string]*MemoryFile) *MemoryFs { + return &MemoryFs{ + files: files, } } -func (fs *Memory) Open(filename string) (File, error) { - return fs.Storage.Get(filename) +func (m *MemoryFs) Open(filename string) (File, error) { + return getFile(m.files, filename) } -func (fs *Memory) ReadDir(path string) (map[string]File, error) { - return fs.Storage.Children(path) +func (fs *MemoryFs) ReadDir(path string) (map[string]File, error) { + return listFilesInDir(fs.files, path) } var _ File = &MemoryFile{} diff --git a/src/host/vfs/memory_test.go b/src/host/vfs/memory_test.go new file mode 100644 index 0000000..9d090d8 --- /dev/null +++ b/src/host/vfs/memory_test.go @@ -0,0 +1,46 @@ +package vfs + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMemory(t *testing.T) { + t.Parallel() + + require := require.New(t) + testData := "Hello" + + c := NewMemoryFS(map[string]*MemoryFile{ + "/dir/here": NewMemoryFile([]byte(testData)), + }) + + // fss := map[string]Filesystem{ + // "/test": mem, + // } + + // c, err := NewContainerFs(fss) + // require.NoError(err) + + f, err := c.Open("/dir/here") + require.NoError(err) + require.NotNil(f) + require.Equal(int64(5), f.Size()) + require.NoError(f.Close()) + + data := make([]byte, 5) + n, err := f.Read(data) + require.NoError(err) + require.Equal(n, 5) + require.Equal(string(data), testData) + + files, err := c.ReadDir("/") + require.NoError(err) + require.Len(files, 1) + + files, err = c.ReadDir("/dir") + require.NoError(err) + require.Len(files, 1) + +} diff --git a/src/host/vfs/os.go b/src/host/vfs/os.go new file mode 100644 index 0000000..035b66d --- /dev/null +++ b/src/host/vfs/os.go @@ -0,0 +1,167 @@ +package vfs + +import ( + "io/fs" + "os" + "path" + "sync" +) + +type OsFS struct { + hostDir string +} + +// Open implements Filesystem. +func (fs *OsFS) Open(filename string) (File, error) { + if path.Clean(filename) == Separator { + return &Dir{}, nil + } + + osfile, err := os.Open(path.Join(fs.hostDir, filename)) + if err != nil { + return nil, err + } + return NewOsFile(osfile), nil +} + +// ReadDir implements Filesystem. +func (o *OsFS) ReadDir(dir string) (map[string]File, error) { + dir = path.Join(o.hostDir, dir) + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + out := map[string]File{} + for _, e := range entries { + if e.IsDir() { + out[e.Name()] = &Dir{} + } else { + out[e.Name()] = NewLazyOsFile(path.Join(dir, e.Name())) + } + } + return out, nil +} + +func NewOsFs(osDir string) *OsFS { + return &OsFS{ + hostDir: osDir, + } +} + +var _ Filesystem = &OsFS{} + +type OsFile struct { + f *os.File +} + +func NewOsFile(f *os.File) *OsFile { + return &OsFile{f: f} +} + +var _ File = &OsFile{} + +// Close implements File. +func (f *OsFile) Close() error { + return f.f.Close() +} + +// Read implements File. +func (f *OsFile) Read(p []byte) (n int, err error) { + return f.f.Read(p) +} + +// ReadAt implements File. +func (f *OsFile) ReadAt(p []byte, off int64) (n int, err error) { + return f.f.ReadAt(p, off) +} + +func (f *OsFile) Stat() (fs.FileInfo, error) { + return f.f.Stat() +} + +// Size implements File. +func (f *OsFile) Size() int64 { + stat, err := f.Stat() + if err != nil { + return 0 + } + return stat.Size() +} + +// IsDir implements File. +func (f *OsFile) IsDir() bool { + stat, err := f.Stat() + if err != nil { + return false + } + return stat.IsDir() +} + +type LazyOsFile struct { + m sync.Mutex + path string + file *os.File +} + +func NewLazyOsFile(path string) *LazyOsFile { + return &LazyOsFile{path: path} +} + +var _ File = &OsFile{} + +func (f *LazyOsFile) open() error { + f.m.Lock() + defer f.m.Unlock() + + if f.file != nil { + return nil + } + + osFile, err := os.Open(f.path) + if err != nil { + return err + } + f.file = osFile + return nil +} + +// Close implements File. +func (f *LazyOsFile) Close() error { + return f.file.Close() +} + +// Read implements File. +func (f *LazyOsFile) Read(p []byte) (n int, err error) { + return f.file.Read(p) +} + +// ReadAt implements File. +func (f *LazyOsFile) ReadAt(p []byte, off int64) (n int, err error) { + return f.file.ReadAt(p, off) +} + +func (f *LazyOsFile) Stat() (fs.FileInfo, error) { + if f.file == nil { + return os.Stat(f.path) + } else { + return f.file.Stat() + } +} + +// Size implements File. +func (f *LazyOsFile) Size() int64 { + stat, err := f.Stat() + if err != nil { + return 0 + } + return stat.Size() +} + +// IsDir implements File. +func (f *LazyOsFile) IsDir() bool { + stat, err := f.Stat() + if err != nil { + return false + } + return stat.IsDir() +} diff --git a/src/host/vfs/resolver.go b/src/host/vfs/resolver.go new file mode 100644 index 0000000..2b058e8 --- /dev/null +++ b/src/host/vfs/resolver.go @@ -0,0 +1,146 @@ +package vfs + +import ( + "fmt" + "strings" + "sync" +) + +type ResolveFS struct { + osDir string + osFS *OsFS + resolver *resolver +} + +func NewResolveFS(osDir string, factories map[string]FsFactory) *ResolveFS { + return &ResolveFS{ + osDir: osDir, + osFS: NewOsFs(osDir), + resolver: newResolver(factories), + } +} + +// Open implements Filesystem. +func (r *ResolveFS) Open(filename string) (File, error) { + fsPath, nestedFs, nestedFsPath, err := r.resolver.resolvePath(filename, r.osFS.Open) + if err != nil { + return nil, err + } + if nestedFs != nil { + return nestedFs.Open(nestedFsPath) + } + + return r.osFS.Open(fsPath) +} + +// ReadDir implements Filesystem. +func (r *ResolveFS) ReadDir(dir string) (map[string]File, error) { + fsPath, nestedFs, nestedFsPath, err := r.resolver.resolvePath(dir, r.osFS.Open) + if err != nil { + return nil, err + } + if nestedFs != nil { + return nestedFs.ReadDir(nestedFsPath) + } + + return r.osFS.ReadDir(fsPath) +} + +var _ Filesystem = &ResolveFS{} + +type FsFactory func(f File) (Filesystem, error) + +const Separator = "/" + +func newResolver(factories map[string]FsFactory) *resolver { + return &resolver{ + factories: factories, + fsmap: map[string]Filesystem{}, + } +} + +type resolver struct { + m sync.Mutex + factories map[string]FsFactory + fsmap map[string]Filesystem // filesystem cache + // TODO: add fsmap clean +} + +type openFile func(path string) (File, error) + +// open requeue raw open, without resolver call +func (r *resolver) resolvePath(name string, rawOpen openFile) (fsPath string, nestedFs Filesystem, nestedFsPath string, err error) { + name = strings.TrimPrefix(name, Separator) + parts := strings.Split(name, Separator) + + nestOn := -1 + var nestFactory FsFactory + +PARTS_LOOP: + for i, part := range parts { + for ext, factory := range r.factories { + if strings.HasSuffix(part, ext) { + nestOn = i + 1 + nestFactory = factory + break PARTS_LOOP + } + } + } + + if nestOn == -1 { + return name, nil, "", nil + } + + fsPath = Clean(strings.Join(parts[:nestOn], Separator)) + nestedFsPath = Clean(strings.Join(parts[nestOn:], Separator)) + + // we dont need lock until now + // it must be before fsmap read to exclude race condition: + // read -> write + // read -> write + r.m.Lock() + defer r.m.Unlock() + + if nestedFs, ok := r.fsmap[fsPath]; ok { + return fsPath, nestedFs, nestedFsPath, nil + } else { + fsFile, err := rawOpen(fsPath) + if err != nil { + return "", nil, "", fmt.Errorf("error opening filesystem file: %s with error: %w", fsPath, err) + } + nestedFs, err := nestFactory(fsFile) + if err != nil { + return "", nil, "", fmt.Errorf("error creating filesystem from file: %s with error: %w", fsPath, err) + } + r.fsmap[fsPath] = nestedFs + + return fsPath, nestedFs, nestedFsPath, nil + } + +} + +// func (r *resolver) resolveFile(name string, fs Filesystem) (File, error) { +// fsPath, nestedFs, nestedFsPath, err := r.resolvePath(name, fs) +// if err != nil { +// return nil, err +// } + +// if nestedFs == nil { +// return fs.Open(fsPath) +// } + +// return nestedFs.Open(nestedFsPath) +// } + +// func (r *resolver) resolveDir(name string, fs Filesystem) (map[string]File, error) { +// fsPath, nestedFs, nestedFsPath, err := r.resolvePath(name, fs) +// if err != nil { +// return nil, err +// } + +// if nestedFs == nil { +// return fs.ReadDir(fsPath) +// } + +// return nestedFs.ReadDir(nestedFsPath) +// } diff --git a/src/host/vfs/resolver_test.go b/src/host/vfs/resolver_test.go new file mode 100644 index 0000000..953ad82 --- /dev/null +++ b/src/host/vfs/resolver_test.go @@ -0,0 +1,192 @@ +package vfs + +import ( + "os" +) + +type Dummy struct { +} + +func (d *Dummy) Size() int64 { + return 0 +} + +func (d *Dummy) IsDir() bool { + return false +} + +func (d *Dummy) Close() error { + return nil +} + +func (d *Dummy) Read(p []byte) (n int, err error) { + return 0, nil +} + +func (d *Dummy) ReadAt(p []byte, off int64) (n int, err error) { + return 0, nil +} + +var _ File = &Dummy{} + +type DummyFs struct { +} + +func (d *DummyFs) Open(filename string) (File, error) { + return &Dummy{}, nil +} + +func (d *DummyFs) ReadDir(path string) (map[string]File, error) { + if path == "/dir/here" { + return map[string]File{ + "file1.txt": &Dummy{}, + "file2.txt": &Dummy{}, + }, nil + } + + return nil, os.ErrNotExist +} + +var _ Filesystem = &DummyFs{} + +// func TestDefaultFactories(t *testing.T) { +// t.Parallel() + +// require := require.New(t) + +// require.Contains(defaultFactories, ".zip") +// require.Contains(defaultFactories, ".rar") +// require.Contains(defaultFactories, ".7z") + +// fs, err := defaultFactories[".zip"](&Dummy{}, nil) +// require.NoError(err) +// require.NotNil(fs) + +// fs, err = defaultFactories[".rar"](&Dummy{}, nil) +// require.NoError(err) +// require.NotNil(fs) + +// fs, err = defaultFactories[".7z"](&Dummy{}, nil) +// require.NoError(err) +// require.NotNil(fs) +// } + +// func TestStorageAddFs(t *testing.T) { +// t.Parallel() + +// require := require.New(t) + +// s := newStorage(dummyFactories) + +// err := s.AddFS(&DummyFs{}, "/test") +// require.NoError(err) + +// f, err := s.Get("/test/dir/here/file1.txt") +// require.NoError(err) +// require.NotNil(f) + +// err = s.AddFS(&DummyFs{}, "/test") +// require.Error(err) +// } + +// func TestStorageWindowsPath(t *testing.T) { +// t.Parallel() + +// require := require.New(t) + +// s := newStorage(dummyFactories) + +// err := s.Add(&Dummy{}, "\\path\\to\\dummy\\file.txt") +// require.NoError(err) + +// file, err := s.Get("\\path\\to\\dummy\\file.txt") +// require.NoError(err) +// require.Equal(&Dummy{}, file) + +// file, err = s.Get("/path/to/dummy/file.txt") +// require.NoError(err) +// require.Equal(&Dummy{}, file) +// } + +// var dummyFactories = map[string]vfs.FsFactory{ +// ".test": func(f vfs.File, factories map[string]vfs.FsFactory) (vfs.Filesystem, error) { +// return &DummyFs{}, nil +// }, +// } + +// func TestStorage(t *testing.T) { +// t.Parallel() + +// require := require.New(t) + +// s := newStorage(dummyFactories) + +// err := s.Add(&Dummy{}, "/path/to/dummy/file.txt") +// require.NoError(err) + +// err = s.Add(&Dummy{}, "/path/to/dummy/file2.txt") +// require.NoError(err) + +// contains := s.Has("/path") +// require.True(contains) + +// contains = s.Has("/path/to/dummy/") +// require.True(contains) + +// file, err := s.Get("/path/to/dummy/file.txt") +// require.NoError(err) +// require.Equal(&Dummy{}, file) + +// file, err = s.Get("/path/to/dummy/file3.txt") +// require.Error(err) +// require.Nil(file) + +// files, err := s.Children("/path/to/dummy/") +// require.NoError(err) +// require.Len(files, 2) +// require.Contains(files, "file.txt") +// require.Contains(files, "file2.txt") + +// err = s.Add(&Dummy{}, "/path/to/dummy/folder/file.txt") +// require.NoError(err) + +// files, err = s.Children("/path/to/dummy/") +// require.NoError(err) +// require.Len(files, 3) +// require.Contains(files, "file.txt") +// require.Contains(files, "file2.txt") +// require.Contains(files, "folder") + +// err = s.Add(&Dummy{}, "path/file4.txt") +// require.NoError(err) + +// require.True(s.Has("/path/file4.txt")) + +// files, err = s.Children("/") +// require.NoError(err) +// require.Len(files, 1) + +// err = s.Add(&Dummy{}, "/path/special_file.test") +// require.NoError(err) + +// file, err = s.Get("/path/special_file.test/dir/here/file1.txt") +// require.NoError(err) +// require.Equal(&Dummy{}, file) + +// files, err = s.Children("/path/special_file.test") +// require.NoError(err) +// require.NotNil(files) + +// files, err = s.Children("/path/special_file.test/dir/here") +// require.NoError(err) +// require.Len(files, 2) + +// err = s.Add(&Dummy{}, "/path/to/__special__path/file3.txt") +// require.NoError(err) + +// file, err = s.Get("/path/to/__special__path/file3.txt") +// require.NoError(err) +// require.Equal(&Dummy{}, file) + +// s.Clear() +// } diff --git a/src/fs/torrent.go b/src/host/vfs/torrent.go similarity index 62% rename from src/fs/torrent.go rename to src/host/vfs/torrent.go index 8605d84..99b2af1 100644 --- a/src/fs/torrent.go +++ b/src/host/vfs/torrent.go @@ -1,4 +1,4 @@ -package fs +package vfs import ( "context" @@ -11,71 +11,66 @@ import ( "github.com/anacrolix/torrent" ) -var _ Filesystem = &Torrent{} +var _ Filesystem = &TorrentFs{} -type Torrent struct { +type TorrentFs struct { mu sync.RWMutex - ts map[string]*torrent.Torrent - s *storage - loaded bool + t *torrent.Torrent readTimeout int + + resolver *resolver } -func NewTorrent(readTimeout int) *Torrent { - return &Torrent{ - s: newStorage(SupportedFactories), - ts: make(map[string]*torrent.Torrent), +func NewTorrentFs(t *torrent.Torrent, readTimeout int) *TorrentFs { + return &TorrentFs{ + t: t, readTimeout: readTimeout, + resolver: newResolver(ArchiveFactories), } } -func (fs *Torrent) AddTorrent(t *torrent.Torrent) { - fs.mu.Lock() - defer fs.mu.Unlock() - fs.loaded = false - fs.ts[t.InfoHash().HexString()] = t -} - -func (fs *Torrent) RemoveTorrent(h string) { - fs.mu.Lock() - defer fs.mu.Unlock() - - fs.s.Clear() - - fs.loaded = false - - delete(fs.ts, h) -} - -func (fs *Torrent) load() { - if fs.loaded { - return - } - fs.mu.RLock() - defer fs.mu.RUnlock() - - for _, t := range fs.ts { - <-t.GotInfo() - for _, file := range t.Files() { - fs.s.Add(&torrentFile{ - readerFunc: file.NewReader, - len: file.Length(), - timeout: fs.readTimeout, - }, file.Path()) +func (fs *TorrentFs) files() map[string]*torrentFile { + files := make(map[string]*torrentFile) + <-fs.t.GotInfo() + for _, file := range fs.t.Files() { + p := Clean(file.Path()) + files[p] = &torrentFile{ + readerFunc: file.NewReader, + len: file.Length(), + timeout: fs.readTimeout, } } - fs.loaded = true + return files } -func (fs *Torrent) Open(filename string) (File, error) { - fs.load() - return fs.s.Get(filename) +func (fs *TorrentFs) rawOpen(path string) (File, error) { + file, err := getFile(fs.files(), path) + return file, err } -func (fs *Torrent) ReadDir(path string) (map[string]File, error) { - fs.load() - return fs.s.Children(path) +func (fs *TorrentFs) Open(filename string) (File, error) { + fsPath, nestedFs, nestedFsPath, err := fs.resolver.resolvePath(filename, fs.rawOpen) + if err != nil { + return nil, err + } + if nestedFs != nil { + return nestedFs.Open(nestedFsPath) + } + + return fs.rawOpen(fsPath) +} + +func (fs *TorrentFs) ReadDir(name string) (map[string]File, error) { + fsPath, nestedFs, nestedFsPath, err := fs.resolver.resolvePath(name, fs.rawOpen) + if err != nil { + return nil, err + } + if nestedFs != nil { + return nestedFs.ReadDir(nestedFsPath) + } + + return listFilesInDir(fs.files(), fsPath) } type reader interface { @@ -93,7 +88,9 @@ type readAtWrapper struct { } func newReadAtWrapper(r torrent.Reader, timeout int) reader { - return &readAtWrapper{Reader: r, timeout: timeout} + w := &readAtWrapper{Reader: r, timeout: timeout} + w.SetResponsive() + return w } func (rw *readAtWrapper) ReadAt(p []byte, off int64) (int, error) { diff --git a/src/fs/torrent_test.go b/src/host/vfs/torrent_test.go similarity index 62% rename from src/fs/torrent_test.go rename to src/host/vfs/torrent_test.go index 9ac3f7e..5f5c3cc 100644 --- a/src/fs/torrent_test.go +++ b/src/host/vfs/torrent_test.go @@ -1,4 +1,4 @@ -package fs +package vfs import ( "os" @@ -34,55 +34,55 @@ func TestMain(m *testing.M) { os.Exit(exitVal) } -func TestTorrentFilesystem(t *testing.T) { - require := require.New(t) +// func TestTorrentFilesystem(t *testing.T) { +// require := require.New(t) - to, err := Cli.AddMagnet(testMagnet) - require.NoError(err) +// to, err := Cli.AddMagnet(testMagnet) +// require.NoError(err) - tfs := NewTorrent(600) - tfs.AddTorrent(to) +// tfs := NewTorrentFs(600) +// tfs.AddTorrent(to) - files, err := tfs.ReadDir("/") - require.NoError(err) - require.Len(files, 1) - require.Contains(files, "The WIRED CD - Rip. Sample. Mash. Share") +// files, err := tfs.ReadDir("/") +// require.NoError(err) +// require.Len(files, 1) +// require.Contains(files, "The WIRED CD - Rip. Sample. Mash. Share") - files, err = tfs.ReadDir("/The WIRED CD - Rip. Sample. Mash. Share") - require.NoError(err) - require.Len(files, 18) +// files, err = tfs.ReadDir("/The WIRED CD - Rip. Sample. Mash. Share") +// require.NoError(err) +// require.Len(files, 18) - f, err := tfs.Open("/The WIRED CD - Rip. Sample. Mash. Share/not_existing_file.txt") - require.Equal(os.ErrNotExist, err) - require.Nil(f) +// f, err := tfs.Open("/The WIRED CD - Rip. Sample. Mash. Share/not_existing_file.txt") +// require.Equal(os.ErrNotExist, err) +// require.Nil(f) - f, err = tfs.Open("/The WIRED CD - Rip. Sample. Mash. Share/01 - Beastie Boys - Now Get Busy.mp3") - require.NoError(err) - require.NotNil(f) - require.Equal(f.Size(), int64(1964275)) +// f, err = tfs.Open("/The WIRED CD - Rip. Sample. Mash. Share/01 - Beastie Boys - Now Get Busy.mp3") +// require.NoError(err) +// require.NotNil(f) +// require.Equal(f.Size(), int64(1964275)) - b := make([]byte, 10) +// b := make([]byte, 10) - n, err := f.Read(b) - require.NoError(err) - require.Equal(10, n) - require.Equal([]byte{0x49, 0x44, 0x33, 0x3, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x76}, b) +// n, err := f.Read(b) +// require.NoError(err) +// require.Equal(10, n) +// require.Equal([]byte{0x49, 0x44, 0x33, 0x3, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x76}, b) - n, err = f.ReadAt(b, 10) - require.NoError(err) - require.Equal(10, n) +// n, err = f.ReadAt(b, 10) +// require.NoError(err) +// require.Equal(10, n) - n, err = f.ReadAt(b, 10000) - require.NoError(err) - require.Equal(10, n) +// n, err = f.ReadAt(b, 10000) +// require.NoError(err) +// require.Equal(10, n) - tfs.RemoveTorrent(to.InfoHash().String()) - files, err = tfs.ReadDir("/") - require.NoError(err) - require.Len(files, 0) +// tfs.RemoveTorrent(to.InfoHash().String()) +// files, err = tfs.ReadDir("/") +// require.NoError(err) +// require.Len(files, 0) - require.NoError(f.Close()) -} +// require.NoError(f.Close()) +// } func TestReadAtTorrent(t *testing.T) { require := require.New(t) diff --git a/src/host/vfs/utils.go b/src/host/vfs/utils.go new file mode 100644 index 0000000..d4b2d77 --- /dev/null +++ b/src/host/vfs/utils.go @@ -0,0 +1,55 @@ +package vfs + +import ( + "io/fs" + "path" + "strings" +) + +var ErrNotExist = fs.ErrNotExist + +func getFile[F File](m map[string]F, name string) (File, error) { + name = Clean(name) + if name == Separator { + return &Dir{}, nil + } + + f, ok := m[name] + if ok { + return f, nil + } + + for p := range m { + if strings.HasPrefix(p, name) { + return &Dir{}, nil + } + } + + return nil, ErrNotExist +} + +func listFilesInDir[F File](m map[string]F, name string) (map[string]File, error) { + name = Clean(name) + + out := map[string]File{} + for p, f := range m { + if strings.HasPrefix(p, name) { + parts := strings.Split(trimRelPath(p, name), Separator) + if len(parts) == 1 { + out[parts[0]] = f + } else { + out[parts[0]] = &Dir{} + } + } + } + + return out, nil +} + +func trimRelPath(p, t string) string { + return strings.Trim(strings.TrimPrefix(p, t), "/") +} + +func Clean(p string) string { + return path.Clean(Separator + strings.ReplaceAll(p, "\\", "/")) +} diff --git a/src/http/api.go b/src/http/api.go index 6422ad4..c227c2e 100644 --- a/src/http/api.go +++ b/src/http/api.go @@ -6,9 +6,8 @@ import ( "math" "net/http" "os" - "sort" - "git.kmsign.ru/royalcat/tstor/src/torrent" + "git.kmsign.ru/royalcat/tstor/src/host/torrent" "github.com/anacrolix/missinggo/v2/filecache" "github.com/gin-gonic/gin" ) @@ -30,56 +29,56 @@ var apiStatusHandler = func(fc *filecache.Cache, ss *torrent.Stats) gin.HandlerF } } -var apiServersHandler = func(ss []*torrent.Server) gin.HandlerFunc { - return func(ctx *gin.Context) { - var infos []*torrent.ServerInfo - for _, s := range ss { - infos = append(infos, s.Info()) - } - ctx.JSON(http.StatusOK, infos) - } -} +// var apiServersHandler = func(ss []*torrent.Server) gin.HandlerFunc { +// return func(ctx *gin.Context) { +// var infos []*torrent.ServerInfo +// for _, s := range ss { +// infos = append(infos, s.Info()) +// } +// ctx.JSON(http.StatusOK, infos) +// } +// } -var apiRoutesHandler = func(ss *torrent.Stats) gin.HandlerFunc { - return func(ctx *gin.Context) { - s := ss.RoutesStats() - sort.Sort(torrent.ByName(s)) - ctx.JSON(http.StatusOK, s) - } -} +// var apiRoutesHandler = func(ss *torrent.Stats) gin.HandlerFunc { +// return func(ctx *gin.Context) { +// s := ss.RoutesStats() +// sort.Sort(torrent.ByName(s)) +// ctx.JSON(http.StatusOK, s) +// } +// } -var apiAddTorrentHandler = func(s *torrent.Service) gin.HandlerFunc { - return func(ctx *gin.Context) { - route := ctx.Param("route") +// var apiAddTorrentHandler = func(s *torrent.Service) gin.HandlerFunc { +// return func(ctx *gin.Context) { +// route := ctx.Param("route") - var json RouteAdd - if err := ctx.ShouldBindJSON(&json); err != nil { - ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } +// var json RouteAdd +// if err := ctx.ShouldBindJSON(&json); err != nil { +// ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) +// return +// } - if err := s.AddMagnet(route, json.Magnet); err != nil { - ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } +// if err := s.AddMagnet(route, json.Magnet); err != nil { +// ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) +// return +// } - ctx.JSON(http.StatusOK, nil) - } -} +// ctx.JSON(http.StatusOK, nil) +// } +// } -var apiDelTorrentHandler = func(s *torrent.Service) gin.HandlerFunc { - return func(ctx *gin.Context) { - route := ctx.Param("route") - hash := ctx.Param("torrent_hash") +// var apiDelTorrentHandler = func(s *torrent.Service) gin.HandlerFunc { +// return func(ctx *gin.Context) { +// route := ctx.Param("route") +// hash := ctx.Param("torrent_hash") - if err := s.RemoveFromHash(route, hash); err != nil { - ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } +// if err := s.RemoveFromHash(route, hash); err != nil { +// ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) +// return +// } - ctx.JSON(http.StatusOK, nil) - } -} +// ctx.JSON(http.StatusOK, nil) +// } +// } var apiLogHandler = func(path string) gin.HandlerFunc { return func(ctx *gin.Context) { diff --git a/src/http/http.go b/src/http/http.go index c91bfd4..7f2fa27 100644 --- a/src/http/http.go +++ b/src/http/http.go @@ -6,14 +6,14 @@ import ( "git.kmsign.ru/royalcat/tstor" "git.kmsign.ru/royalcat/tstor/src/config" - "git.kmsign.ru/royalcat/tstor/src/torrent" + "git.kmsign.ru/royalcat/tstor/src/host/torrent" "github.com/anacrolix/missinggo/v2/filecache" "github.com/gin-gonic/gin" "github.com/rs/zerolog/log" "github.com/shurcooL/httpfs/html/vfstemplate" ) -func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.Config, tss []*torrent.Server, fs http.FileSystem, logPath string, cfg *config.Config) error { +func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, logPath string, cfg *config.Config) error { gin.SetMode(gin.ReleaseMode) r := gin.New() r.Use(gin.Recovery()) @@ -24,14 +24,6 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config. c.FileFromFS(c.Request.URL.Path, http.FS(tstor.Assets)) }) - if cfg.Mounts.HttpFs.Enabled { - log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d/fs", cfg.Mounts.HttpFs.Port)).Msg("starting HTTPFS") - r.GET("/fs/*filepath", func(c *gin.Context) { - path := c.Param("filepath") - c.FileFromFS(path, fs) - }) - } - t, err := vfstemplate.ParseGlob(http.FS(tstor.Templates), nil, "/templates/*") if err != nil { return fmt.Errorf("error parsing html: %w", err) @@ -40,7 +32,7 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config. r.SetHTMLTemplate(t) r.GET("/", indexHandler) - r.GET("/routes", routesHandler(ss)) + // r.GET("/routes", routesHandler(ss)) r.GET("/logs", logsHandler) r.GET("/servers", serversFoldersHandler()) @@ -48,11 +40,11 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config. { api.GET("/log", apiLogHandler(logPath)) api.GET("/status", apiStatusHandler(fc, ss)) - api.GET("/servers", apiServersHandler(tss)) + // api.GET("/servers", apiServersHandler(tss)) - api.GET("/routes", apiRoutesHandler(ss)) - api.POST("/routes/:route/torrent", apiAddTorrentHandler(s)) - api.DELETE("/routes/:route/torrent/:torrent_hash", apiDelTorrentHandler(s)) + // api.GET("/routes", apiRoutesHandler(ss)) + // api.POST("/routes/:route/torrent", apiAddTorrentHandler(s)) + // api.DELETE("/routes/:route/torrent/:torrent_hash", apiDelTorrentHandler(s)) } diff --git a/src/http/web.go b/src/http/web.go index 9140c5f..860347f 100644 --- a/src/http/web.go +++ b/src/http/web.go @@ -3,7 +3,6 @@ package http import ( "net/http" - "git.kmsign.ru/royalcat/tstor/src/torrent" "github.com/gin-gonic/gin" ) @@ -11,11 +10,11 @@ var indexHandler = func(c *gin.Context) { c.HTML(http.StatusOK, "index.html", nil) } -var routesHandler = func(ss *torrent.Stats) gin.HandlerFunc { - return func(c *gin.Context) { - c.HTML(http.StatusOK, "routes.html", ss.RoutesStats()) - } -} +// var routesHandler = func(ss *torrent.Stats) gin.HandlerFunc { +// return func(c *gin.Context) { +// c.HTML(http.StatusOK, "routes.html", ss.RoutesStats()) +// } +// } var logsHandler = func(c *gin.Context) { c.HTML(http.StatusOK, "logs.html", nil) diff --git a/src/iio/wrapper_test.go b/src/iio/wrapper_test.go index 7197d1f..071ce92 100644 --- a/src/iio/wrapper_test.go +++ b/src/iio/wrapper_test.go @@ -4,7 +4,7 @@ import ( "io" "testing" - "git.kmsign.ru/royalcat/tstor/src/fs" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" "git.kmsign.ru/royalcat/tstor/src/iio" "github.com/stretchr/testify/require" ) @@ -16,7 +16,7 @@ func TestSeekerWrapper(t *testing.T) { require := require.New(t) - mf := fs.NewMemoryFile(testData) + mf := vfs.NewMemoryFile(testData) r := iio.NewSeekerWrapper(mf, mf.Size()) defer r.Close() diff --git a/src/mounts/fuse/handler.go b/src/mounts/fuse/handler.go index 8b3c129..c3b4af9 100644 --- a/src/mounts/fuse/handler.go +++ b/src/mounts/fuse/handler.go @@ -5,7 +5,7 @@ import ( "path/filepath" "runtime" - "git.kmsign.ru/royalcat/tstor/src/fs" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/billziss-gh/cgofuse/fuse" "github.com/rs/zerolog/log" @@ -25,7 +25,7 @@ func NewHandler(fuseAllowOther bool, path string) *Handler { } } -func (s *Handler) Mount(fss map[string]fs.Filesystem) error { +func (s *Handler) Mount(vfs vfs.Filesystem) error { folder := s.path // On windows, the folder must don't exist if runtime.GOOS == "windows" { @@ -38,12 +38,7 @@ func (s *Handler) Mount(fss map[string]fs.Filesystem) error { } } - cfs, err := fs.NewContainerFs(fss) - if err != nil { - return err - } - - host := fuse.NewFileSystemHost(NewFS(cfs)) + host := fuse.NewFileSystemHost(NewFS(vfs)) // TODO improve error handling here go func() { diff --git a/src/mounts/fuse/mount.go b/src/mounts/fuse/mount.go index 3de1568..e57a6f2 100644 --- a/src/mounts/fuse/mount.go +++ b/src/mounts/fuse/mount.go @@ -7,7 +7,7 @@ import ( "os" "sync" - "git.kmsign.ru/royalcat/tstor/src/fs" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/billziss-gh/cgofuse/fuse" "github.com/rs/zerolog" @@ -21,7 +21,7 @@ type FS struct { log zerolog.Logger } -func NewFS(fs fs.Filesystem) fuse.FileSystemInterface { +func NewFS(fs vfs.Filesystem) fuse.FileSystemInterface { l := log.Logger.With().Str("component", "fuse").Logger() return &FS{ fh: &fileHandler{fs: fs}, @@ -154,11 +154,11 @@ var ErrBadHolderIndex = errors.New("holder index too big") type fileHandler struct { mu sync.RWMutex - opened []fs.File - fs fs.Filesystem + opened []vfs.File + fs vfs.Filesystem } -func (fh *fileHandler) GetFile(path string, fhi uint64) (fs.File, error) { +func (fh *fileHandler) GetFile(path string, fhi uint64) (vfs.File, error) { fh.mu.RLock() defer fh.mu.RUnlock() @@ -204,7 +204,7 @@ func (fh *fileHandler) OpenHolder(path string) (uint64, error) { return uint64(len(fh.opened) - 1), nil } -func (fh *fileHandler) get(fhi uint64) (fs.File, error) { +func (fh *fileHandler) get(fhi uint64) (vfs.File, error) { if int(fhi) >= len(fh.opened) { return nil, ErrBadHolderIndex } @@ -240,7 +240,7 @@ func (fh *fileHandler) Remove(fhi uint64) error { return nil } -func (fh *fileHandler) lookupFile(path string) (fs.File, error) { +func (fh *fileHandler) lookupFile(path string) (vfs.File, error) { file, err := fh.fs.Open(path) if err != nil { return nil, err diff --git a/src/mounts/fuse/mount_test.go b/src/mounts/fuse/mount_test.go index 0851926..0717955 100644 --- a/src/mounts/fuse/mount_test.go +++ b/src/mounts/fuse/mount_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "git.kmsign.ru/royalcat/tstor/src/fs" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/stretchr/testify/require" ) @@ -22,12 +22,11 @@ func TestHandler(t *testing.T) { h := NewHandler(false, p) - mem := fs.NewMemory() + mem := vfs.NewMemoryFS(map[string]*vfs.MemoryFile{ + "/test.txt": vfs.NewMemoryFile([]byte("test")), + }) - err := mem.Storage.Add(fs.NewMemoryFile([]byte("test")), "/test.txt") - require.NoError(err) - - err = h.Mount(map[string]fs.Filesystem{"/mem": mem}) + err := h.Mount(mem) require.NoError(err) time.Sleep(5 * time.Second) @@ -50,12 +49,11 @@ func TestHandlerDriveLetter(t *testing.T) { h := NewHandler(false, p) - mem := fs.NewMemory() + mem := vfs.NewMemoryFS(map[string]*vfs.MemoryFile{ + "/test.txt": vfs.NewMemoryFile([]byte("test")), + }) - err := mem.Storage.Add(fs.NewMemoryFile([]byte("test")), "/test.txt") - require.NoError(err) - - err = h.Mount(map[string]fs.Filesystem{"/mem": mem}) + err := h.Mount(mem) require.NoError(err) time.Sleep(5 * time.Second) diff --git a/src/mounts/httpfs/httpfs.go b/src/mounts/httpfs/httpfs.go index 233572f..f593903 100644 --- a/src/mounts/httpfs/httpfs.go +++ b/src/mounts/httpfs/httpfs.go @@ -7,17 +7,17 @@ import ( "os" "sync" - dfs "git.kmsign.ru/royalcat/tstor/src/fs" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" "git.kmsign.ru/royalcat/tstor/src/iio" ) var _ http.FileSystem = &HTTPFS{} type HTTPFS struct { - fs dfs.Filesystem + fs vfs.Filesystem } -func NewHTTPFS(fs dfs.Filesystem) *HTTPFS { +func NewHTTPFS(fs vfs.Filesystem) *HTTPFS { return &HTTPFS{fs: fs} } @@ -27,7 +27,7 @@ func (fs *HTTPFS) Open(name string) (http.File, error) { return nil, err } - fi := dfs.NewFileInfo(name, f.Size(), f.IsDir()) + fi := vfs.NewFileInfo(name, f.Size(), f.IsDir()) // TODO make this lazy fis, err := fs.filesToFileInfo(name) @@ -46,7 +46,7 @@ func (fs *HTTPFS) filesToFileInfo(path string) ([]fs.FileInfo, error) { var out []os.FileInfo for n, f := range files { - out = append(out, dfs.NewFileInfo(n, f.Size(), f.IsDir())) + out = append(out, vfs.NewFileInfo(n, f.Size(), f.IsDir())) } return out, nil @@ -65,7 +65,7 @@ type httpFile struct { fi fs.FileInfo } -func newHTTPFile(f dfs.File, fis []fs.FileInfo, fi fs.FileInfo) *httpFile { +func newHTTPFile(f vfs.File, fis []fs.FileInfo, fi fs.FileInfo) *httpFile { return &httpFile{ dirContent: fis, fi: fi, diff --git a/src/mounts/webdav/fs.go b/src/mounts/webdav/fs.go index c82838c..29f2727 100644 --- a/src/mounts/webdav/fs.go +++ b/src/mounts/webdav/fs.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "git.kmsign.ru/royalcat/tstor/src/fs" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" "git.kmsign.ru/royalcat/tstor/src/iio" "golang.org/x/net/webdav" ) @@ -16,10 +16,10 @@ import ( var _ webdav.FileSystem = &WebDAV{} type WebDAV struct { - fs fs.Filesystem + fs vfs.Filesystem } -func newFS(fs fs.Filesystem) *WebDAV { +func newFS(fs vfs.Filesystem) *WebDAV { return &WebDAV{fs: fs} } @@ -59,7 +59,7 @@ func (wd *WebDAV) Rename(ctx context.Context, oldName, newName string) error { return webdav.ErrNotImplemented } -func (wd *WebDAV) lookupFile(path string) (fs.File, error) { +func (wd *WebDAV) lookupFile(path string) (vfs.File, error) { return wd.fs.Open(path) } @@ -93,7 +93,7 @@ type webDAVFile struct { dirContent []os.FileInfo } -func newFile(name string, f fs.File, df func() ([]os.FileInfo, error)) *webDAVFile { +func newFile(name string, f vfs.File, df func() ([]os.FileInfo, error)) *webDAVFile { return &webDAVFile{ fi: newFileInfo(name, f.Size(), f.IsDir()), dirFunc: df, diff --git a/src/mounts/webdav/fs_test.go b/src/mounts/webdav/fs_test.go index f25f380..5d703e8 100644 --- a/src/mounts/webdav/fs_test.go +++ b/src/mounts/webdav/fs_test.go @@ -6,7 +6,7 @@ import ( "os" "testing" - "git.kmsign.ru/royalcat/tstor/src/fs" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/stretchr/testify/require" "golang.org/x/net/webdav" ) @@ -16,10 +16,9 @@ func TestWebDAVFilesystem(t *testing.T) { require := require.New(t) - mfs := fs.NewMemory() - mf := fs.NewMemoryFile([]byte("test file content.")) - err := mfs.Storage.Add(mf, "/folder/file.txt") - require.NoError(err) + mfs := vfs.NewMemoryFS(map[string]*vfs.MemoryFile{ + "/folder/file.txt": vfs.NewMemoryFile([]byte("test file content.")), + }) wfs := newFS(mfs) @@ -67,10 +66,9 @@ func TestErrNotImplemented(t *testing.T) { require := require.New(t) - mfs := fs.NewMemory() - mf := fs.NewMemoryFile([]byte("test file content.")) - err := mfs.Storage.Add(mf, "/folder/file.txt") - require.NoError(err) + mfs := vfs.NewMemoryFS(map[string]*vfs.MemoryFile{ + "/folder/file.txt": vfs.NewMemoryFile([]byte("test file content.")), + }) wfs := newFS(mfs) diff --git a/src/mounts/webdav/handler.go b/src/mounts/webdav/handler.go index f7cb756..3c478ce 100644 --- a/src/mounts/webdav/handler.go +++ b/src/mounts/webdav/handler.go @@ -3,12 +3,12 @@ package webdav import ( "net/http" - "git.kmsign.ru/royalcat/tstor/src/fs" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/rs/zerolog/log" "golang.org/x/net/webdav" ) -func newHandler(fs fs.Filesystem) *webdav.Handler { +func newHandler(fs vfs.Filesystem) *webdav.Handler { l := log.Logger.With().Str("component", "webDAV").Logger() return &webdav.Handler{ Prefix: "/", diff --git a/src/mounts/webdav/http.go b/src/mounts/webdav/http.go index 266c259..3b95a83 100644 --- a/src/mounts/webdav/http.go +++ b/src/mounts/webdav/http.go @@ -4,11 +4,11 @@ import ( "fmt" "net/http" - "git.kmsign.ru/royalcat/tstor/src/fs" + "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/rs/zerolog/log" ) -func NewWebDAVServer(fs fs.Filesystem, port int, user, pass string) error { +func NewWebDAVServer(fs vfs.Filesystem, port int, user, pass string) error { log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", port)).Msg("starting webDAV server") srv := newHandler(fs) diff --git a/src/torrent/loader/config.go b/src/torrent/loader/config.go deleted file mode 100644 index e541975..0000000 --- a/src/torrent/loader/config.go +++ /dev/null @@ -1,45 +0,0 @@ -package loader - -import "git.kmsign.ru/royalcat/tstor/src/config" - -var _ Loader = &Config{} - -type Config struct { - c []config.Route -} - -func NewConfig(r []config.Route) *Config { - return &Config{ - c: r, - } -} - -func (l *Config) ListMagnets() (map[string][]string, error) { - out := make(map[string][]string) - for _, r := range l.c { - for _, t := range r.Torrents { - if t.MagnetURI == "" { - continue - } - - out[r.Name] = append(out[r.Name], t.MagnetURI) - } - } - - return out, nil -} - -func (l *Config) ListTorrentPaths() (map[string][]string, error) { - out := make(map[string][]string) - for _, r := range l.c { - for _, t := range r.Torrents { - if t.TorrentPath == "" { - continue - } - - out[r.Name] = append(out[r.Name], t.TorrentPath) - } - } - - return out, nil -} diff --git a/src/torrent/loader/db.go b/src/torrent/loader/db.go deleted file mode 100644 index f4a419f..0000000 --- a/src/torrent/loader/db.go +++ /dev/null @@ -1,112 +0,0 @@ -package loader - -import ( - "path" - - dlog "git.kmsign.ru/royalcat/tstor/src/log" - "github.com/anacrolix/torrent/metainfo" - "github.com/dgraph-io/badger/v3" - "github.com/rs/zerolog/log" -) - -var _ LoaderAdder = &DB{} - -const routeRootKey = "/route/" - -type DB struct { - db *badger.DB -} - -func NewDB(path string) (*DB, error) { - l := log.Logger.With().Str("component", "torrent-store").Logger() - - opts := badger.DefaultOptions(path). - WithLogger(&dlog.Badger{L: l}). - WithValueLogFileSize(1<<26 - 1) - - db, err := badger.Open(opts) - if err != nil { - return nil, err - } - - err = db.RunValueLogGC(0.5) - if err != nil && err != badger.ErrNoRewrite { - return nil, err - } - - return &DB{ - db: db, - }, nil -} - -func (l *DB) AddMagnet(r, m string) error { - err := l.db.Update(func(txn *badger.Txn) error { - spec, err := metainfo.ParseMagnetUri(m) - if err != nil { - return err - } - - ih := spec.InfoHash.HexString() - - rp := path.Join(routeRootKey, ih, r) - return txn.Set([]byte(rp), []byte(m)) - }) - - if err != nil { - return err - } - - return l.db.Sync() -} - -func (l *DB) RemoveFromHash(r, h string) (bool, error) { - tx := l.db.NewTransaction(true) - defer tx.Discard() - - var mh metainfo.Hash - if err := mh.FromHexString(h); err != nil { - return false, err - } - - rp := path.Join(routeRootKey, h, r) - if _, err := tx.Get([]byte(rp)); err != nil { - return false, nil - } - - if err := tx.Delete([]byte(rp)); err != nil { - return false, err - } - - return true, tx.Commit() -} - -func (l *DB) ListMagnets() (map[string][]string, error) { - tx := l.db.NewTransaction(false) - defer tx.Discard() - - it := tx.NewIterator(badger.DefaultIteratorOptions) - defer it.Close() - - prefix := []byte(routeRootKey) - out := make(map[string][]string) - for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - _, r := path.Split(string(it.Item().Key())) - i := it.Item() - if err := i.Value(func(v []byte) error { - out[r] = append(out[r], string(v)) - return nil - }); err != nil { - return nil, err - } - } - - return out, nil -} - -func (l *DB) ListTorrentPaths() (map[string][]string, error) { - return nil, nil -} - -func (l *DB) Close() error { - return l.db.Close() -} diff --git a/src/torrent/loader/db_test.go b/src/torrent/loader/db_test.go deleted file mode 100644 index 5811d22..0000000 --- a/src/torrent/loader/db_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package loader - -import ( - "os" - "testing" - - "github.com/anacrolix/torrent/storage" - "github.com/stretchr/testify/require" -) - -const m1 = "magnet:?xt=urn:btih:c9e15763f722f23e98a29decdfae341b98d53056" - -func TestDB(t *testing.T) { - require := require.New(t) - - tmpService, err := os.MkdirTemp("", "service") - require.NoError(err) - tmpStorage, err := os.MkdirTemp("", "storage") - require.NoError(err) - - cs := storage.NewFile(tmpStorage) - defer cs.Close() - - s, err := NewDB(tmpService) - require.NoError(err) - defer s.Close() - - err = s.AddMagnet("route1", "WRONG MAGNET") - require.Error(err) - - err = s.AddMagnet("route1", m1) - require.NoError(err) - - err = s.AddMagnet("route2", m1) - require.NoError(err) - - l, err := s.ListMagnets() - require.NoError(err) - require.Len(l, 2) - require.Len(l["route1"], 1) - require.Equal(l["route1"][0], m1) - require.Len(l["route2"], 1) - require.Equal(l["route2"][0], m1) - - removed, err := s.RemoveFromHash("other", "c9e15763f722f23e98a29decdfae341b98d53056") - require.NoError(err) - require.False(removed) - - removed, err = s.RemoveFromHash("route1", "c9e15763f722f23e98a29decdfae341b98d53056") - require.NoError(err) - require.True(removed) - - l, err = s.ListMagnets() - require.NoError(err) - require.Len(l, 1) - require.Len(l["route2"], 1) - require.Equal(l["route2"][0], m1) - - require.NoError(s.Close()) - require.NoError(cs.Close()) - -} diff --git a/src/torrent/loader/folder.go b/src/torrent/loader/folder.go deleted file mode 100644 index bd5efc0..0000000 --- a/src/torrent/loader/folder.go +++ /dev/null @@ -1,55 +0,0 @@ -package loader - -import ( - "io/fs" - "path" - "path/filepath" - - "git.kmsign.ru/royalcat/tstor/src/config" -) - -var _ Loader = &Folder{} - -type Folder struct { - c []config.Route -} - -func NewFolder(r []config.Route) *Folder { - return &Folder{ - c: r, - } -} - -func (f *Folder) ListMagnets() (map[string][]string, error) { - return nil, nil -} - -func (f *Folder) ListTorrentPaths() (map[string][]string, error) { - out := make(map[string][]string) - for _, r := range f.c { - if r.TorrentFolder == "" { - continue - } - - err := filepath.WalkDir(r.TorrentFolder, func(p string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - if d.IsDir() { - return nil - } - if path.Ext(p) == ".torrent" { - out[r.Name] = append(out[r.Name], p) - } - - return nil - }) - - if err != nil { - return nil, err - } - } - - return out, nil -} diff --git a/src/torrent/loader/loader.go b/src/torrent/loader/loader.go deleted file mode 100644 index 586fd61..0000000 --- a/src/torrent/loader/loader.go +++ /dev/null @@ -1,13 +0,0 @@ -package loader - -type Loader interface { - ListMagnets() (map[string][]string, error) - ListTorrentPaths() (map[string][]string, error) -} - -type LoaderAdder interface { - Loader - - RemoveFromHash(r, h string) (bool, error) - AddMagnet(r, m string) error -} diff --git a/src/torrent/server.go b/src/torrent/server.go deleted file mode 100644 index c6a380d..0000000 --- a/src/torrent/server.go +++ /dev/null @@ -1,255 +0,0 @@ -package torrent - -import ( - "fmt" - "os" - "path/filepath" - "sync" - "sync/atomic" - "time" - - "git.kmsign.ru/royalcat/tstor/src/config" - "github.com/anacrolix/torrent" - "github.com/anacrolix/torrent/bencode" - "github.com/anacrolix/torrent/metainfo" - "github.com/anacrolix/torrent/storage" - "github.com/fsnotify/fsnotify" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" -) - -type ServerState int - -const ( - UNKNOWN ServerState = iota - SEEDING - READING - UPDATING - STOPPED - ERROR -) - -func (ss ServerState) String() string { - return [...]string{"Unknown", "Seeding", "Reading", "Updating", "Stopped", "Error"}[ss] -} - -type ServerInfo struct { - Magnet string `json:"magnetUri"` - UpdatedAt int64 `json:"updatedAt"` - Name string `json:"name"` - Folder string `json:"folder"` - State string `json:"state"` - Peers int `json:"peers"` - Seeds int `json:"seeds"` -} - -type Server struct { - cfg *config.Server - log zerolog.Logger - - fw *fsnotify.Watcher - - eventsCount uint64 - - c *torrent.Client - pc storage.PieceCompletion - - mu sync.RWMutex - t *torrent.Torrent - si ServerInfo -} - -func NewServer(c *torrent.Client, pc storage.PieceCompletion, cfg *config.Server) *Server { - l := log.Logger.With().Str("component", "server").Str("name", cfg.Name).Logger() - - return &Server{ - cfg: cfg, - log: l, - c: c, - pc: pc, - } -} - -func (s *Server) Start() error { - s.log.Info().Msg("starting new server folder") - w, err := fsnotify.NewWatcher() - if err != nil { - return err - } - - if err := os.MkdirAll(s.cfg.Path, 0744); err != nil { - return fmt.Errorf("error creating server folder: %s. Error: %w", s.cfg.Path, err) - } - - if err := filepath.Walk(s.cfg.Path, - func(path string, info os.FileInfo, err error) error { - if info.Mode().IsDir() { - s.log.Debug().Str("folder", path).Msg("adding new folder") - return w.Add(path) - } - - return nil - }); err != nil { - return err - } - - s.fw = w - go func() { - if err := s.makeMagnet(); err != nil { - s.updateState(ERROR) - s.log.Error().Err(err).Msg("error generating magnet on start") - } - - s.watch() - }() - - go func() { - for { - select { - case event, ok := <-w.Events: - if !ok { - return - } - - s.log.Info().Str("file", event.Name).Str("op", event.Op.String()).Msg("file changed inside server folder") - atomic.AddUint64(&s.eventsCount, 1) - case err, ok := <-w.Errors: - if !ok { - return - } - - s.updateState(STOPPED) - s.log.Error().Err(err).Msg("error watching server folder") - } - } - }() - - s.log.Info().Msg("server folder started") - - return nil -} - -func (s *Server) watch() { - s.log.Info().Msg("starting watcher") - for range time.Tick(time.Second * 5) { - if s.eventsCount == 0 { - continue - } - - ec := s.eventsCount - if err := s.makeMagnet(); err != nil { - s.updateState(ERROR) - s.log.Error().Err(err).Msg("error generating magnet") - } - - atomic.AddUint64(&s.eventsCount, -ec) - } -} - -func (s *Server) makeMagnet() error { - - s.log.Info().Msg("starting serving new torrent") - - info := metainfo.Info{ - PieceLength: 2 << 18, - } - - s.updateState(READING) - - if err := info.BuildFromFilePath(s.cfg.Path); err != nil { - return err - } - - s.updateState(UPDATING) - - if len(info.Files) == 0 { - s.mu.Lock() - s.si.Magnet = "" - s.si.Folder = s.cfg.Path - s.si.Name = s.cfg.Name - s.si.UpdatedAt = time.Now().Unix() - s.mu.Unlock() - s.log.Info().Msg("not creating magnet from empty folder") - - s.updateState(STOPPED) - return nil - } - - mi := metainfo.MetaInfo{ - InfoBytes: bencode.MustMarshal(info), - } - - ih := mi.HashInfoBytes() - - to, _ := s.c.AddTorrentOpt(torrent.AddTorrentOpts{ - InfoHash: ih, - Storage: storage.NewFileOpts(storage.NewFileClientOpts{ - ClientBaseDir: s.cfg.Path, - FilePathMaker: func(opts storage.FilePathMakerOpts) string { - return filepath.Join(opts.File.Path...) - }, - TorrentDirMaker: nil, - PieceCompletion: s.pc, - }), - }) - - tks := s.trackers() - - err := to.MergeSpec(&torrent.TorrentSpec{ - InfoBytes: mi.InfoBytes, - - Trackers: [][]string{tks}, - }) - if err != nil { - return err - } - - m := metainfo.Magnet{ - InfoHash: ih, - DisplayName: s.cfg.Name, - Trackers: tks, - } - - s.mu.Lock() - s.t = to - s.si.Magnet = m.String() - s.si.Folder = s.cfg.Path - s.si.Name = s.cfg.Name - s.si.UpdatedAt = time.Now().Unix() - s.mu.Unlock() - s.updateState(SEEDING) - - s.log.Info().Str("hash", ih.HexString()).Msg("new torrent is ready") - - return nil -} - -func (s *Server) updateState(ss ServerState) { - s.mu.Lock() - s.si.State = ss.String() - s.mu.Unlock() -} - -func (s *Server) trackers() []string { - // TODO load trackers from URL too - return s.cfg.Trackers -} - -func (s *Server) Close() error { - if s.fw == nil { - return nil - } - return s.fw.Close() -} - -func (s *Server) Info() *ServerInfo { - s.mu.RLock() - defer s.mu.RUnlock() - if s.t != nil { - st := s.t.Stats() - s.si.Peers = st.TotalPeers - s.si.Seeds = st.ConnectedSeeders - } - - return &s.si -} diff --git a/src/torrent/service.go b/src/torrent/service.go deleted file mode 100644 index b147b16..0000000 --- a/src/torrent/service.go +++ /dev/null @@ -1,203 +0,0 @@ -package torrent - -import ( - "errors" - "fmt" - "path" - "sync" - "time" - - "git.kmsign.ru/royalcat/tstor/src/fs" - "git.kmsign.ru/royalcat/tstor/src/torrent/loader" - "github.com/anacrolix/torrent" - "github.com/anacrolix/torrent/metainfo" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" -) - -type Service struct { - c *torrent.Client - - s *Stats - - mu sync.Mutex - fss map[string]fs.Filesystem - - loaders []loader.Loader - db loader.LoaderAdder - - log zerolog.Logger - addTimeout, readTimeout int -} - -func NewService(loaders []loader.Loader, db loader.LoaderAdder, stats *Stats, c *torrent.Client, addTimeout, readTimeout int) *Service { - l := log.Logger.With().Str("component", "torrent-service").Logger() - return &Service{ - log: l, - s: stats, - c: c, - fss: make(map[string]fs.Filesystem), - loaders: loaders, - db: db, - addTimeout: addTimeout, - readTimeout: readTimeout, - } -} - -func (s *Service) Load() (map[string]fs.Filesystem, error) { - // Load from config - s.log.Info().Msg("adding torrents from configuration") - for _, loader := range s.loaders { - if err := s.load(loader); err != nil { - return nil, err - } - } - - // Load from DB - s.log.Info().Msg("adding torrents from database") - return s.fss, s.load(s.db) -} - -func (s *Service) load(l loader.Loader) error { - list, err := l.ListMagnets() - if err != nil { - return err - } - for r, ms := range list { - s.addRoute(r) - for _, m := range ms { - if err := s.addMagnet(r, m); err != nil { - return err - } - } - } - - list, err = l.ListTorrentPaths() - if err != nil { - return err - } - for r, ms := range list { - s.addRoute(r) - for _, p := range ms { - if err := s.addTorrentPath(r, p); err != nil { - return err - } - } - } - - return nil -} - -func (s *Service) AddMagnet(r, m string) error { - if err := s.addMagnet(r, m); err != nil { - return err - } - - // Add to db - return s.db.AddMagnet(r, m) -} - -func (s *Service) addTorrentPath(r, p string) error { - // Add to client - t, err := s.c.AddTorrentFromFile(p) - if err != nil { - return err - } - - return s.addTorrent(r, t) -} - -func (s *Service) addMagnet(r, m string) error { - // Add to client - t, err := s.c.AddMagnet(m) - if err != nil { - return err - } - - return s.addTorrent(r, t) - -} - -func (s *Service) addRoute(r string) { - s.s.AddRoute(r) - - // Add to filesystems - folder := path.Join("/", r) - s.mu.Lock() - defer s.mu.Unlock() - _, ok := s.fss[folder] - if !ok { - s.fss[folder] = fs.NewTorrent(s.readTimeout) - } -} - -func (s *Service) addTorrent(r string, t *torrent.Torrent) error { - // only get info if name is not available - if t.Info() == nil { - s.log.Info().Str("hash", t.InfoHash().String()).Msg("getting torrent info") - select { - case <-time.After(time.Duration(s.addTimeout) * time.Second): - s.log.Error().Str("hash", t.InfoHash().String()).Msg("timeout getting torrent info") - return errors.New("timeout getting torrent info") - case <-t.GotInfo(): - s.log.Info().Str("hash", t.InfoHash().String()).Msg("obtained torrent info") - } - - } - - // Add to stats - s.s.Add(r, t) - - // Add to filesystems - folder := path.Join("/", r) - s.mu.Lock() - defer s.mu.Unlock() - - tfs, ok := s.fss[folder].(*fs.Torrent) - if !ok { - return errors.New("error adding torrent to filesystem") - } - - tfs.AddTorrent(t) - s.log.Info().Str("name", t.Info().Name).Str("route", r).Msg("torrent added") - - return nil -} - -func (s *Service) RemoveFromHash(r, h string) error { - // Remove from db - deleted, err := s.db.RemoveFromHash(r, h) - if err != nil { - return err - } - - if !deleted { - return fmt.Errorf("element with hash %v on route %v cannot be removed", h, r) - } - - // Remove from stats - s.s.Del(r, h) - - // Remove from fs - folder := path.Join("/", r) - - tfs, ok := s.fss[folder].(*fs.Torrent) - if !ok { - return errors.New("error removing torrent from filesystem") - } - - tfs.RemoveTorrent(h) - - // Remove from client - var mh metainfo.Hash - if err := mh.FromHexString(h); err != nil { - return err - } - - t, ok := s.c.Torrent(metainfo.NewHashFromHex(h)) - if ok { - t.Drop() - } - - return nil -}