diff --git a/cmd/tstor/main.go b/cmd/tstor/main.go index 7db93bc..645e4a5 100644 --- a/cmd/tstor/main.go +++ b/cmd/tstor/main.go @@ -14,8 +14,8 @@ import ( "git.kmsign.ru/royalcat/tstor/src/config" "git.kmsign.ru/royalcat/tstor/src/host" - "git.kmsign.ru/royalcat/tstor/src/host/repository" - "git.kmsign.ru/royalcat/tstor/src/host/torrent" + "git.kmsign.ru/royalcat/tstor/src/host/service" + "git.kmsign.ru/royalcat/tstor/src/host/storage" "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" @@ -73,40 +73,40 @@ func run(configPath string) error { log.Err(err).Msg("set priority failed") } - rep, err := repository.NewTorrentMetaRepository(conf.TorrentClient.MetadataFolder) - if err != nil { - return err - } - if err := os.MkdirAll(conf.TorrentClient.MetadataFolder, 0744); err != nil { return fmt.Errorf("error creating metadata folder: %w", err) } - fis, err := torrent.NewFileItemStore(filepath.Join(conf.TorrentClient.MetadataFolder, "items"), 2*time.Hour) + fis, err := storage.NewFileItemStore(filepath.Join(conf.TorrentClient.MetadataFolder, "items"), 2*time.Hour) if err != nil { return fmt.Errorf("error starting item store: %w", err) } defer fis.Close() - id, err := torrent.GetOrCreatePeerID(filepath.Join(conf.TorrentClient.MetadataFolder, "ID")) + id, err := storage.GetOrCreatePeerID(filepath.Join(conf.TorrentClient.MetadataFolder, "ID")) if err != nil { return fmt.Errorf("error creating node ID: %w", err) } - st, _, err := torrent.SetupStorage(conf.TorrentClient) + st, _, err := storage.SetupStorage(conf.TorrentClient) if err != nil { return err } defer st.Close() - c, err := torrent.NewClient(st, fis, &conf.TorrentClient, id) + rep, err := storage.NewTorrentMetaRepository(conf.TorrentClient.MetadataFolder, st) + if err != nil { + return err + } + + c, err := storage.NewClient(st, fis, &conf.TorrentClient, id) if err != nil { return fmt.Errorf("error starting torrent client: %w", err) } c.AddDhtNodes(conf.TorrentClient.DHTNodes) defer c.Close() - ts := torrent.NewService(c, rep, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout) + ts := service.NewService(c, rep, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout) if err := os.MkdirAll(conf.DataFolder, 0744); err != nil { return fmt.Errorf("error creating data folder: %w", err) diff --git a/go.mod b/go.mod index 5ad4fc0..77d8869 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/billziss-gh/cgofuse v1.5.0 github.com/bodgit/sevenzip v1.4.5 github.com/dgraph-io/badger/v4 v4.2.0 - github.com/edsrzf/mmap-go v1.1.0 github.com/gin-contrib/pprof v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/go-git/go-billy/v5 v5.5.0 @@ -24,20 +23,20 @@ require ( github.com/nwaples/rardecode/v2 v2.0.0-beta.2 github.com/philippgille/gokv v0.6.0 github.com/philippgille/gokv/badgerdb v0.6.0 - github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 + github.com/philippgille/gokv/encoding v0.6.0 github.com/rs/zerolog v1.31.0 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c github.com/stretchr/testify v1.8.4 - github.com/urfave/cli/v2 v2.26.0 + github.com/urfave/cli/v2 v2.27.0 github.com/willscott/go-nfs v0.0.1 - golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 + golang.org/x/exp v0.0.0-20231226003508-02704c960a9b golang.org/x/net v0.19.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect - github.com/RoaringBitmap/roaring v1.6.0 // indirect + github.com/RoaringBitmap/roaring v1.7.0 // indirect github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect github.com/alecthomas/atomic v0.1.0-alpha2 // indirect github.com/anacrolix/chansync v0.3.0 // indirect @@ -54,7 +53,7 @@ require ( github.com/andybalholm/brotli v1.0.6 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/benbjohnson/immutable v0.4.3 // indirect - github.com/bits-and-blooms/bitset v1.12.0 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/windows v1.0.1 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect @@ -64,17 +63,18 @@ require ( github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgraph-io/badger v1.6.0 // indirect + github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect - github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect - github.com/go-llsqlite/crawshaw v0.4.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-llsqlite/adapter v0.1.0 // indirect + github.com/go-llsqlite/crawshaw v0.5.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -106,7 +106,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect - github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 // indirect + github.com/philippgille/gokv/util v0.6.0 // indirect github.com/pierrec/lz4/v4 v4.1.19 // indirect github.com/pion/datachannel v1.5.5 // indirect github.com/pion/dtls/v2 v2.2.8 // indirect @@ -144,16 +144,16 @@ require ( go.opentelemetry.io/otel/trace v1.21.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/arch v0.6.0 // indirect - golang.org/x/crypto v0.16.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - modernc.org/libc v1.22.3 // indirect - modernc.org/mathutil v1.5.0 // indirect - modernc.org/memory v1.5.0 // indirect - modernc.org/sqlite v1.21.1 // indirect - zombiezen.com/go/sqlite v0.13.1 // indirect + modernc.org/libc v1.38.0 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.2 // indirect + modernc.org/sqlite v1.28.0 // indirect + zombiezen.com/go/sqlite v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index fb686fc..c83405c 100644 --- a/go.sum +++ b/go.sum @@ -24,11 +24,12 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/RoaringBitmap/roaring v1.6.0 h1:dc7kRiroETgJcHhWX6BerXkZz2b3JgLGg9nTURJL/og= -github.com/RoaringBitmap/roaring v1.6.0/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= +github.com/RoaringBitmap/roaring v1.7.0 h1:OZF303tJCER1Tj3x+aArx/S5X7hrT186ri6JjrGvG68= +github.com/RoaringBitmap/roaring v1.7.0/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 h1:byYvvbfSo3+9efR4IeReh77gVs4PnNDR3AMOE9NJ7a0= @@ -112,9 +113,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/billziss-gh/cgofuse v1.5.0 h1:kH516I/s+Ab4diL/Y/ayFeUjjA8ey+JK12xDfBf4HEs= github.com/billziss-gh/cgofuse v1.5.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/sevenzip v1.4.5 h1:HFJQ+nbjppfyf2xbQEJBbmVo+o2kTg1FXV4i7YOx87s= @@ -130,6 +131,7 @@ github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -156,14 +158,17 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -209,15 +214,15 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 h1:OyQmpAN302wAopDgwVjgs2HkFawP9ahIEqkUYz7V7CA= -github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU= -github.com/go-llsqlite/crawshaw v0.4.0 h1:L02s2jZBBJj80xm1VkkdyB/JlQ/Fi0kLbNHfXA8yrec= -github.com/go-llsqlite/crawshaw v0.4.0/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= +github.com/go-llsqlite/adapter v0.1.0 h1:wGSQNsu/rtYeu/lqZNZQMjwUdEF3OW66xTLvsFwJQUw= +github.com/go-llsqlite/adapter v0.1.0/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU= +github.com/go-llsqlite/crawshaw v0.5.0 h1:Olbqkth53vkkh4WvmkYjrtfOBcxXD3rMYBYuk6FNH3E= +github.com/go-llsqlite/crawshaw v0.5.0/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -431,12 +436,14 @@ github.com/philippgille/gokv v0.6.0 h1:fNEx/tSwV73nzlYd3iRYB8F+SEVJNNFzH1gsaT8SK github.com/philippgille/gokv v0.6.0/go.mod h1:tjXRFw9xDHgxLS8WJdfYotKGWp8TWqu4RdXjMDG/XBo= github.com/philippgille/gokv/badgerdb v0.6.0 h1:4Qigf2SpyXLF8KaM5nA5/D/0aD/bZevuAnrW4ZsDsjA= github.com/philippgille/gokv/badgerdb v0.6.0/go.mod h1:3u2avs8gtmCc0R0Bw4jKV8aaDfLb5V9JToSASyhpFGM= -github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 h1:IgQDuUPuEFVf22mBskeCLAtvd5c9XiiJG2UYud6eGHI= github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:SjxSrCoeYrYn85oTtroyG1ePY8aE72nvLQlw8IYwAN8= +github.com/philippgille/gokv/encoding v0.6.0 h1:P1TN+Aulpd6Qd7qcLqgPwoxzOQ42UHBXOovWvFxJRI8= +github.com/philippgille/gokv/encoding v0.6.0/go.mod h1:/yKvq2BKJlKJsH7KMDrhDlEw2Pt3V1nKyFhs4iOqz5U= github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61 h1:4tVyBgfpK0NSqu7tNZTwYfC/pbyWUR2y+O7mxEg5BTQ= github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:EUc+s9ONc1+VOr9NUEd8S0YbGRrQd/gz/p+2tvwt12s= -github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 h1:ril/jI0JgXNjPWwDkvcRxlZ09kgHXV2349xChjbsQ4o= github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:2dBhsJgY/yVIkjY5V3AnDUxUbEPzT6uQ3LvoVT8TR20= +github.com/philippgille/gokv/util v0.6.0 h1:GrTxVENzKBxs8lB3tnaA88mKOuVPT7atZPplxX+PNmo= +github.com/philippgille/gokv/util v0.6.0/go.mod h1:ovoDHZ2Svr7YX972SPPJQRXbhHEy3Gb20HRH/Tr9BiQ= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4= github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -514,7 +521,6 @@ github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4 github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -543,6 +549,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= @@ -581,8 +589,8 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= -github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.27.0 h1:uNs1K8JwTFL84X68j5Fjny6hfANh9nTlJ6dRtZAFAHY= +github.com/urfave/cli/v2 v2.27.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willscott/go-nfs v0.0.1 h1:392gV283iuisKFeV9hkKwTdCRfizP+R9FC+gYg2skj0= @@ -632,8 +640,8 @@ golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -643,8 +651,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -889,8 +897,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -918,18 +926,18 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= -modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= -modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/sqlite v1.21.1 h1:GyDFqNnESLOhwwDRaHGdp2jKLDzpyT/rNLglX3ZkMSU= -modernc.org/sqlite v1.21.1/go.mod h1:XwQ0wZPIh1iKb5mkvCJ3szzbhk+tykC8ZWqTRTgYRwI= +modernc.org/libc v1.38.0 h1:o4Lpk0zNDSdsjfEXnF1FGXWQ9PDi1NOdWcLP5n13FGo= +modernc.org/libc v1.38.0/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ= +modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -zombiezen.com/go/sqlite v0.13.1 h1:qDzxyWWmMtSSEH5qxamqBFmqA2BLSSbtODi3ojaE02o= -zombiezen.com/go/sqlite v0.13.1/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4= +zombiezen.com/go/sqlite v1.0.0 h1:D2EvOZqumJBy+6t+0uNTTXnepUpB/pKG45op/UziI1o= +zombiezen.com/go/sqlite v1.0.0/go.mod h1:Yx7FJ77tr7Ucwi5solhXAxpflyxk/BHNXArZ/JvDm60= diff --git a/src/host/torrent/service.go b/src/host/service/service.go similarity index 87% rename from src/host/torrent/service.go rename to src/host/service/service.go index 2f1761f..43e9caa 100644 --- a/src/host/torrent/service.go +++ b/src/host/service/service.go @@ -1,4 +1,4 @@ -package torrent +package service import ( "context" @@ -6,7 +6,7 @@ import ( "log/slog" "time" - "git.kmsign.ru/royalcat/tstor/src/host/repository" + "git.kmsign.ru/royalcat/tstor/src/host/storage" "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/anacrolix/torrent" "github.com/anacrolix/torrent/metainfo" @@ -15,7 +15,7 @@ import ( type Service struct { c *torrent.Client - rep repository.TorrentsRepository + rep storage.TorrentsRepository // stats *Stats DefaultPriority types.PiecePriority @@ -24,7 +24,7 @@ type Service struct { addTimeout, readTimeout int } -func NewService(c *torrent.Client, rep repository.TorrentsRepository, addTimeout, readTimeout int) *Service { +func NewService(c *torrent.Client, rep storage.TorrentsRepository, addTimeout, readTimeout int) *Service { l := slog.With("component", "torrent-service") return &Service{ log: l, diff --git a/src/host/torrent/stats.go b/src/host/service/stats.go similarity index 99% rename from src/host/torrent/stats.go rename to src/host/service/stats.go index 9b6f956..85c53dd 100644 --- a/src/host/torrent/stats.go +++ b/src/host/service/stats.go @@ -1,4 +1,4 @@ -package torrent +package service import ( "errors" diff --git a/src/host/storage.go b/src/host/storage.go index c080d72..bc5bfb0 100644 --- a/src/host/storage.go +++ b/src/host/storage.go @@ -1,11 +1,11 @@ package host import ( - "git.kmsign.ru/royalcat/tstor/src/host/torrent" + "git.kmsign.ru/royalcat/tstor/src/host/service" "git.kmsign.ru/royalcat/tstor/src/host/vfs" ) -func NewStorage(dataPath string, tsrv *torrent.Service) vfs.Filesystem { +func NewStorage(dataPath string, tsrv *service.Service) vfs.Filesystem { factories := map[string]vfs.FsFactory{ ".torrent": tsrv.NewTorrentFs, } diff --git a/src/host/torrent/client.go b/src/host/storage/client.go similarity index 64% rename from src/host/torrent/client.go rename to src/host/storage/client.go index d432883..bacdc15 100644 --- a/src/host/torrent/client.go +++ b/src/host/storage/client.go @@ -1,4 +1,4 @@ -package torrent +package storage import ( "time" @@ -20,20 +20,6 @@ func NewClient(st storage.ClientImpl, fis bep44.Store, cfg *config.TorrentClient torrentCfg.PeerID = string(id[:]) torrentCfg.DefaultStorage = st - // torrentCfg.DisableIPv6 = cfg.DisableIPv6 - // torrentCfg.DropDuplicatePeerIds = true - // torrentCfg.TorrentPeersLowWater = 10 - // torrentCfg.TorrentPeersHighWater = 100 - // torrentCfg.DisableWebtorrent = true - // torrentCfg.DisableAggressiveUpload = true - // torrentCfg.DisableWebseeds = true - // torrentCfg.DisableUTP = false - // torrentCfg.NoDefaultPortForwarding = true - // torrentCfg.AlwaysWantConns = false - // torrentCfg.ClientDhtConfig = torrent.ClientDhtConfig{ - // NoDHT: true, - // } - l := log.Logger.With().Str("component", "torrent-client").Logger() tl := tlog.NewLogger() diff --git a/src/host/torrent/id.go b/src/host/storage/id.go similarity index 96% rename from src/host/torrent/id.go rename to src/host/storage/id.go index e18471c..9422893 100644 --- a/src/host/torrent/id.go +++ b/src/host/storage/id.go @@ -1,4 +1,4 @@ -package torrent +package storage import ( "crypto/rand" diff --git a/src/host/torrent/piece-completion.go b/src/host/storage/piece-completion.go similarity index 99% rename from src/host/torrent/piece-completion.go rename to src/host/storage/piece-completion.go index 4b7eeec..b98ebaa 100644 --- a/src/host/torrent/piece-completion.go +++ b/src/host/storage/piece-completion.go @@ -1,4 +1,4 @@ -package torrent +package storage import ( "encoding/binary" diff --git a/src/host/repository/repository.go b/src/host/storage/repository.go similarity index 73% rename from src/host/repository/repository.go rename to src/host/storage/repository.go index e17ed86..b988485 100644 --- a/src/host/repository/repository.go +++ b/src/host/storage/repository.go @@ -1,10 +1,11 @@ -package repository +package storage import ( "errors" "path/filepath" "sync" + "github.com/anacrolix/torrent" "github.com/anacrolix/torrent/metainfo" "github.com/philippgille/gokv" "github.com/philippgille/gokv/badgerdb" @@ -12,13 +13,13 @@ import ( ) type TorrentsRepository interface { - ExcludeFile(hash metainfo.Hash, file ...string) error + ExcludeFile(file *torrent.File) error ExcludedFiles(hash metainfo.Hash) ([]string, error) } -func NewTorrentMetaRepository(dir string) (TorrentsRepository, error) { +func NewTorrentMetaRepository(metaDir string, storage *FileStorage) (TorrentsRepository, error) { excludedFilesStore, err := badgerdb.NewStore(badgerdb.Options{ - Dir: filepath.Join(dir, "excluded-files"), + Dir: filepath.Join(metaDir, "excluded-files"), Codec: encoding.JSON, }) @@ -28,6 +29,7 @@ func NewTorrentMetaRepository(dir string) (TorrentsRepository, error) { r := &torrentRepositoryImpl{ excludedFiles: excludedFilesStore, + storage: storage, } return r, nil @@ -36,14 +38,16 @@ func NewTorrentMetaRepository(dir string) (TorrentsRepository, error) { type torrentRepositoryImpl struct { m sync.RWMutex excludedFiles gokv.Store + storage *FileStorage } var ErrNotFound = errors.New("not found") -func (r *torrentRepositoryImpl) ExcludeFile(hash metainfo.Hash, file ...string) error { +func (r *torrentRepositoryImpl) ExcludeFile(file *torrent.File) error { r.m.Lock() defer r.m.Unlock() + hash := file.Torrent().InfoHash() var excludedFiles []string found, err := r.excludedFiles.Get(hash.AsString(), &excludedFiles) if err != nil { @@ -52,7 +56,12 @@ func (r *torrentRepositoryImpl) ExcludeFile(hash metainfo.Hash, file ...string) if !found { excludedFiles = []string{} } - excludedFiles = unique(append(excludedFiles, file...)) + excludedFiles = unique(append(excludedFiles, file.Path())) + + err = r.storage.DeleteFile(file) + if err != nil { + return err + } return r.excludedFiles.Set(hash.AsString(), excludedFiles) } diff --git a/src/host/storage/storage.go b/src/host/storage/storage.go new file mode 100644 index 0000000..b1719ac --- /dev/null +++ b/src/host/storage/storage.go @@ -0,0 +1,51 @@ +package storage + +import ( + "fmt" + "os" + "path/filepath" + + "git.kmsign.ru/royalcat/tstor/src/config" + "github.com/anacrolix/torrent/storage" +) + +func SetupStorage(cfg config.TorrentClient) (*FileStorage, storage.PieceCompletion, error) { + pcp := filepath.Join(cfg.DataFolder, "piece-completion") + if err := os.MkdirAll(pcp, 0744); err != nil { + return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err) + } + pc, err := storage.NewBoltPieceCompletion(pcp) + if err != nil { + return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err) + } + + // pc, err := NewBadgerPieceCompletion(pcp) + // if err != nil { + // return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err) + // } + + // TODO implement cache/storage switching + // cacheDir := filepath.Join(tcfg.DataFolder, "cache") + // if err := os.MkdirAll(cacheDir, 0744); err != nil { + // return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err) + // } + // fc, err := filecache.NewCache(cacheDir) + // if err != nil { + // return nil, nil, fmt.Errorf("error creating cache: %w", err) + // } + // log.Info().Msg(fmt.Sprintf("setting cache size to %d MB", 1024)) + // fc.SetCapacity(1024 * 1024 * 1024) + + // rp := storage.NewResourcePieces(fc.AsResourceProvider()) + // st := &stc{rp} + + filesDir := filepath.Join(cfg.DataFolder, "files") + if err := os.MkdirAll(pcp, 0744); err != nil { + return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err) + } + + // st := storage.NewMMapWithCompletion(filesDir, pc) + st := NewFileStorage(filesDir, pc) + + return st, pc, nil +} diff --git a/src/host/storage/storage_files.go b/src/host/storage/storage_files.go new file mode 100644 index 0000000..e6d109f --- /dev/null +++ b/src/host/storage/storage_files.go @@ -0,0 +1,302 @@ +package storage + +import ( + "fmt" + "io" + "log" + "os" + "path" + "path/filepath" + "strings" + + "github.com/anacrolix/missinggo" + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/common" + "github.com/anacrolix/torrent/metainfo" + "github.com/anacrolix/torrent/segments" + "github.com/anacrolix/torrent/storage" +) + +// NewFileStorage creates a new ClientImplCloser that stores files using the OS native filesystem. +func NewFileStorage(baseDir string, pc storage.PieceCompletion) *FileStorage { + return &FileStorage{baseDir: baseDir, pieceCompletion: pc} +} + +// File-based storage for torrents, that isn't yet bound to a particular torrent. +type FileStorage struct { + baseDir string + pieceCompletion storage.PieceCompletion +} + +func (me *FileStorage) Close() error { + return me.pieceCompletion.Close() +} + +func (me *FileStorage) torrentDir(info *metainfo.Info, infoHash metainfo.Hash) string { + return filepath.Join(me.baseDir, info.Name) +} + +func (me *FileStorage) filePath(file metainfo.FileInfo) string { + return filepath.Join(file.Path...) +} + +func (fs *FileStorage) DeleteFile(file *torrent.File) error { + info := file.Torrent().Info() + infoHash := file.Torrent().InfoHash() + torrentDir := fs.torrentDir(info, infoHash) + relFilePath := fs.filePath(file.FileInfo()) + filePath := path.Join(torrentDir, relFilePath) + return os.Remove(filePath) +} + +func (fs FileStorage) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (storage.TorrentImpl, error) { + dir := fs.torrentDir(info, infoHash) + upvertedFiles := info.UpvertedFiles() + files := make([]file, 0, len(upvertedFiles)) + for i, fileInfo := range upvertedFiles { + filePath := filepath.Join(dir, fs.filePath(fileInfo)) + if !isSubFilepath(dir, filePath) { + return storage.TorrentImpl{}, fmt.Errorf("file %v: path %q is not sub path of %q", i, filePath, fs.baseDir) + } + + f := file{ + path: filePath, + length: fileInfo.Length, + } + if f.length == 0 { + err := CreateNativeZeroLengthFile(f.path) + if err != nil { + return storage.TorrentImpl{}, fmt.Errorf("creating zero length file: %w", err) + } + } + files = append(files, f) + } + t := &fileTorrentImpl{ + files: files, + segmentLocater: segments.NewIndex(common.LengthIterFromUpvertedFiles(upvertedFiles)), + infoHash: infoHash, + completion: fs.pieceCompletion, + } + return storage.TorrentImpl{ + Piece: t.Piece, + Close: t.Close, + }, nil +} + +type file struct { + // The safe, OS-local file path. + path string + length int64 +} + +type fileTorrentImpl struct { + files []file + segmentLocater segments.Index + infoHash metainfo.Hash + completion storage.PieceCompletion +} + +func (fts *fileTorrentImpl) Piece(p metainfo.Piece) storage.PieceImpl { + // Create a view onto the file-based torrent storage. + _io := fileTorrentImplIO{fts} + // Return the appropriate segments of this. + return &filePieceImpl{ + fileTorrentImpl: fts, + p: p, + WriterAt: missinggo.NewSectionWriter(_io, p.Offset(), p.Length()), + ReaderAt: io.NewSectionReader(_io, p.Offset(), p.Length()), + } +} + +func (fs *fileTorrentImpl) Close() error { + return nil +} + +// A helper to create zero-length files which won't appear for file-orientated storage since no +// writes will ever occur to them (no torrent data is associated with a zero-length file). The +// caller should make sure the file name provided is safe/sanitized. +func CreateNativeZeroLengthFile(name string) error { + err := os.MkdirAll(filepath.Dir(name), 0o777) + if err != nil { + return err + } + f, err := os.Create(name) + if err != nil { + return err + } + return f.Close() +} + +// Exposes file-based storage of a torrent, as one big ReadWriterAt. +type fileTorrentImplIO struct { + fts *fileTorrentImpl +} + +// Returns EOF on short or missing file. +func (fst *fileTorrentImplIO) readFileAt(file file, b []byte, off int64) (n int, err error) { + f, err := os.Open(file.path) + if os.IsNotExist(err) { + // File missing is treated the same as a short file. + err = io.EOF + return + } + if err != nil { + return + } + defer f.Close() + // Limit the read to within the expected bounds of this file. + if int64(len(b)) > file.length-off { + b = b[:file.length-off] + } + for off < file.length && len(b) != 0 { + n1, err1 := f.ReadAt(b, off) + b = b[n1:] + n += n1 + off += int64(n1) + if n1 == 0 { + err = err1 + break + } + } + return +} + +// Only returns EOF at the end of the torrent. Premature EOF is ErrUnexpectedEOF. +func (fst fileTorrentImplIO) ReadAt(b []byte, off int64) (n int, err error) { + fst.fts.segmentLocater.Locate( + segments.Extent{Start: off, Length: int64(len(b))}, + func(i int, e segments.Extent) bool { + n1, err1 := fst.readFileAt(fst.fts.files[i], b[:e.Length], e.Start) + n += n1 + b = b[n1:] + err = err1 + return err == nil // && int64(n1) == e.Length + }, + ) + if len(b) != 0 && err == nil { + err = io.EOF + } + return +} + +func (fst fileTorrentImplIO) WriteAt(p []byte, off int64) (n int, err error) { + // log.Printf("write at %v: %v bytes", off, len(p)) + fst.fts.segmentLocater.Locate( + segments.Extent{Start: off, Length: int64(len(p))}, + func(i int, e segments.Extent) bool { + name := fst.fts.files[i].path + err = os.MkdirAll(filepath.Dir(name), 0o777) + if err != nil { + return false + } + var f *os.File + f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0o666) + if err != nil { + return false + } + var n1 int + n1, err = f.WriteAt(p[:e.Length], e.Start) + // log.Printf("%v %v wrote %v: %v", i, e, n1, err) + closeErr := f.Close() + n += n1 + p = p[n1:] + if err == nil { + err = closeErr + } + if err == nil && int64(n1) != e.Length { + err = io.ErrShortWrite + } + return err == nil + }, + ) + return n, err +} + +type filePieceImpl struct { + *fileTorrentImpl + p metainfo.Piece + io.WriterAt + io.ReaderAt +} + +var _ storage.PieceImpl = (*filePieceImpl)(nil) + +func (me *filePieceImpl) pieceKey() metainfo.PieceKey { + return metainfo.PieceKey{InfoHash: me.infoHash, Index: me.p.Index()} +} + +func (fs *filePieceImpl) Completion() storage.Completion { + c, err := fs.completion.Get(fs.pieceKey()) + if err != nil { + log.Printf("error getting piece completion: %s", err) + c.Ok = false + return c + } + + verified := true + if c.Complete { + // If it's allegedly complete, check that its constituent files have the necessary length. + for _, fi := range extentCompleteRequiredLengths(fs.p.Info, fs.p.Offset(), fs.p.Length()) { + s, err := os.Stat(fs.files[fi.fileIndex].path) + if err != nil || s.Size() < fi.length { + verified = false + break + } + } + } + + if !verified { + // The completion was wrong, fix it. + c.Complete = false + fs.completion.Set(fs.pieceKey(), false) + } + + return c +} + +func (fs *filePieceImpl) MarkComplete() error { + return fs.completion.Set(fs.pieceKey(), true) +} + +func (fs *filePieceImpl) MarkNotComplete() error { + return fs.completion.Set(fs.pieceKey(), false) +} + +type requiredLength struct { + fileIndex int + length int64 +} + +func isSubFilepath(base, sub string) bool { + rel, err := filepath.Rel(base, sub) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) +} + +func extentCompleteRequiredLengths(info *metainfo.Info, off, n int64) (ret []requiredLength) { + if n == 0 { + return + } + for i, fi := range info.UpvertedFiles() { + if off >= fi.Length { + off -= fi.Length + continue + } + n1 := n + if off+n1 > fi.Length { + n1 = fi.Length - off + } + ret = append(ret, requiredLength{ + fileIndex: i, + length: off + n1, + }) + n -= n1 + if n == 0 { + return + } + off = 0 + } + panic("extent exceeds torrent bounds") +} diff --git a/src/host/torrent/store.go b/src/host/storage/store.go similarity index 99% rename from src/host/torrent/store.go rename to src/host/storage/store.go index f762671..0f9adf2 100644 --- a/src/host/torrent/store.go +++ b/src/host/storage/store.go @@ -1,4 +1,4 @@ -package torrent +package storage import ( "bytes" diff --git a/src/host/torrent/storage.go b/src/host/torrent/storage.go deleted file mode 100644 index 24f694a..0000000 --- a/src/host/torrent/storage.go +++ /dev/null @@ -1,306 +0,0 @@ -package torrent - -import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - - "git.kmsign.ru/royalcat/tstor/src/config" - "github.com/anacrolix/missinggo" - "github.com/anacrolix/torrent" - "github.com/anacrolix/torrent/metainfo" - "github.com/anacrolix/torrent/mmap_span" - "github.com/anacrolix/torrent/storage" - "github.com/edsrzf/mmap-go" -) - -type Torrent struct { - client *torrent.Client - data storage.ClientImplCloser - pc storage.PieceCompletion -} - -func SetupStorage(cfg config.TorrentClient) (storage.ClientImplCloser, storage.PieceCompletion, error) { - pcp := filepath.Join(cfg.DataFolder, "piece-completion") - if err := os.MkdirAll(pcp, 0744); err != nil { - return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err) - } - pc, err := storage.NewBoltPieceCompletion(pcp) - if err != nil { - return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err) - } - - // pc, err := NewBadgerPieceCompletion(pcp) - // if err != nil { - // return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err) - // } - - // TODO implement cache/storage switching - // cacheDir := filepath.Join(tcfg.DataFolder, "cache") - // if err := os.MkdirAll(cacheDir, 0744); err != nil { - // return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err) - // } - // fc, err := filecache.NewCache(cacheDir) - // if err != nil { - // return nil, nil, fmt.Errorf("error creating cache: %w", err) - // } - // log.Info().Msg(fmt.Sprintf("setting cache size to %d MB", 1024)) - // fc.SetCapacity(1024 * 1024 * 1024) - - // rp := storage.NewResourcePieces(fc.AsResourceProvider()) - // st := &stc{rp} - - filesDir := filepath.Join(cfg.DataFolder, "files") - if err := os.MkdirAll(pcp, 0744); err != nil { - return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err) - } - - // st := storage.NewMMapWithCompletion(filesDir, pc) - st := storage.NewFileOpts(storage.NewFileClientOpts{ - ClientBaseDir: filesDir, - PieceCompletion: pc, - }) - - return st, pc, nil -} - -func (s Torrent) Remove(f *torrent.File) error { - - return nil -} - -// type dupePieces struct { -// } - -// func (s Torrent) dedupe(f1, f2 *os.File) error { -// for _, t := range s.client.Torrents() { -// for i := 0; i < t.NumPieces(); i++ { -// p := t.Piece(i) -// p.Info().Hash() -// } -// } - -// // https://go-review.googlesource.com/c/sys/+/284352/10/unix/syscall_linux_test.go#856 -// // dedupe := unix.FileDedupeRange{ -// // Src_offset: uint64(0), -// // Src_length: uint64(4096), -// // Info: []unix.FileDedupeRangeInfo{ -// // unix.FileDedupeRangeInfo{ -// // Dest_fd: int64(f2.Fd()), -// // Dest_offset: uint64(0), -// // }, -// // unix.FileDedupeRangeInfo{ -// // Dest_fd: int64(f2.Fd()), -// // Dest_offset: uint64(4096), -// // }, -// // }} -// // err := unix.IoctlFileDedupeRange(int(f1.Fd()), &dedupe) -// // if err == unix.EOPNOTSUPP || err == unix.EINVAL { -// // t.Skip("deduplication not supported on this filesystem") -// // } else if err != nil { -// // t.Fatal(err) -// // } - -// return nil -// } - -type mmapClientImpl struct { - baseDir string - pc storage.PieceCompletion -} - -func NewMMapWithCompletion(baseDir string, completion storage.PieceCompletion) *mmapClientImpl { - return &mmapClientImpl{ - baseDir: baseDir, - pc: completion, - } -} - -func (s *mmapClientImpl) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (_ storage.TorrentImpl, err error) { - t, err := newMMapTorrent(info, infoHash, s.baseDir, s.pc) - if err != nil { - return storage.TorrentImpl{}, err - } - return storage.TorrentImpl{Piece: t.Piece, Close: t.Close, Flush: t.Flush}, nil -} - -func (s *mmapClientImpl) Close() error { - return s.pc.Close() -} - -func newMMapTorrent(md *metainfo.Info, infoHash metainfo.Hash, location string, pc storage.PieceCompletionGetSetter) (*mmapTorrent, error) { - span := &mmap_span.MMapSpan{} - basePath, err := storage.ToSafeFilePath(md.Name) - if err != nil { - return nil, err - } - basePath = filepath.Join(location, basePath) - - for _, miFile := range md.UpvertedFiles() { - var safeName string - safeName, err = storage.ToSafeFilePath(miFile.Path...) - if err != nil { - return nil, err - } - fileName := filepath.Join(basePath, safeName) - var mm FileMapping - mm, err = mmapFile(fileName, miFile.Length) - if err != nil { - err = fmt.Errorf("file %q: %s", miFile.DisplayPath(md), err) - return nil, err - } - span.Append(mm) - } - span.InitIndex() - - return &mmapTorrent{ - infoHash: infoHash, - span: span, - pc: pc, - }, nil -} - -type mmapTorrent struct { - infoHash metainfo.Hash - span *mmap_span.MMapSpan - pc storage.PieceCompletionGetSetter -} - -func (ts *mmapTorrent) Piece(p metainfo.Piece) storage.PieceImpl { - return mmapPiece{ - pc: ts.pc, - p: p, - ih: ts.infoHash, - ReaderAt: io.NewSectionReader(ts.span, p.Offset(), p.Length()), - WriterAt: missinggo.NewSectionWriter(ts.span, p.Offset(), p.Length()), - } -} - -func (ts *mmapTorrent) Close() error { - errs := ts.span.Close() - if len(errs) > 0 { - return errs[0] - } - return nil -} - -func (ts *mmapTorrent) Flush() error { - errs := ts.span.Flush() - if len(errs) > 0 { - return errs[0] - } - return nil -} - -type mmapPiece struct { - pc storage.PieceCompletionGetSetter - p metainfo.Piece - ih metainfo.Hash - io.ReaderAt - io.WriterAt -} - -func (me mmapPiece) pieceKey() metainfo.PieceKey { - return metainfo.PieceKey{InfoHash: me.ih, Index: me.p.Index()} -} - -func (sp mmapPiece) Completion() storage.Completion { - c, err := sp.pc.Get(sp.pieceKey()) - if err != nil { - panic(err) - } - return c -} - -func (sp mmapPiece) MarkComplete() error { - return sp.pc.Set(sp.pieceKey(), true) -} - -func (sp mmapPiece) MarkNotComplete() error { - return sp.pc.Set(sp.pieceKey(), false) -} - -func mmapFile(name string, size int64) (_ FileMapping, err error) { - dir := filepath.Dir(name) - err = os.MkdirAll(dir, 0o750) - if err != nil { - return nil, fmt.Errorf("making directory %q: %s", dir, err) - } - var file *os.File - file, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0o666) - if err != nil { - return nil, err - } - defer func() { - if err != nil { - file.Close() - } - }() - var fi os.FileInfo - fi, err = file.Stat() - if err != nil { - return nil, err - } - if fi.Size() < size { - // I think this is necessary on HFS+. Maybe Linux will SIGBUS too if - // you overmap a file but I'm not sure. - err = file.Truncate(size) - if err != nil { - return nil, err - } - } - return func() (ret mmapWithFile, err error) { - ret.f = file - if size == 0 { - // Can't mmap() regions with length 0. - return - } - intLen := int(size) - if int64(intLen) != size { - err = errors.New("size too large for system") - return - } - ret.mmap, err = mmap.MapRegion(file, intLen, mmap.RDWR, 0, 0) - if err != nil { - err = fmt.Errorf("error mapping region: %s", err) - return - } - if int64(len(ret.mmap)) != size { - panic(len(ret.mmap)) - } - return - }() -} - -type FileMapping = mmap_span.Mmap - -// Handles closing the mmap's file handle (needed for Windows). Could be implemented differently by -// OS. -type mmapWithFile struct { - f *os.File - mmap mmap.MMap -} - -func (m mmapWithFile) Flush() error { - return m.mmap.Flush() -} - -func (m mmapWithFile) Unmap() (err error) { - if m.mmap != nil { - err = m.mmap.Unmap() - } - fileErr := m.f.Close() - if err == nil { - err = fileErr - } - return -} - -func (m mmapWithFile) Bytes() []byte { - if m.mmap == nil { - return nil - } - return m.mmap -} diff --git a/src/host/vfs/torrent.go b/src/host/vfs/torrent.go index 88f8b4f..ccf38ef 100644 --- a/src/host/vfs/torrent.go +++ b/src/host/vfs/torrent.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "git.kmsign.ru/royalcat/tstor/src/host/repository" + "git.kmsign.ru/royalcat/tstor/src/host/storage" "git.kmsign.ru/royalcat/tstor/src/iio" "github.com/anacrolix/missinggo/v2" "github.com/anacrolix/torrent" @@ -21,7 +21,7 @@ var _ Filesystem = &TorrentFs{} type TorrentFs struct { mu sync.Mutex t *torrent.Torrent - rep repository.TorrentsRepository + rep storage.TorrentsRepository readTimeout int @@ -31,7 +31,7 @@ type TorrentFs struct { resolver *resolver } -func NewTorrentFs(t *torrent.Torrent, rep repository.TorrentsRepository, readTimeout int) *TorrentFs { +func NewTorrentFs(t *torrent.Torrent, rep storage.TorrentsRepository, readTimeout int) *TorrentFs { return &TorrentFs{ t: t, rep: rep, @@ -53,17 +53,17 @@ func (fs *TorrentFs) files() (map[string]*torrentFile, error) { fs.filesCache = make(map[string]*torrentFile) for _, file := range files { - p := AbsPath(file.Path()) - if slices.Contains(excludedFiles, p) { + if slices.Contains(excludedFiles, file.Path()) { continue } + p := AbsPath(file.Path()) + fs.filesCache[p] = &torrentFile{ - name: path.Base(p), - readerFunc: file.NewReader, - len: file.Length(), - timeout: fs.readTimeout, + name: path.Base(p), + timeout: fs.readTimeout, + file: file, } } fs.mu.Unlock() @@ -144,6 +144,8 @@ func (fs *TorrentFs) ReadDir(name string) ([]fs.DirEntry, error) { } func (fs *TorrentFs) Unlink(name string) error { + name = AbsPath(name) + fs.mu.Lock() defer fs.mu.Unlock() @@ -151,14 +153,15 @@ func (fs *TorrentFs) Unlink(name string) error { if err != nil { return err } - file := AbsPath(name) - if !slices.Contains(maps.Keys(files), file) { + if !slices.Contains(maps.Keys(files), name) { return ErrNotExist } - fs.filesCache = nil - return fs.rep.ExcludeFile(fs.t.InfoHash(), file) + file := files[name] + delete(fs.filesCache, name) + + return fs.rep.ExcludeFile(file.file) } type reader interface { @@ -224,25 +227,25 @@ var _ File = &torrentFile{} type torrentFile struct { name string - readerFunc func() torrent.Reader - reader reader - len int64 - timeout int + reader reader + timeout int + + file *torrent.File } func (d *torrentFile) Stat() (fs.FileInfo, error) { - return newFileInfo(d.name, d.len), nil + return newFileInfo(d.name, d.file.Length()), nil } func (d *torrentFile) load() { if d.reader != nil { return } - d.reader = newReadAtWrapper(d.readerFunc(), d.timeout) + d.reader = newReadAtWrapper(d.file.NewReader(), d.timeout) } func (d *torrentFile) Size() int64 { - return d.len + return d.file.Length() } func (d *torrentFile) IsDir() bool { diff --git a/src/host/vfs/torrent_test.go b/src/host/vfs/torrent_test.go index bb9fdb0..2a8fac9 100644 --- a/src/host/vfs/torrent_test.go +++ b/src/host/vfs/torrent_test.go @@ -96,9 +96,8 @@ func TestReadAtTorrent(t *testing.T) { torrFile := to.Files()[0] tf := torrentFile{ - readerFunc: torrFile.NewReader, - len: torrFile.Length(), - timeout: 500, + file: torrFile, + timeout: 500, } defer tf.Close() diff --git a/src/http/api.go b/src/http/api.go index c227c2e..9a232e8 100644 --- a/src/http/api.go +++ b/src/http/api.go @@ -7,12 +7,12 @@ import ( "net/http" "os" - "git.kmsign.ru/royalcat/tstor/src/host/torrent" + "git.kmsign.ru/royalcat/tstor/src/host/service" "github.com/anacrolix/missinggo/v2/filecache" "github.com/gin-gonic/gin" ) -var apiStatusHandler = func(fc *filecache.Cache, ss *torrent.Stats) gin.HandlerFunc { +var apiStatusHandler = func(fc *filecache.Cache, ss *service.Stats) gin.HandlerFunc { return func(ctx *gin.Context) { stat := gin.H{ "torrentStats": ss.GlobalStats(), diff --git a/src/http/http.go b/src/http/http.go index 55a6590..d10064a 100644 --- a/src/http/http.go +++ b/src/http/http.go @@ -6,7 +6,7 @@ import ( "git.kmsign.ru/royalcat/tstor" "git.kmsign.ru/royalcat/tstor/src/config" - "git.kmsign.ru/royalcat/tstor/src/host/torrent" + "git.kmsign.ru/royalcat/tstor/src/host/service" "github.com/anacrolix/missinggo/v2/filecache" "github.com/gin-contrib/pprof" "github.com/gin-gonic/gin" @@ -14,7 +14,7 @@ import ( "github.com/shurcooL/httpfs/html/vfstemplate" ) -func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, logPath string, cfg *config.Config) error { +func New(fc *filecache.Cache, ss *service.Stats, s *service.Service, logPath string, cfg *config.Config) error { gin.SetMode(gin.ReleaseMode) r := gin.New() r.Use(gin.Recovery()) diff --git a/src/iio/disk_test.go b/src/iio/disk_test.go index c540d02..5df8814 100644 --- a/src/iio/disk_test.go +++ b/src/iio/disk_test.go @@ -26,7 +26,7 @@ func TestReadData(t *testing.T) { require.Equal(5, n) require.Equal("World", string(toRead)) - r.ReadAt(toRead, 0) + n, err = r.ReadAt(toRead, 0) require.NoError(err) require.Equal(5, n) require.Equal("Hello", string(toRead))