royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

file delete on exclude

Browse source
This commit is contained in:
royalcat 2024-01-01 01:54:55 +03:00
parent 5f8d497de1
commit 49444bd70d
19 changed files with 481 additions and 429 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
cmd/tstor/main.go
View file

@ -14,8 +14,8 @@ import (
"git.kmsign.ru/royalcat/tstor/src/config" "git.kmsign.ru/royalcat/tstor/src/config"
"git.kmsign.ru/royalcat/tstor/src/host" "git.kmsign.ru/royalcat/tstor/src/host"
"git.kmsign.ru/royalcat/tstor/src/host/repository" "git.kmsign.ru/royalcat/tstor/src/host/service"
"git.kmsign.ru/royalcat/tstor/src/host/torrent" "git.kmsign.ru/royalcat/tstor/src/host/storage"
"git.kmsign.ru/royalcat/tstor/src/host/vfs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
@ -73,40 +73,40 @@ func run(configPath string) error {
log.Err(err).Msg("set priority failed") log.Err(err).Msg("set priority failed")
} }
rep, err := repository.NewTorrentMetaRepository(conf.TorrentClient.MetadataFolder)
if err != nil {
return err
}
if err := os.MkdirAll(conf.TorrentClient.MetadataFolder, 0744); err != nil { if err := os.MkdirAll(conf.TorrentClient.MetadataFolder, 0744); err != nil {
return fmt.Errorf("error creating metadata folder: %w", err) return fmt.Errorf("error creating metadata folder: %w", err)
} }
fis, err := torrent.NewFileItemStore(filepath.Join(conf.TorrentClient.MetadataFolder, "items"), 2*time.Hour) fis, err := storage.NewFileItemStore(filepath.Join(conf.TorrentClient.MetadataFolder, "items"), 2*time.Hour)
if err != nil { if err != nil {
return fmt.Errorf("error starting item store: %w", err) return fmt.Errorf("error starting item store: %w", err)
} }
defer fis.Close() defer fis.Close()
id, err := torrent.GetOrCreatePeerID(filepath.Join(conf.TorrentClient.MetadataFolder, "ID")) id, err := storage.GetOrCreatePeerID(filepath.Join(conf.TorrentClient.MetadataFolder, "ID"))
if err != nil { if err != nil {
return fmt.Errorf("error creating node ID: %w", err) return fmt.Errorf("error creating node ID: %w", err)
} }
st, _, err := torrent.SetupStorage(conf.TorrentClient) st, _, err := storage.SetupStorage(conf.TorrentClient)
if err != nil { if err != nil {
return err return err
} }
defer st.Close() defer st.Close()
c, err := torrent.NewClient(st, fis, &conf.TorrentClient, id) rep, err := storage.NewTorrentMetaRepository(conf.TorrentClient.MetadataFolder, st)
if err != nil {
return err
}
c, err := storage.NewClient(st, fis, &conf.TorrentClient, id)
if err != nil { if err != nil {
return fmt.Errorf("error starting torrent client: %w", err) return fmt.Errorf("error starting torrent client: %w", err)
} }
c.AddDhtNodes(conf.TorrentClient.DHTNodes) c.AddDhtNodes(conf.TorrentClient.DHTNodes)
defer c.Close() defer c.Close()
ts := torrent.NewService(c, rep, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout) ts := service.NewService(c, rep, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout)
if err := os.MkdirAll(conf.DataFolder, 0744); err != nil { if err := os.MkdirAll(conf.DataFolder, 0744); err != nil {
return fmt.Errorf("error creating data folder: %w", err) return fmt.Errorf("error creating data folder: %w", err)

38
go.mod
View file

@ -11,7 +11,6 @@ require (
github.com/billziss-gh/cgofuse v1.5.0 github.com/billziss-gh/cgofuse v1.5.0
github.com/bodgit/sevenzip v1.4.5 github.com/bodgit/sevenzip v1.4.5
github.com/dgraph-io/badger/v4 v4.2.0 github.com/dgraph-io/badger/v4 v4.2.0
github.com/edsrzf/mmap-go v1.1.0
github.com/gin-contrib/pprof v1.4.0 github.com/gin-contrib/pprof v1.4.0
github.com/gin-gonic/gin v1.9.1 github.com/gin-gonic/gin v1.9.1
github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-billy/v5 v5.5.0
@ -24,20 +23,20 @@ require (
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 github.com/nwaples/rardecode/v2 v2.0.0-beta.2
github.com/philippgille/gokv v0.6.0 github.com/philippgille/gokv v0.6.0
github.com/philippgille/gokv/badgerdb v0.6.0 github.com/philippgille/gokv/badgerdb v0.6.0
github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 github.com/philippgille/gokv/encoding v0.6.0
github.com/rs/zerolog v1.31.0 github.com/rs/zerolog v1.31.0
github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c
github.com/stretchr/testify v1.8.4 github.com/stretchr/testify v1.8.4
github.com/urfave/cli/v2 v2.26.0 github.com/urfave/cli/v2 v2.27.0
github.com/willscott/go-nfs v0.0.1 github.com/willscott/go-nfs v0.0.1
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 golang.org/x/exp v0.0.0-20231226003508-02704c960a9b
golang.org/x/net v0.19.0 golang.org/x/net v0.19.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1
) )
require ( require (
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
github.com/RoaringBitmap/roaring v1.6.0 // indirect github.com/RoaringBitmap/roaring v1.7.0 // indirect
github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/anacrolix/chansync v0.3.0 // indirect github.com/anacrolix/chansync v0.3.0 // indirect
@ -54,7 +53,7 @@ require (
github.com/andybalholm/brotli v1.0.6 // indirect github.com/andybalholm/brotli v1.0.6 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/benbjohnson/immutable v0.4.3 // indirect github.com/benbjohnson/immutable v0.4.3 // indirect
github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/windows v1.0.1 // indirect github.com/bodgit/windows v1.0.1 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@ -64,17 +63,18 @@ require (
github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgraph-io/badger v1.6.0 // indirect github.com/dgraph-io/badger v1.6.2 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/fatih/structs v1.1.0 // indirect github.com/fatih/structs v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect github.com/go-llsqlite/adapter v0.1.0 // indirect
github.com/go-llsqlite/crawshaw v0.4.0 // indirect github.com/go-llsqlite/crawshaw v0.5.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect
@ -106,7 +106,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mschoch/smat v0.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect
github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 // indirect github.com/philippgille/gokv/util v0.6.0 // indirect
github.com/pierrec/lz4/v4 v4.1.19 // indirect github.com/pierrec/lz4/v4 v4.1.19 // indirect
github.com/pion/datachannel v1.5.5 // indirect github.com/pion/datachannel v1.5.5 // indirect
github.com/pion/dtls/v2 v2.2.8 // indirect github.com/pion/dtls/v2 v2.2.8 // indirect
@ -144,16 +144,16 @@ require (
go.opentelemetry.io/otel/trace v1.21.0 // indirect go.opentelemetry.io/otel/trace v1.21.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/arch v0.6.0 // indirect golang.org/x/arch v0.6.0 // indirect
golang.org/x/crypto v0.16.0 // indirect golang.org/x/crypto v0.17.0 // indirect
golang.org/x/sync v0.5.0 // indirect golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.15.0 // indirect golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect golang.org/x/time v0.5.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect google.golang.org/protobuf v1.32.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.22.3 // indirect modernc.org/libc v1.38.0 // indirect
modernc.org/mathutil v1.5.0 // indirect modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.5.0 // indirect modernc.org/memory v1.7.2 // indirect
modernc.org/sqlite v1.21.1 // indirect modernc.org/sqlite v1.28.0 // indirect
zombiezen.com/go/sqlite v0.13.1 // indirect zombiezen.com/go/sqlite v1.0.0 // indirect
) )

74
go.sum
View file

@ -24,11 +24,12 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI=
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
github.com/RoaringBitmap/roaring v1.6.0 h1:dc7kRiroETgJcHhWX6BerXkZz2b3JgLGg9nTURJL/og= github.com/RoaringBitmap/roaring v1.7.0 h1:OZF303tJCER1Tj3x+aArx/S5X7hrT186ri6JjrGvG68=
github.com/RoaringBitmap/roaring v1.6.0/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/RoaringBitmap/roaring v1.7.0/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 h1:byYvvbfSo3+9efR4IeReh77gVs4PnNDR3AMOE9NJ7a0= github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 h1:byYvvbfSo3+9efR4IeReh77gVs4PnNDR3AMOE9NJ7a0=
@ -112,9 +113,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/billziss-gh/cgofuse v1.5.0 h1:kH516I/s+Ab4diL/Y/ayFeUjjA8ey+JK12xDfBf4HEs= github.com/billziss-gh/cgofuse v1.5.0 h1:kH516I/s+Ab4diL/Y/ayFeUjjA8ey+JK12xDfBf4HEs=
github.com/billziss-gh/cgofuse v1.5.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM= github.com/billziss-gh/cgofuse v1.5.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
github.com/bodgit/sevenzip v1.4.5 h1:HFJQ+nbjppfyf2xbQEJBbmVo+o2kTg1FXV4i7YOx87s= github.com/bodgit/sevenzip v1.4.5 h1:HFJQ+nbjppfyf2xbQEJBbmVo+o2kTg1FXV4i7YOx87s=
@ -130,6 +131,7 @@ github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ
github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@ -156,14 +158,17 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8=
github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs=
github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak=
github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@ -209,15 +214,15 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 h1:OyQmpAN302wAopDgwVjgs2HkFawP9ahIEqkUYz7V7CA= github.com/go-llsqlite/adapter v0.1.0 h1:wGSQNsu/rtYeu/lqZNZQMjwUdEF3OW66xTLvsFwJQUw=
github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU= github.com/go-llsqlite/adapter v0.1.0/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU=
github.com/go-llsqlite/crawshaw v0.4.0 h1:L02s2jZBBJj80xm1VkkdyB/JlQ/Fi0kLbNHfXA8yrec= github.com/go-llsqlite/crawshaw v0.5.0 h1:Olbqkth53vkkh4WvmkYjrtfOBcxXD3rMYBYuk6FNH3E=
github.com/go-llsqlite/crawshaw v0.4.0/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= github.com/go-llsqlite/crawshaw v0.5.0/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@ -431,12 +436,14 @@ github.com/philippgille/gokv v0.6.0 h1:fNEx/tSwV73nzlYd3iRYB8F+SEVJNNFzH1gsaT8SK
github.com/philippgille/gokv v0.6.0/go.mod h1:tjXRFw9xDHgxLS8WJdfYotKGWp8TWqu4RdXjMDG/XBo= github.com/philippgille/gokv v0.6.0/go.mod h1:tjXRFw9xDHgxLS8WJdfYotKGWp8TWqu4RdXjMDG/XBo=
github.com/philippgille/gokv/badgerdb v0.6.0 h1:4Qigf2SpyXLF8KaM5nA5/D/0aD/bZevuAnrW4ZsDsjA= github.com/philippgille/gokv/badgerdb v0.6.0 h1:4Qigf2SpyXLF8KaM5nA5/D/0aD/bZevuAnrW4ZsDsjA=
github.com/philippgille/gokv/badgerdb v0.6.0/go.mod h1:3u2avs8gtmCc0R0Bw4jKV8aaDfLb5V9JToSASyhpFGM= github.com/philippgille/gokv/badgerdb v0.6.0/go.mod h1:3u2avs8gtmCc0R0Bw4jKV8aaDfLb5V9JToSASyhpFGM=
github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 h1:IgQDuUPuEFVf22mBskeCLAtvd5c9XiiJG2UYud6eGHI=
github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:SjxSrCoeYrYn85oTtroyG1ePY8aE72nvLQlw8IYwAN8= github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:SjxSrCoeYrYn85oTtroyG1ePY8aE72nvLQlw8IYwAN8=
github.com/philippgille/gokv/encoding v0.6.0 h1:P1TN+Aulpd6Qd7qcLqgPwoxzOQ42UHBXOovWvFxJRI8=
github.com/philippgille/gokv/encoding v0.6.0/go.mod h1:/yKvq2BKJlKJsH7KMDrhDlEw2Pt3V1nKyFhs4iOqz5U=
github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61 h1:4tVyBgfpK0NSqu7tNZTwYfC/pbyWUR2y+O7mxEg5BTQ= github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61 h1:4tVyBgfpK0NSqu7tNZTwYfC/pbyWUR2y+O7mxEg5BTQ=
github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:EUc+s9ONc1+VOr9NUEd8S0YbGRrQd/gz/p+2tvwt12s= github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:EUc+s9ONc1+VOr9NUEd8S0YbGRrQd/gz/p+2tvwt12s=
github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 h1:ril/jI0JgXNjPWwDkvcRxlZ09kgHXV2349xChjbsQ4o=
github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:2dBhsJgY/yVIkjY5V3AnDUxUbEPzT6uQ3LvoVT8TR20= github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:2dBhsJgY/yVIkjY5V3AnDUxUbEPzT6uQ3LvoVT8TR20=
github.com/philippgille/gokv/util v0.6.0 h1:GrTxVENzKBxs8lB3tnaA88mKOuVPT7atZPplxX+PNmo=
github.com/philippgille/gokv/util v0.6.0/go.mod h1:ovoDHZ2Svr7YX972SPPJQRXbhHEy3Gb20HRH/Tr9BiQ=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4= github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4=
github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@ -514,7 +521,6 @@ github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@ -543,6 +549,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
@ -581,8 +589,8 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= github.com/urfave/cli/v2 v2.27.0 h1:uNs1K8JwTFL84X68j5Fjny6hfANh9nTlJ6dRtZAFAHY=
github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/cli/v2 v2.27.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/willscott/go-nfs v0.0.1 h1:392gV283iuisKFeV9hkKwTdCRfizP+R9FC+gYg2skj0= github.com/willscott/go-nfs v0.0.1 h1:392gV283iuisKFeV9hkKwTdCRfizP+R9FC+gYg2skj0=
@ -632,8 +640,8 @@ golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -643,8 +651,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4=
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -889,8 +897,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@ -918,18 +926,18 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= modernc.org/libc v1.38.0 h1:o4Lpk0zNDSdsjfEXnF1FGXWQ9PDi1NOdWcLP5n13FGo=
modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw= modernc.org/libc v1.38.0/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
modernc.org/sqlite v1.21.1 h1:GyDFqNnESLOhwwDRaHGdp2jKLDzpyT/rNLglX3ZkMSU= modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ=
modernc.org/sqlite v1.21.1/go.mod h1:XwQ0wZPIh1iKb5mkvCJ3szzbhk+tykC8ZWqTRTgYRwI= modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0=
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
zombiezen.com/go/sqlite v0.13.1 h1:qDzxyWWmMtSSEH5qxamqBFmqA2BLSSbtODi3ojaE02o= zombiezen.com/go/sqlite v1.0.0 h1:D2EvOZqumJBy+6t+0uNTTXnepUpB/pKG45op/UziI1o=
zombiezen.com/go/sqlite v0.13.1/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4= zombiezen.com/go/sqlite v1.0.0/go.mod h1:Yx7FJ77tr7Ucwi5solhXAxpflyxk/BHNXArZ/JvDm60=

8
src/host/torrent/service.go → src/host/service/service.go
View file

@ -1,4 +1,4 @@
package torrent package service
import ( import (
"context" "context"
@ -6,7 +6,7 @@ import (
"log/slog" "log/slog"
"time" "time"
"git.kmsign.ru/royalcat/tstor/src/host/repository" "git.kmsign.ru/royalcat/tstor/src/host/storage"
"git.kmsign.ru/royalcat/tstor/src/host/vfs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
"github.com/anacrolix/torrent" "github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/metainfo"
@ -15,7 +15,7 @@ import (
type Service struct { type Service struct {
c *torrent.Client c *torrent.Client
rep repository.TorrentsRepository rep storage.TorrentsRepository
// stats *Stats // stats *Stats
DefaultPriority types.PiecePriority DefaultPriority types.PiecePriority
@ -24,7 +24,7 @@ type Service struct {
addTimeout, readTimeout int addTimeout, readTimeout int
} }
func NewService(c *torrent.Client, rep repository.TorrentsRepository, addTimeout, readTimeout int) *Service { func NewService(c *torrent.Client, rep storage.TorrentsRepository, addTimeout, readTimeout int) *Service {
l := slog.With("component", "torrent-service") l := slog.With("component", "torrent-service")
return &Service{ return &Service{
log: l, log: l,

2
src/host/torrent/stats.go → src/host/service/stats.go
View file

@ -1,4 +1,4 @@
package torrent package service
import ( import (
"errors" "errors"

4
src/host/storage.go
View file

@ -1,11 +1,11 @@
package host package host
import ( import (
"git.kmsign.ru/royalcat/tstor/src/host/torrent" "git.kmsign.ru/royalcat/tstor/src/host/service"
"git.kmsign.ru/royalcat/tstor/src/host/vfs" "git.kmsign.ru/royalcat/tstor/src/host/vfs"
) )
func NewStorage(dataPath string, tsrv *torrent.Service) vfs.Filesystem { func NewStorage(dataPath string, tsrv *service.Service) vfs.Filesystem {
factories := map[string]vfs.FsFactory{ factories := map[string]vfs.FsFactory{
".torrent": tsrv.NewTorrentFs, ".torrent": tsrv.NewTorrentFs,
} }

16
src/host/torrent/client.go → src/host/storage/client.go
View file

@ -1,4 +1,4 @@
package torrent package storage
import ( import (
"time" "time"
@ -20,20 +20,6 @@ func NewClient(st storage.ClientImpl, fis bep44.Store, cfg *config.TorrentClient
torrentCfg.PeerID = string(id[:]) torrentCfg.PeerID = string(id[:])
torrentCfg.DefaultStorage = st torrentCfg.DefaultStorage = st
// torrentCfg.DisableIPv6 = cfg.DisableIPv6
// torrentCfg.DropDuplicatePeerIds = true
// torrentCfg.TorrentPeersLowWater = 10
// torrentCfg.TorrentPeersHighWater = 100
// torrentCfg.DisableWebtorrent = true
// torrentCfg.DisableAggressiveUpload = true
// torrentCfg.DisableWebseeds = true
// torrentCfg.DisableUTP = false
// torrentCfg.NoDefaultPortForwarding = true
// torrentCfg.AlwaysWantConns = false
// torrentCfg.ClientDhtConfig = torrent.ClientDhtConfig{
// NoDHT: true,
// }
l := log.Logger.With().Str("component", "torrent-client").Logger() l := log.Logger.With().Str("component", "torrent-client").Logger()
tl := tlog.NewLogger() tl := tlog.NewLogger()

2
src/host/torrent/id.go → src/host/storage/id.go
View file

@ -1,4 +1,4 @@
package torrent package storage
import ( import (
"crypto/rand" "crypto/rand"

2
src/host/torrent/piece-completion.go → src/host/storage/piece-completion.go
View file

@ -1,4 +1,4 @@
package torrent package storage
import ( import (
"encoding/binary" "encoding/binary"

21
src/host/repository/repository.go → src/host/storage/repository.go
View file

@ -1,10 +1,11 @@
package repository package storage
import ( import (
"errors" "errors"
"path/filepath" "path/filepath"
"sync" "sync"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/metainfo"
"github.com/philippgille/gokv" "github.com/philippgille/gokv"
"github.com/philippgille/gokv/badgerdb" "github.com/philippgille/gokv/badgerdb"
@ -12,13 +13,13 @@ import (
) )
type TorrentsRepository interface { type TorrentsRepository interface {
ExcludeFile(hash metainfo.Hash, file ...string) error ExcludeFile(file *torrent.File) error
ExcludedFiles(hash metainfo.Hash) ([]string, error) ExcludedFiles(hash metainfo.Hash) ([]string, error)
} }
func NewTorrentMetaRepository(dir string) (TorrentsRepository, error) { func NewTorrentMetaRepository(metaDir string, storage *FileStorage) (TorrentsRepository, error) {
excludedFilesStore, err := badgerdb.NewStore(badgerdb.Options{ excludedFilesStore, err := badgerdb.NewStore(badgerdb.Options{
Dir: filepath.Join(dir, "excluded-files"), Dir: filepath.Join(metaDir, "excluded-files"),
Codec: encoding.JSON, Codec: encoding.JSON,
}) })
@ -28,6 +29,7 @@ func NewTorrentMetaRepository(dir string) (TorrentsRepository, error) {
r := &torrentRepositoryImpl{ r := &torrentRepositoryImpl{
excludedFiles: excludedFilesStore, excludedFiles: excludedFilesStore,
storage: storage,
} }
return r, nil return r, nil
@ -36,14 +38,16 @@ func NewTorrentMetaRepository(dir string) (TorrentsRepository, error) {
type torrentRepositoryImpl struct { type torrentRepositoryImpl struct {
m sync.RWMutex m sync.RWMutex
excludedFiles gokv.Store excludedFiles gokv.Store
storage *FileStorage
} }
var ErrNotFound = errors.New("not found") var ErrNotFound = errors.New("not found")
func (r *torrentRepositoryImpl) ExcludeFile(hash metainfo.Hash, file ...string) error { func (r *torrentRepositoryImpl) ExcludeFile(file *torrent.File) error {
r.m.Lock() r.m.Lock()
defer r.m.Unlock() defer r.m.Unlock()
hash := file.Torrent().InfoHash()
var excludedFiles []string var excludedFiles []string
found, err := r.excludedFiles.Get(hash.AsString(), &excludedFiles) found, err := r.excludedFiles.Get(hash.AsString(), &excludedFiles)
if err != nil { if err != nil {
@ -52,7 +56,12 @@ func (r *torrentRepositoryImpl) ExcludeFile(hash metainfo.Hash, file ...string)
if !found { if !found {
excludedFiles = []string{} excludedFiles = []string{}
} }
excludedFiles = unique(append(excludedFiles, file...)) excludedFiles = unique(append(excludedFiles, file.Path()))
err = r.storage.DeleteFile(file)
if err != nil {
return err
}
return r.excludedFiles.Set(hash.AsString(), excludedFiles) return r.excludedFiles.Set(hash.AsString(), excludedFiles)
} }

51
src/host/storage/storage.go Normal file
View file

@ -0,0 +1,51 @@
package storage
import (
"fmt"
"os"
"path/filepath"
"git.kmsign.ru/royalcat/tstor/src/config"
"github.com/anacrolix/torrent/storage"
)
func SetupStorage(cfg config.TorrentClient) (*FileStorage, storage.PieceCompletion, error) {
pcp := filepath.Join(cfg.DataFolder, "piece-completion")
if err := os.MkdirAll(pcp, 0744); err != nil {
return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err)
}
pc, err := storage.NewBoltPieceCompletion(pcp)
if err != nil {
return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err)
}
// pc, err := NewBadgerPieceCompletion(pcp)
// if err != nil {
// return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err)
// }
// TODO implement cache/storage switching
// cacheDir := filepath.Join(tcfg.DataFolder, "cache")
// if err := os.MkdirAll(cacheDir, 0744); err != nil {
// return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err)
// }
// fc, err := filecache.NewCache(cacheDir)
// if err != nil {
// return nil, nil, fmt.Errorf("error creating cache: %w", err)
// }
// log.Info().Msg(fmt.Sprintf("setting cache size to %d MB", 1024))
// fc.SetCapacity(1024 * 1024 * 1024)
// rp := storage.NewResourcePieces(fc.AsResourceProvider())
// st := &stc{rp}
filesDir := filepath.Join(cfg.DataFolder, "files")
if err := os.MkdirAll(pcp, 0744); err != nil {
return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err)
}
// st := storage.NewMMapWithCompletion(filesDir, pc)
st := NewFileStorage(filesDir, pc)
return st, pc, nil
}

302
src/host/storage/storage_files.go Normal file
View file

@ -0,0 +1,302 @@
package storage
import (
"fmt"
"io"
"log"
"os"
"path"
"path/filepath"
"strings"
"github.com/anacrolix/missinggo"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/common"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/segments"
"github.com/anacrolix/torrent/storage"
)
// NewFileStorage creates a new ClientImplCloser that stores files using the OS native filesystem.
func NewFileStorage(baseDir string, pc storage.PieceCompletion) *FileStorage {
return &FileStorage{baseDir: baseDir, pieceCompletion: pc}
}
// File-based storage for torrents, that isn't yet bound to a particular torrent.
type FileStorage struct {
baseDir string
pieceCompletion storage.PieceCompletion
}
func (me *FileStorage) Close() error {
return me.pieceCompletion.Close()
}
func (me *FileStorage) torrentDir(info *metainfo.Info, infoHash metainfo.Hash) string {
return filepath.Join(me.baseDir, info.Name)
}
func (me *FileStorage) filePath(file metainfo.FileInfo) string {
return filepath.Join(file.Path...)
}
func (fs *FileStorage) DeleteFile(file *torrent.File) error {
info := file.Torrent().Info()
infoHash := file.Torrent().InfoHash()
torrentDir := fs.torrentDir(info, infoHash)
relFilePath := fs.filePath(file.FileInfo())
filePath := path.Join(torrentDir, relFilePath)
return os.Remove(filePath)
}
func (fs FileStorage) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (storage.TorrentImpl, error) {
dir := fs.torrentDir(info, infoHash)
upvertedFiles := info.UpvertedFiles()
files := make([]file, 0, len(upvertedFiles))
for i, fileInfo := range upvertedFiles {
filePath := filepath.Join(dir, fs.filePath(fileInfo))
if !isSubFilepath(dir, filePath) {
return storage.TorrentImpl{}, fmt.Errorf("file %v: path %q is not sub path of %q", i, filePath, fs.baseDir)
}
f := file{
path: filePath,
length: fileInfo.Length,
}
if f.length == 0 {
err := CreateNativeZeroLengthFile(f.path)
if err != nil {
return storage.TorrentImpl{}, fmt.Errorf("creating zero length file: %w", err)
}
}
files = append(files, f)
}
t := &fileTorrentImpl{
files: files,
segmentLocater: segments.NewIndex(common.LengthIterFromUpvertedFiles(upvertedFiles)),
infoHash: infoHash,
completion: fs.pieceCompletion,
}
return storage.TorrentImpl{
Piece: t.Piece,
Close: t.Close,
}, nil
}
type file struct {
// The safe, OS-local file path.
path string
length int64
}
type fileTorrentImpl struct {
files []file
segmentLocater segments.Index
infoHash metainfo.Hash
completion storage.PieceCompletion
}
func (fts *fileTorrentImpl) Piece(p metainfo.Piece) storage.PieceImpl {
// Create a view onto the file-based torrent storage.
_io := fileTorrentImplIO{fts}
// Return the appropriate segments of this.
return &filePieceImpl{
fileTorrentImpl: fts,
p: p,
WriterAt: missinggo.NewSectionWriter(_io, p.Offset(), p.Length()),
ReaderAt: io.NewSectionReader(_io, p.Offset(), p.Length()),
}
}
func (fs *fileTorrentImpl) Close() error {
return nil
}
// A helper to create zero-length files which won't appear for file-orientated storage since no
// writes will ever occur to them (no torrent data is associated with a zero-length file). The
// caller should make sure the file name provided is safe/sanitized.
func CreateNativeZeroLengthFile(name string) error {
err := os.MkdirAll(filepath.Dir(name), 0o777)
if err != nil {
return err
}
f, err := os.Create(name)
if err != nil {
return err
}
return f.Close()
}
// Exposes file-based storage of a torrent, as one big ReadWriterAt.
type fileTorrentImplIO struct {
fts *fileTorrentImpl
}
// Returns EOF on short or missing file.
func (fst *fileTorrentImplIO) readFileAt(file file, b []byte, off int64) (n int, err error) {
f, err := os.Open(file.path)
if os.IsNotExist(err) {
// File missing is treated the same as a short file.
err = io.EOF
return
}
if err != nil {
return
}
defer f.Close()
// Limit the read to within the expected bounds of this file.
if int64(len(b)) > file.length-off {
b = b[:file.length-off]
}
for off < file.length && len(b) != 0 {
n1, err1 := f.ReadAt(b, off)
b = b[n1:]
n += n1
off += int64(n1)
if n1 == 0 {
err = err1
break
}
}
return
}
// Only returns EOF at the end of the torrent. Premature EOF is ErrUnexpectedEOF.
func (fst fileTorrentImplIO) ReadAt(b []byte, off int64) (n int, err error) {
fst.fts.segmentLocater.Locate(
segments.Extent{Start: off, Length: int64(len(b))},
func(i int, e segments.Extent) bool {
n1, err1 := fst.readFileAt(fst.fts.files[i], b[:e.Length], e.Start)
n += n1
b = b[n1:]
err = err1
return err == nil // && int64(n1) == e.Length
},
)
if len(b) != 0 && err == nil {
err = io.EOF
}
return
}
func (fst fileTorrentImplIO) WriteAt(p []byte, off int64) (n int, err error) {
// log.Printf("write at %v: %v bytes", off, len(p))
fst.fts.segmentLocater.Locate(
segments.Extent{Start: off, Length: int64(len(p))},
func(i int, e segments.Extent) bool {
name := fst.fts.files[i].path
err = os.MkdirAll(filepath.Dir(name), 0o777)
if err != nil {
return false
}
var f *os.File
f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0o666)
if err != nil {
return false
}
var n1 int
n1, err = f.WriteAt(p[:e.Length], e.Start)
// log.Printf("%v %v wrote %v: %v", i, e, n1, err)
closeErr := f.Close()
n += n1
p = p[n1:]
if err == nil {
err = closeErr
}
if err == nil && int64(n1) != e.Length {
err = io.ErrShortWrite
}
return err == nil
},
)
return n, err
}
type filePieceImpl struct {
*fileTorrentImpl
p metainfo.Piece
io.WriterAt
io.ReaderAt
}
var _ storage.PieceImpl = (*filePieceImpl)(nil)
func (me *filePieceImpl) pieceKey() metainfo.PieceKey {
return metainfo.PieceKey{InfoHash: me.infoHash, Index: me.p.Index()}
}
func (fs *filePieceImpl) Completion() storage.Completion {
c, err := fs.completion.Get(fs.pieceKey())
if err != nil {
log.Printf("error getting piece completion: %s", err)
c.Ok = false
return c
}
verified := true
if c.Complete {
// If it's allegedly complete, check that its constituent files have the necessary length.
for _, fi := range extentCompleteRequiredLengths(fs.p.Info, fs.p.Offset(), fs.p.Length()) {
s, err := os.Stat(fs.files[fi.fileIndex].path)
if err != nil || s.Size() < fi.length {
verified = false
break
}
}
}
if !verified {
// The completion was wrong, fix it.
c.Complete = false
fs.completion.Set(fs.pieceKey(), false)
}
return c
}
func (fs *filePieceImpl) MarkComplete() error {
return fs.completion.Set(fs.pieceKey(), true)
}
func (fs *filePieceImpl) MarkNotComplete() error {
return fs.completion.Set(fs.pieceKey(), false)
}
type requiredLength struct {
fileIndex int
length int64
}
func isSubFilepath(base, sub string) bool {
rel, err := filepath.Rel(base, sub)
if err != nil {
return false
}
return rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator))
}
func extentCompleteRequiredLengths(info *metainfo.Info, off, n int64) (ret []requiredLength) {
if n == 0 {
return
}
for i, fi := range info.UpvertedFiles() {
if off >= fi.Length {
off -= fi.Length
continue
}
n1 := n
if off+n1 > fi.Length {
n1 = fi.Length - off
}
ret = append(ret, requiredLength{
fileIndex: i,
length: off + n1,
})
n -= n1
if n == 0 {
return
}
off = 0
}
panic("extent exceeds torrent bounds")
}

2
src/host/torrent/store.go → src/host/storage/store.go
View file

@ -1,4 +1,4 @@
package torrent package storage
import ( import (
"bytes" "bytes"

306
src/host/torrent/storage.go
View file

@ -1,306 +0,0 @@
package torrent
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"git.kmsign.ru/royalcat/tstor/src/config"
"github.com/anacrolix/missinggo"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/mmap_span"
"github.com/anacrolix/torrent/storage"
"github.com/edsrzf/mmap-go"
)
type Torrent struct {
client *torrent.Client
data storage.ClientImplCloser
pc storage.PieceCompletion
}
func SetupStorage(cfg config.TorrentClient) (storage.ClientImplCloser, storage.PieceCompletion, error) {
pcp := filepath.Join(cfg.DataFolder, "piece-completion")
if err := os.MkdirAll(pcp, 0744); err != nil {
return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err)
}
pc, err := storage.NewBoltPieceCompletion(pcp)
if err != nil {
return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err)
}
// pc, err := NewBadgerPieceCompletion(pcp)
// if err != nil {
// return nil, nil, fmt.Errorf("error creating servers piece completion: %w", err)
// }
// TODO implement cache/storage switching
// cacheDir := filepath.Join(tcfg.DataFolder, "cache")
// if err := os.MkdirAll(cacheDir, 0744); err != nil {
// return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err)
// }
// fc, err := filecache.NewCache(cacheDir)
// if err != nil {
// return nil, nil, fmt.Errorf("error creating cache: %w", err)
// }
// log.Info().Msg(fmt.Sprintf("setting cache size to %d MB", 1024))
// fc.SetCapacity(1024 * 1024 * 1024)
// rp := storage.NewResourcePieces(fc.AsResourceProvider())
// st := &stc{rp}
filesDir := filepath.Join(cfg.DataFolder, "files")
if err := os.MkdirAll(pcp, 0744); err != nil {
return nil, nil, fmt.Errorf("error creating piece completion folder: %w", err)
}
// st := storage.NewMMapWithCompletion(filesDir, pc)
st := storage.NewFileOpts(storage.NewFileClientOpts{
ClientBaseDir: filesDir,
PieceCompletion: pc,
})
return st, pc, nil
}
func (s Torrent) Remove(f *torrent.File) error {
return nil
}
// type dupePieces struct {
// }
// func (s Torrent) dedupe(f1, f2 *os.File) error {
// for _, t := range s.client.Torrents() {
// for i := 0; i < t.NumPieces(); i++ {
// p := t.Piece(i)
// p.Info().Hash()
// }
// }
// // https://go-review.googlesource.com/c/sys/+/284352/10/unix/syscall_linux_test.go#856
// // dedupe := unix.FileDedupeRange{
// // Src_offset: uint64(0),
// // Src_length: uint64(4096),
// // Info: []unix.FileDedupeRangeInfo{
// // unix.FileDedupeRangeInfo{
// // Dest_fd: int64(f2.Fd()),
// // Dest_offset: uint64(0),
// // },
// // unix.FileDedupeRangeInfo{
// // Dest_fd: int64(f2.Fd()),
// // Dest_offset: uint64(4096),
// // },
// // }}
// // err := unix.IoctlFileDedupeRange(int(f1.Fd()), &dedupe)
// // if err == unix.EOPNOTSUPP || err == unix.EINVAL {
// // t.Skip("deduplication not supported on this filesystem")
// // } else if err != nil {
// // t.Fatal(err)
// // }
// return nil
// }
type mmapClientImpl struct {
baseDir string
pc storage.PieceCompletion
}
func NewMMapWithCompletion(baseDir string, completion storage.PieceCompletion) *mmapClientImpl {
return &mmapClientImpl{
baseDir: baseDir,
pc: completion,
}
}
func (s *mmapClientImpl) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (_ storage.TorrentImpl, err error) {
t, err := newMMapTorrent(info, infoHash, s.baseDir, s.pc)
if err != nil {
return storage.TorrentImpl{}, err
}
return storage.TorrentImpl{Piece: t.Piece, Close: t.Close, Flush: t.Flush}, nil
}
func (s *mmapClientImpl) Close() error {
return s.pc.Close()
}
func newMMapTorrent(md *metainfo.Info, infoHash metainfo.Hash, location string, pc storage.PieceCompletionGetSetter) (*mmapTorrent, error) {
span := &mmap_span.MMapSpan{}
basePath, err := storage.ToSafeFilePath(md.Name)
if err != nil {
return nil, err
}
basePath = filepath.Join(location, basePath)
for _, miFile := range md.UpvertedFiles() {
var safeName string
safeName, err = storage.ToSafeFilePath(miFile.Path...)
if err != nil {
return nil, err
}
fileName := filepath.Join(basePath, safeName)
var mm FileMapping
mm, err = mmapFile(fileName, miFile.Length)
if err != nil {
err = fmt.Errorf("file %q: %s", miFile.DisplayPath(md), err)
return nil, err
}
span.Append(mm)
}
span.InitIndex()
return &mmapTorrent{
infoHash: infoHash,
span: span,
pc: pc,
}, nil
}
type mmapTorrent struct {
infoHash metainfo.Hash
span *mmap_span.MMapSpan
pc storage.PieceCompletionGetSetter
}
func (ts *mmapTorrent) Piece(p metainfo.Piece) storage.PieceImpl {
return mmapPiece{
pc: ts.pc,
p: p,
ih: ts.infoHash,
ReaderAt: io.NewSectionReader(ts.span, p.Offset(), p.Length()),
WriterAt: missinggo.NewSectionWriter(ts.span, p.Offset(), p.Length()),
}
}
func (ts *mmapTorrent) Close() error {
errs := ts.span.Close()
if len(errs) > 0 {
return errs[0]
}
return nil
}
func (ts *mmapTorrent) Flush() error {
errs := ts.span.Flush()
if len(errs) > 0 {
return errs[0]
}
return nil
}
type mmapPiece struct {
pc storage.PieceCompletionGetSetter
p metainfo.Piece
ih metainfo.Hash
io.ReaderAt
io.WriterAt
}
func (me mmapPiece) pieceKey() metainfo.PieceKey {
return metainfo.PieceKey{InfoHash: me.ih, Index: me.p.Index()}
}
func (sp mmapPiece) Completion() storage.Completion {
c, err := sp.pc.Get(sp.pieceKey())
if err != nil {
panic(err)
}
return c
}
func (sp mmapPiece) MarkComplete() error {
return sp.pc.Set(sp.pieceKey(), true)
}
func (sp mmapPiece) MarkNotComplete() error {
return sp.pc.Set(sp.pieceKey(), false)
}
func mmapFile(name string, size int64) (_ FileMapping, err error) {
dir := filepath.Dir(name)
err = os.MkdirAll(dir, 0o750)
if err != nil {
return nil, fmt.Errorf("making directory %q: %s", dir, err)
}
var file *os.File
file, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0o666)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
file.Close()
}
}()
var fi os.FileInfo
fi, err = file.Stat()
if err != nil {
return nil, err
}
if fi.Size() < size {
// I think this is necessary on HFS+. Maybe Linux will SIGBUS too if
// you overmap a file but I'm not sure.
err = file.Truncate(size)
if err != nil {
return nil, err
}
}
return func() (ret mmapWithFile, err error) {
ret.f = file
if size == 0 {
// Can't mmap() regions with length 0.
return
}
intLen := int(size)
if int64(intLen) != size {
err = errors.New("size too large for system")
return
}
ret.mmap, err = mmap.MapRegion(file, intLen, mmap.RDWR, 0, 0)
if err != nil {
err = fmt.Errorf("error mapping region: %s", err)
return
}
if int64(len(ret.mmap)) != size {
panic(len(ret.mmap))
}
return
}()
}
type FileMapping = mmap_span.Mmap
// Handles closing the mmap's file handle (needed for Windows). Could be implemented differently by
// OS.
type mmapWithFile struct {
f *os.File
mmap mmap.MMap
}
func (m mmapWithFile) Flush() error {
return m.mmap.Flush()
}
func (m mmapWithFile) Unmap() (err error) {
if m.mmap != nil {
err = m.mmap.Unmap()
}
fileErr := m.f.Close()
if err == nil {
err = fileErr
}
return
}
func (m mmapWithFile) Bytes() []byte {
if m.mmap == nil {
return nil
}
return m.mmap
}

43
src/host/vfs/torrent.go
View file

@ -9,7 +9,7 @@ import (
"sync" "sync"
"time" "time"
"git.kmsign.ru/royalcat/tstor/src/host/repository" "git.kmsign.ru/royalcat/tstor/src/host/storage"
"git.kmsign.ru/royalcat/tstor/src/iio" "git.kmsign.ru/royalcat/tstor/src/iio"
"github.com/anacrolix/missinggo/v2" "github.com/anacrolix/missinggo/v2"
"github.com/anacrolix/torrent" "github.com/anacrolix/torrent"
@ -21,7 +21,7 @@ var _ Filesystem = &TorrentFs{}
type TorrentFs struct { type TorrentFs struct {
mu sync.Mutex mu sync.Mutex
t *torrent.Torrent t *torrent.Torrent
rep repository.TorrentsRepository rep storage.TorrentsRepository
readTimeout int readTimeout int
@ -31,7 +31,7 @@ type TorrentFs struct {
resolver *resolver resolver *resolver
} }
func NewTorrentFs(t *torrent.Torrent, rep repository.TorrentsRepository, readTimeout int) *TorrentFs { func NewTorrentFs(t *torrent.Torrent, rep storage.TorrentsRepository, readTimeout int) *TorrentFs {
return &TorrentFs{ return &TorrentFs{
t: t, t: t,
rep: rep, rep: rep,
@ -53,17 +53,17 @@ func (fs *TorrentFs) files() (map[string]*torrentFile, error) {
fs.filesCache = make(map[string]*torrentFile) fs.filesCache = make(map[string]*torrentFile)
for _, file := range files { for _, file := range files {
p := AbsPath(file.Path())
if slices.Contains(excludedFiles, p) { if slices.Contains(excludedFiles, file.Path()) {
continue continue
} }
p := AbsPath(file.Path())
fs.filesCache[p] = &torrentFile{ fs.filesCache[p] = &torrentFile{
name: path.Base(p), name: path.Base(p),
readerFunc: file.NewReader, timeout: fs.readTimeout,
len: file.Length(), file: file,
timeout: fs.readTimeout,
} }
} }
fs.mu.Unlock() fs.mu.Unlock()
@ -144,6 +144,8 @@ func (fs *TorrentFs) ReadDir(name string) ([]fs.DirEntry, error) {
} }
func (fs *TorrentFs) Unlink(name string) error { func (fs *TorrentFs) Unlink(name string) error {
name = AbsPath(name)
fs.mu.Lock() fs.mu.Lock()
defer fs.mu.Unlock() defer fs.mu.Unlock()
@ -151,14 +153,15 @@ func (fs *TorrentFs) Unlink(name string) error {
if err != nil { if err != nil {
return err return err
} }
file := AbsPath(name)
if !slices.Contains(maps.Keys(files), file) { if !slices.Contains(maps.Keys(files), name) {
return ErrNotExist return ErrNotExist
} }
fs.filesCache = nil
return fs.rep.ExcludeFile(fs.t.InfoHash(), file) file := files[name]
delete(fs.filesCache, name)
return fs.rep.ExcludeFile(file.file)
} }
type reader interface { type reader interface {
@ -224,25 +227,25 @@ var _ File = &torrentFile{}
type torrentFile struct { type torrentFile struct {
name string name string
readerFunc func() torrent.Reader reader reader
reader reader timeout int
len int64
timeout int file *torrent.File
} }
func (d *torrentFile) Stat() (fs.FileInfo, error) { func (d *torrentFile) Stat() (fs.FileInfo, error) {
return newFileInfo(d.name, d.len), nil return newFileInfo(d.name, d.file.Length()), nil
} }
func (d *torrentFile) load() { func (d *torrentFile) load() {
if d.reader != nil { if d.reader != nil {
return return
} }
d.reader = newReadAtWrapper(d.readerFunc(), d.timeout) d.reader = newReadAtWrapper(d.file.NewReader(), d.timeout)
} }
func (d *torrentFile) Size() int64 { func (d *torrentFile) Size() int64 {
return d.len return d.file.Length()
} }
func (d *torrentFile) IsDir() bool { func (d *torrentFile) IsDir() bool {

5
src/host/vfs/torrent_test.go
View file

@ -96,9 +96,8 @@ func TestReadAtTorrent(t *testing.T) {
torrFile := to.Files()[0] torrFile := to.Files()[0]
tf := torrentFile{ tf := torrentFile{
readerFunc: torrFile.NewReader, file: torrFile,
len: torrFile.Length(), timeout: 500,
timeout: 500,
} }
defer tf.Close() defer tf.Close()

4
src/http/api.go
View file

@ -7,12 +7,12 @@ import (
"net/http" "net/http"
"os" "os"
"git.kmsign.ru/royalcat/tstor/src/host/torrent" "git.kmsign.ru/royalcat/tstor/src/host/service"
"github.com/anacrolix/missinggo/v2/filecache" "github.com/anacrolix/missinggo/v2/filecache"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
) )
var apiStatusHandler = func(fc *filecache.Cache, ss *torrent.Stats) gin.HandlerFunc { var apiStatusHandler = func(fc *filecache.Cache, ss *service.Stats) gin.HandlerFunc {
return func(ctx *gin.Context) { return func(ctx *gin.Context) {
stat := gin.H{ stat := gin.H{
"torrentStats": ss.GlobalStats(), "torrentStats": ss.GlobalStats(),

4
src/http/http.go
View file

@ -6,7 +6,7 @@ import (
"git.kmsign.ru/royalcat/tstor" "git.kmsign.ru/royalcat/tstor"
"git.kmsign.ru/royalcat/tstor/src/config" "git.kmsign.ru/royalcat/tstor/src/config"
"git.kmsign.ru/royalcat/tstor/src/host/torrent" "git.kmsign.ru/royalcat/tstor/src/host/service"
"github.com/anacrolix/missinggo/v2/filecache" "github.com/anacrolix/missinggo/v2/filecache"
"github.com/gin-contrib/pprof" "github.com/gin-contrib/pprof"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
@ -14,7 +14,7 @@ import (
"github.com/shurcooL/httpfs/html/vfstemplate" "github.com/shurcooL/httpfs/html/vfstemplate"
) )
func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, logPath string, cfg *config.Config) error { func New(fc *filecache.Cache, ss *service.Stats, s *service.Service, logPath string, cfg *config.Config) error {
gin.SetMode(gin.ReleaseMode) gin.SetMode(gin.ReleaseMode)
r := gin.New() r := gin.New()
r.Use(gin.Recovery()) r.Use(gin.Recovery())

2
src/iio/disk_test.go
View file

@ -26,7 +26,7 @@ func TestReadData(t *testing.T) {
require.Equal(5, n) require.Equal(5, n)
require.Equal("World", string(toRead)) require.Equal("World", string(toRead))
r.ReadAt(toRead, 0) n, err = r.ReadAt(toRead, 0)
require.NoError(err) require.NoError(err)
require.Equal(5, n) require.Equal(5, n)
require.Equal("Hello", string(toRead)) require.Equal("Hello", string(toRead))