From 033220656097a1b809fc5ec03ec7a25028b2c9ca Mon Sep 17 00:00:00 2001 From: royalcat Date: Tue, 26 Dec 2023 01:11:03 +0300 Subject: [PATCH] [feature] file exclude --- cmd/tstor/main.go | 13 ++++- go.mod | 9 +++- go.sum | 44 +++++++++++++++ src/host/repository/repository.go | 90 +++++++++++++++++++++++++++++++ src/host/torrent/service.go | 9 ++-- src/host/vfs/archive.go | 5 +- src/host/vfs/fs.go | 1 + src/host/vfs/memory.go | 5 ++ src/host/vfs/os.go | 5 ++ src/host/vfs/resolver.go | 13 +++++ src/host/vfs/resolver_test.go | 4 ++ src/host/vfs/torrent.go | 62 +++++++++++++++++---- src/mounts/webdav/fs.go | 2 +- 13 files changed, 243 insertions(+), 19 deletions(-) create mode 100644 src/host/repository/repository.go diff --git a/cmd/tstor/main.go b/cmd/tstor/main.go index eddba22..8ded9b5 100644 --- a/cmd/tstor/main.go +++ b/cmd/tstor/main.go @@ -14,6 +14,7 @@ import ( "git.kmsign.ru/royalcat/tstor/src/config" "git.kmsign.ru/royalcat/tstor/src/host" + "git.kmsign.ru/royalcat/tstor/src/host/repository" "git.kmsign.ru/royalcat/tstor/src/host/torrent" "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/rs/zerolog/log" @@ -72,6 +73,14 @@ func run(configPath string) error { log.Err(err).Msg("set priority failed") } + if err := os.MkdirAll(filepath.Join(conf.TorrentClient.MetadataFolder, "meta"), 0744); err != nil { + return fmt.Errorf("error creating metadata folder: %w", err) + } + rep, err := repository.NewTorrentMetaRepository(filepath.Join(conf.TorrentClient.MetadataFolder, "meta")) + if err != nil { + return err + } + if err := os.MkdirAll(conf.TorrentClient.MetadataFolder, 0744); err != nil { return fmt.Errorf("error creating metadata folder: %w", err) } @@ -100,7 +109,7 @@ func run(configPath string) error { c.AddDhtNodes(conf.TorrentClient.DHTNodes) defer c.Close() - ts := torrent.NewService(c, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout) + ts := torrent.NewService(c, rep, conf.TorrentClient.AddTimeout, conf.TorrentClient.ReadTimeout) if err := os.MkdirAll(conf.DataFolder, 0744); err != nil { return fmt.Errorf("error creating data folder: %w", err) @@ -139,7 +148,7 @@ func run(configPath string) error { // c.FileFromFS(path, httpfs) // }) - // log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port)).Msg("starting HTTPFS") + log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port)).Msg("starting HTTPFS") // if err := r.Run(fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port)); err != nil { // log.Error().Err(err).Msg("error starting HTTPFS") // } diff --git a/go.mod b/go.mod index a9b95cd..5ad4fc0 100644 --- a/go.mod +++ b/go.mod @@ -22,16 +22,21 @@ require ( github.com/knadh/koanf/v2 v2.0.1 github.com/mattn/go-colorable v0.1.13 github.com/nwaples/rardecode/v2 v2.0.0-beta.2 + github.com/philippgille/gokv v0.6.0 + github.com/philippgille/gokv/badgerdb v0.6.0 + github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 github.com/rs/zerolog v1.31.0 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.26.0 github.com/willscott/go-nfs v0.0.1 + golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 golang.org/x/net v0.19.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( + github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/RoaringBitmap/roaring v1.6.0 // indirect github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect github.com/alecthomas/atomic v0.1.0-alpha2 // indirect @@ -59,7 +64,9 @@ require ( github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/badger v1.6.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -99,6 +106,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 // indirect github.com/pierrec/lz4/v4 v4.1.19 // indirect github.com/pion/datachannel v1.5.5 // indirect github.com/pion/dtls/v2 v2.2.8 // indirect @@ -137,7 +145,6 @@ require ( go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/arch v0.6.0 // indirect golang.org/x/crypto v0.16.0 // indirect - golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index b8ad50d..fb686fc 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,9 @@ crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= @@ -98,6 +101,7 @@ github.com/anacrolix/utp v0.2.0/go.mod h1:HGk4GYQw1O/3T1+yhqT/F6EcBd+AAwlo9dYErN github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= @@ -141,13 +145,19 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= @@ -224,6 +234,8 @@ github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqR github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= +github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -310,6 +322,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -320,6 +333,7 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -364,6 +378,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -374,6 +389,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -403,10 +420,23 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/philippgille/gokv v0.0.0-20191001201555-5ac9a20de634/go.mod h1:OCoWPt+mbYuTO1FUVrQ2SxQU0oaaHBsn6lRhFX3JHOc= +github.com/philippgille/gokv v0.5.1-0.20191011213304-eb77f15b9c61/go.mod h1:OCoWPt+mbYuTO1FUVrQ2SxQU0oaaHBsn6lRhFX3JHOc= +github.com/philippgille/gokv v0.6.0 h1:fNEx/tSwV73nzlYd3iRYB8F+SEVJNNFzH1gsaT8SK2c= +github.com/philippgille/gokv v0.6.0/go.mod h1:tjXRFw9xDHgxLS8WJdfYotKGWp8TWqu4RdXjMDG/XBo= +github.com/philippgille/gokv/badgerdb v0.6.0 h1:4Qigf2SpyXLF8KaM5nA5/D/0aD/bZevuAnrW4ZsDsjA= +github.com/philippgille/gokv/badgerdb v0.6.0/go.mod h1:3u2avs8gtmCc0R0Bw4jKV8aaDfLb5V9JToSASyhpFGM= +github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 h1:IgQDuUPuEFVf22mBskeCLAtvd5c9XiiJG2UYud6eGHI= +github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:SjxSrCoeYrYn85oTtroyG1ePY8aE72nvLQlw8IYwAN8= +github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61 h1:4tVyBgfpK0NSqu7tNZTwYfC/pbyWUR2y+O7mxEg5BTQ= +github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:EUc+s9ONc1+VOr9NUEd8S0YbGRrQd/gz/p+2tvwt12s= +github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 h1:ril/jI0JgXNjPWwDkvcRxlZ09kgHXV2349xChjbsQ4o= +github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:2dBhsJgY/yVIkjY5V3AnDUxUbEPzT6uQ3LvoVT8TR20= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4= github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -498,6 +528,7 @@ github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -512,6 +543,12 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -538,6 +575,7 @@ github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= @@ -551,6 +589,7 @@ github.com/willscott/go-nfs v0.0.1 h1:392gV283iuisKFeV9hkKwTdCRfizP+R9FC+gYg2skj github.com/willscott/go-nfs v0.0.1/go.mod h1:hBPyqKNde3v8rzxDVWtloP6MtLnx/7aVz3XxxP89W7k= github.com/willscott/go-nfs-client v0.0.0-20200605172546-271fa9065b33 h1:Wd8wdpRzPXskyHvZLyw7Wc1fp5oCE2mhBCj7bAiibUs= github.com/willscott/go-nfs-client v0.0.0-20200605172546-271fa9065b33/go.mod h1:cOUKSNty+RabZqKhm5yTJT5Vq/Fe83ZRWAJ5Kj8nRes= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -580,6 +619,7 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc= golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -642,6 +682,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191011234655-491137f69257/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -685,6 +726,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -693,9 +735,11 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/src/host/repository/repository.go b/src/host/repository/repository.go new file mode 100644 index 0000000..aac2021 --- /dev/null +++ b/src/host/repository/repository.go @@ -0,0 +1,90 @@ +package repository + +import ( + "errors" + "sync" + + "github.com/anacrolix/torrent/metainfo" + "github.com/philippgille/gokv" + "github.com/philippgille/gokv/badgerdb" + "github.com/philippgille/gokv/encoding" +) + +type TorrentMetaRepository interface { + ExcludeFile(hash metainfo.Hash, file ...string) error + ExcludedFiles(hash metainfo.Hash) ([]string, error) +} + +func NewTorrentMetaRepository(dir string) (TorrentMetaRepository, error) { + store, err := badgerdb.NewStore(badgerdb.Options{ + Dir: dir, + Codec: encoding.JSON, + }) + if err != nil { + return nil, err + } + + r := &torrentRepositoryImpl{ + store: store, + } + + return r, nil +} + +type torrentRepositoryImpl struct { + m sync.RWMutex + store gokv.Store +} + +type torrentMeta struct { + ExludedFiles []string +} + +var ErrNotFound = errors.New("not found") + +func (r *torrentRepositoryImpl) ExcludeFile(hash metainfo.Hash, file ...string) error { + r.m.Lock() + defer r.m.Unlock() + + var meta torrentMeta + found, err := r.store.Get(hash.AsString(), &meta) + if err != nil { + return err + } + if !found { + meta = torrentMeta{ + ExludedFiles: file, + } + } + meta.ExludedFiles = unique(append(meta.ExludedFiles, file...)) + + return r.store.Set(hash.AsString(), meta) +} + +func (r *torrentRepositoryImpl) ExcludedFiles(hash metainfo.Hash) ([]string, error) { + r.m.Lock() + defer r.m.Unlock() + + var meta torrentMeta + found, err := r.store.Get(hash.AsString(), &meta) + if err != nil { + return nil, err + } + if !found { + return nil, nil + } + + return meta.ExludedFiles, nil +} + +func unique[C comparable](intSlice []C) []C { + keys := make(map[C]bool) + list := []C{} + for _, entry := range intSlice { + if _, value := keys[entry]; !value { + keys[entry] = true + list = append(list, entry) + } + } + return list +} diff --git a/src/host/torrent/service.go b/src/host/torrent/service.go index 7a074de..a3bc382 100644 --- a/src/host/torrent/service.go +++ b/src/host/torrent/service.go @@ -6,6 +6,7 @@ import ( "log/slog" "time" + "git.kmsign.ru/royalcat/tstor/src/host/repository" "git.kmsign.ru/royalcat/tstor/src/host/vfs" "github.com/anacrolix/torrent" "github.com/anacrolix/torrent/metainfo" @@ -13,7 +14,8 @@ import ( ) type Service struct { - c *torrent.Client + c *torrent.Client + rep repository.TorrentMetaRepository // stats *Stats DefaultPriority types.PiecePriority @@ -22,12 +24,13 @@ type Service struct { addTimeout, readTimeout int } -func NewService(c *torrent.Client, addTimeout, readTimeout int) *Service { +func NewService(c *torrent.Client, rep repository.TorrentMetaRepository, addTimeout, readTimeout int) *Service { l := slog.With("component", "torrent-service") return &Service{ log: l, c: c, DefaultPriority: types.PiecePriorityNone, + rep: rep, // stats: newStats(), // TODO persistent addTimeout: addTimeout, readTimeout: readTimeout, @@ -63,7 +66,7 @@ func (s *Service) NewTorrentFs(f vfs.File) (vfs.Filesystem, error) { t.AllowDataDownload() } - return vfs.NewTorrentFs(t, s.readTimeout), nil + return vfs.NewTorrentFs(t, s.rep, s.readTimeout), nil } func (s *Service) Stats() (*Stats, error) { diff --git a/src/host/vfs/archive.go b/src/host/vfs/archive.go index 585f7f9..f4e5649 100644 --- a/src/host/vfs/archive.go +++ b/src/host/vfs/archive.go @@ -49,7 +49,10 @@ func NewArchive(r iio.Reader, size int64, loader ArchiveLoader) *archive { } } -var _ Filesystem = &archive{} +// Unlink implements Filesystem. +func (a *archive) Unlink(filename string) error { + return ErrNotImplemented +} func (a *archive) Open(filename string) (File, error) { files, err := a.files() diff --git a/src/host/vfs/fs.go b/src/host/vfs/fs.go index 3cfc1fc..5f1951b 100644 --- a/src/host/vfs/fs.go +++ b/src/host/vfs/fs.go @@ -30,6 +30,7 @@ type Filesystem interface { ReadDir(path string) ([]fs.DirEntry, error) Stat(filename string) (fs.FileInfo, error) + Unlink(filename string) error } const defaultMode = fs.FileMode(0555) diff --git a/src/host/vfs/memory.go b/src/host/vfs/memory.go index eada6e0..2ef42fb 100644 --- a/src/host/vfs/memory.go +++ b/src/host/vfs/memory.go @@ -12,6 +12,11 @@ type MemoryFs struct { files map[string]*MemoryFile } +// Unlink implements Filesystem. +func (fs *MemoryFs) Unlink(filename string) error { + return ErrNotImplemented +} + func NewMemoryFS(files map[string]*MemoryFile) *MemoryFs { return &MemoryFs{ files: files, diff --git a/src/host/vfs/os.go b/src/host/vfs/os.go index 4d09b05..3ec6ad5 100644 --- a/src/host/vfs/os.go +++ b/src/host/vfs/os.go @@ -11,6 +11,11 @@ type OsFS struct { hostDir string } +// Unlink implements Filesystem. +func (fs *OsFS) Unlink(filename string) error { + return fs.Unlink(filename) +} + // Stat implements Filesystem. func (fs *OsFS) Stat(filename string) (fs.FileInfo, error) { if path.Clean(filename) == Separator { diff --git a/src/host/vfs/resolver.go b/src/host/vfs/resolver.go index 9bab3b8..919fe50 100644 --- a/src/host/vfs/resolver.go +++ b/src/host/vfs/resolver.go @@ -72,6 +72,19 @@ func (r *ResolveFS) Stat(filename string) (fs.FileInfo, error) { return r.rootFS.Stat(fsPath) } +// Unlink implements Filesystem. +func (r *ResolveFS) Unlink(filename string) error { + fsPath, nestedFs, nestedFsPath, err := r.resolver.resolvePath(filename, r.rootFS.Open) + if err != nil { + return err + } + if nestedFs != nil { + return nestedFs.Unlink(nestedFsPath) + } + + return r.rootFS.Unlink(fsPath) +} + var _ Filesystem = &ResolveFS{} type FsFactory func(f File) (Filesystem, error) diff --git a/src/host/vfs/resolver_test.go b/src/host/vfs/resolver_test.go index ce2af5b..aa6882e 100644 --- a/src/host/vfs/resolver_test.go +++ b/src/host/vfs/resolver_test.go @@ -52,6 +52,10 @@ func (d *DummyFs) Open(filename string) (File, error) { return &Dummy{}, nil } +func (d *DummyFs) Unlink(filename string) error { + return ErrNotImplemented +} + func (d *DummyFs) ReadDir(path string) ([]fs.DirEntry, error) { if path == "/dir/here" { return []fs.DirEntry{ diff --git a/src/host/vfs/torrent.go b/src/host/vfs/torrent.go index e4d9247..ddb248c 100644 --- a/src/host/vfs/torrent.go +++ b/src/host/vfs/torrent.go @@ -5,19 +5,23 @@ import ( "io" "io/fs" "path" + "slices" "sync" "time" + "git.kmsign.ru/royalcat/tstor/src/host/repository" "git.kmsign.ru/royalcat/tstor/src/iio" "github.com/anacrolix/missinggo/v2" "github.com/anacrolix/torrent" + "golang.org/x/exp/maps" ) var _ Filesystem = &TorrentFs{} type TorrentFs struct { - mu sync.Mutex - t *torrent.Torrent + mu sync.Mutex + t *torrent.Torrent + rep repository.TorrentMetaRepository readTimeout int @@ -27,22 +31,34 @@ type TorrentFs struct { resolver *resolver } -func NewTorrentFs(t *torrent.Torrent, readTimeout int) *TorrentFs { +func NewTorrentFs(t *torrent.Torrent, rep repository.TorrentMetaRepository, readTimeout int) *TorrentFs { return &TorrentFs{ t: t, + rep: rep, readTimeout: readTimeout, resolver: newResolver(ArchiveFactories), } } -func (fs *TorrentFs) files() map[string]*torrentFile { +func (fs *TorrentFs) files() (map[string]*torrentFile, error) { if fs.filesCache == nil { fs.mu.Lock() <-fs.t.GotInfo() files := fs.t.Files() + + excludedFiles, err := fs.rep.ExcludedFiles(fs.t.InfoHash()) + if err != nil { + return nil, err + } + fs.filesCache = make(map[string]*torrentFile) for _, file := range files { p := AbsPath(file.Path()) + + if slices.Contains(excludedFiles, p) { + continue + } + fs.filesCache[p] = &torrentFile{ name: path.Base(p), readerFunc: file.NewReader, @@ -53,16 +69,24 @@ func (fs *TorrentFs) files() map[string]*torrentFile { fs.mu.Unlock() } - return fs.filesCache + return fs.filesCache, nil } func (fs *TorrentFs) rawOpen(path string) (File, error) { - file, err := getFile(fs.files(), path) + files, err := fs.files() + if err != nil { + return nil, err + } + file, err := getFile(files, path) return file, err } func (fs *TorrentFs) rawStat(filename string) (fs.FileInfo, error) { - file, err := getFile(fs.files(), filename) + files, err := fs.files() + if err != nil { + return nil, err + } + file, err := getFile(files, filename) if err != nil { return nil, err } @@ -111,14 +135,30 @@ func (fs *TorrentFs) ReadDir(name string) ([]fs.DirEntry, error) { if nestedFs != nil { return nestedFs.ReadDir(nestedFsPath) } + files, err := fs.files() + if err != nil { + return nil, err + } - return listDirFromFiles(fs.files(), fsPath) + return listDirFromFiles(files, fsPath) } func (fs *TorrentFs) Unlink(name string) error { - file := fs.t.Files()[0] - file.SetPriority(torrent.PiecePriorityNone) - return nil + fs.mu.Lock() + defer fs.mu.Unlock() + + files, err := fs.files() + if err != nil { + return err + } + file := AbsPath(name) + + if !slices.Contains(maps.Keys(files), file) { + return ErrNotExist + } + fs.filesCache = nil + + return fs.rep.ExcludeFile(fs.t.InfoHash(), file) } type reader interface { diff --git a/src/mounts/webdav/fs.go b/src/mounts/webdav/fs.go index 9ff2561..e6411ab 100644 --- a/src/mounts/webdav/fs.go +++ b/src/mounts/webdav/fs.go @@ -48,7 +48,7 @@ func (wd *WebDAV) Mkdir(ctx context.Context, name string, perm fs.FileMode) erro } func (wd *WebDAV) RemoveAll(ctx context.Context, name string) error { - return webdav.ErrNotImplemented + return wd.fs.Unlink(name) } func (wd *WebDAV) Rename(ctx context.Context, oldName, newName string) error {