From 80884aca6ab59e60457fa5a32671266f24fbc9e6 Mon Sep 17 00:00:00 2001 From: royalcat Date: Tue, 24 Sep 2024 16:26:15 +0300 Subject: [PATCH] fix --- go.mod | 20 +--- go.sum | 46 +------- pkg/qbittorrent/torrent.go | 33 ++++-- src/delivery/web.go | 27 ----- src/export/nfs/wrapper.go | 4 + src/sources/qbittorrent/client.go | 64 +++++++++- src/sources/qbittorrent/daemon.go | 33 +++--- src/sources/qbittorrent/fs.go | 189 +++++++++++++++++++++++++----- src/sources/storage.go | 10 +- src/vfs/archive.go | 9 +- src/vfs/hash.go | 3 + src/vfs/log.go | 10 +- src/vfs/resolver.go | 2 +- src/vfs/utils.go | 4 + 14 files changed, 299 insertions(+), 155 deletions(-) delete mode 100644 src/delivery/web.go diff --git a/go.mod b/go.mod index 8517af4..17c1a9f 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/bodgit/sevenzip v1.5.1 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dustin/go-humanize v1.0.1 - github.com/gin-gonic/gin v1.9.1 github.com/go-git/go-billy/v5 v5.5.0 github.com/gofrs/uuid/v5 v5.1.0 github.com/google/go-github/v63 v63.0.0 @@ -34,6 +33,7 @@ require ( github.com/prometheus/client_golang v1.19.1 github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 github.com/ravilushqa/otelgqlgen v0.15.0 + github.com/royalcat/btrgo v0.0.0-20240318160410-19bd27154450 github.com/royalcat/ctxio v0.0.0-20240602060200-590d464c39be github.com/royalcat/ctxprogress v0.0.0-20240614113930-3cc5bb935bff github.com/royalcat/kv v0.0.0-20240707205211-fedd4883af85 @@ -86,13 +86,9 @@ require ( github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/windows v1.0.1 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect - github.com/bytedance/sonic v1.11.9 // indirect - github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/cyphar/filepath-securejoin v0.2.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -100,17 +96,11 @@ require ( github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-llsqlite/adapter v0.1.0 // indirect github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect - github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/glog v1.2.1 // indirect @@ -126,27 +116,22 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.4.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/labstack/gommon v0.4.2 // indirect - github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.6 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pion/datachannel v1.5.6 // indirect github.com/pion/dtls/v2 v2.2.11 // indirect @@ -179,8 +164,6 @@ require ( github.com/sosodev/duration v1.3.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tidwall/btree v1.7.0 // indirect - github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.12 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect @@ -192,7 +175,6 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/go.sum b/go.sum index b7203a6..d1076ad 100644 --- a/go.sum +++ b/go.sum @@ -136,11 +136,6 @@ github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2w github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= -github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg= -github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= -github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -153,10 +148,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -200,12 +191,6 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= -github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -231,20 +216,10 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= -github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid/v5 v5.1.0 h1:S5rqVKIigghZTCBKPCw0Y+bXkn26K3TB5mvQq2Ix8dk= github.com/gofrs/uuid/v5 v5.1.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= @@ -356,8 +331,6 @@ github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -384,7 +357,6 @@ github.com/knadh/koanf/providers/structs v0.1.0 h1:wJRteCNn1qvLtE5h8KQBvLJovidSd github.com/knadh/koanf/providers/structs v0.1.0/go.mod h1:sw2YZ3txUcqA3Z27gPlmmBzWn1h8Nt9O6EP/91MkcWE= github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -402,8 +374,6 @@ github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+k github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= -github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= -github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattetti/filebuffer v1.0.1 h1:gG7pyfnSIZCxdoKq+cPa8T0hhYtD9NxCdI4D7PTjRLM= github.com/mattetti/filebuffer v1.0.1/go.mod h1:YdMURNDOttIiruleeVr6f56OrMc+MydEnTcXwtkxNVs= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -422,12 +392,9 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -451,8 +418,6 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -557,6 +522,8 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/royalcat/btrgo v0.0.0-20240318160410-19bd27154450 h1:AZyZxXZLniAR0DaZhTS4RVcHtOvYMW8IunplqC9A0mk= +github.com/royalcat/btrgo v0.0.0-20240318160410-19bd27154450/go.mod h1:m3TPa9l/wMKpm/7WHrMs3dSFUxo7kLHaI8ap+SFGYhQ= github.com/royalcat/ctxio v0.0.0-20240602060200-590d464c39be h1:Ui+Imq1Vk26rfpkLUsgvVdYO/UOJkzDyPzESfrTqWfM= github.com/royalcat/ctxio v0.0.0-20240602060200-590d464c39be/go.mod h1:NFNp3OsEMUPYj5LZUFDiyDt+2E6gR/g8JLd0k+y8XWI= github.com/royalcat/ctxprogress v0.0.0-20240614113930-3cc5bb935bff h1:KlZaOEZYhCzyNYIp0LcE7MNR2Ar0PJS3eJU6A5mMTpk= @@ -625,10 +592,6 @@ github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EU github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= -github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -697,9 +660,6 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= -golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1016,9 +976,7 @@ modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= zombiezen.com/go/sqlite v1.3.0 h1:98g1gnCm+CNz6AuQHu0gqyw7gR2WU3O3PJufDOStpUs= diff --git a/pkg/qbittorrent/torrent.go b/pkg/qbittorrent/torrent.go index 0777157..f7c62c2 100644 --- a/pkg/qbittorrent/torrent.go +++ b/pkg/qbittorrent/torrent.go @@ -62,7 +62,7 @@ type Torrent interface { // MinPriority minimal torrent priority MinPriority(ctx context.Context, hashes []string) error // SetFilePriority set file priority - SetFilePriority(ctx context.Context, hash string, id string, priority int) error + SetFilePriority(ctx context.Context, hash string, id int, priority Priority) error // GetDownloadLimit get torrent download limit GetDownloadLimit(ctx context.Context, hashes []string) (map[string]int, error) // SetDownloadLimit set torrent download limit, limit in bytes per second, if no limit please set value zero @@ -273,15 +273,24 @@ type TorrentWebSeed struct { URL string `json:"url"` } +type Priority int + +const ( + PriorityDoNotDownload Priority = 0 + PriorityNormal Priority = 1 + PriorityHigh Priority = 6 + PriorityMax Priority = 7 +) + type TorrentContent struct { - Availability float64 `json:"availability,omitempty"` - Index int `json:"index,omitempty"` - IsSeed bool `json:"is_seed,omitempty"` - Name string `json:"name,omitempty"` - PieceRange []int `json:"piece_range,omitempty"` - Priority int `json:"priority,omitempty"` - Progress float64 `json:"progress,omitempty"` - Size int64 `json:"size,omitempty"` + Availability float64 `json:"availability,omitempty"` + Index int `json:"index,omitempty"` + IsSeed bool `json:"is_seed,omitempty"` + Name string `json:"name,omitempty"` + PieceRange []int `json:"piece_range,omitempty"` + Priority Priority `json:"priority,omitempty"` + Progress float64 `json:"progress,omitempty"` + Size int64 `json:"size,omitempty"` } type TorrentAddFileMetadata struct { @@ -929,14 +938,14 @@ func (c *client) MinPriority(ctx context.Context, hashes []string) error { return nil } -func (c *client) SetFilePriority(ctx context.Context, hash string, id string, priority int) error { +func (c *client) SetFilePriority(ctx context.Context, hash string, id int, priority Priority) error { ctx, span := trace.Start(ctx, "qbittorrent.Torrent.SetFilePriority") defer span.End() var formData = url.Values{} formData.Add("hash", hash) - formData.Add("id", id) - formData.Add("priority", strconv.Itoa(priority)) + formData.Add("id", strconv.Itoa(id)) + formData.Add("priority", strconv.Itoa(int(priority))) var apiUrl = fmt.Sprintf("%s/api/v2/torrents/filePrio", c.config.Address) result, err := c.doRequest(ctx, &requestData{ url: apiUrl, diff --git a/src/delivery/web.go b/src/delivery/web.go deleted file mode 100644 index 9bcc202..0000000 --- a/src/delivery/web.go +++ /dev/null @@ -1,27 +0,0 @@ -package delivery - -import ( - "net/http" - - "github.com/gin-gonic/gin" -) - -var indexHandler = func(c *gin.Context) { - c.HTML(http.StatusOK, "index.html", nil) -} - -// var routesHandler = func(ss *torrent.Stats) gin.HandlerFunc { -// return func(c *gin.Context) { -// c.HTML(http.StatusOK, "routes.html", ss.RoutesStats()) -// } -// } - -var logsHandler = func(c *gin.Context) { - c.HTML(http.StatusOK, "logs.html", nil) -} - -var serversFoldersHandler = func() gin.HandlerFunc { - return func(c *gin.Context) { - c.HTML(http.StatusOK, "servers.html", nil) - } -} diff --git a/src/export/nfs/wrapper.go b/src/export/nfs/wrapper.go index cf19f5d..cebf09e 100644 --- a/src/export/nfs/wrapper.go +++ b/src/export/nfs/wrapper.go @@ -86,6 +86,10 @@ func (bfs *fsWrapper) ReadDir(ctx context.Context, path string) ([]fs.FileInfo, out := make([]fs.FileInfo, 0, len(ffs)) for _, v := range ffs { + if v == nil { + continue + } + if info, ok := v.(fs.FileInfo); ok { out = append(out, info) } else { diff --git a/src/sources/qbittorrent/client.go b/src/sources/qbittorrent/client.go index 96a772d..8263ca9 100644 --- a/src/sources/qbittorrent/client.go +++ b/src/sources/qbittorrent/client.go @@ -7,24 +7,75 @@ import ( "time" "git.kmsign.ru/royalcat/tstor/pkg/qbittorrent" + "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/royalcat/btrgo/btrsync" ) type cacheClient struct { qb qbittorrent.Client + + propertiesCache *expirable.LRU[string, qbittorrent.TorrentProperties] + torrentsCache *expirable.LRU[string, qbittorrent.TorrentInfo] + + pieceCache btrsync.MapOf[pieceKey, int] +} + +type pieceKey struct { + hash string + index int } func wrapClient(qb qbittorrent.Client) *cacheClient { - return &cacheClient{qb: qb} + + const ( + cacheSize = 5000 + cacheTTL = time.Minute + ) + + return &cacheClient{ + qb: qb, + propertiesCache: expirable.NewLRU[string, qbittorrent.TorrentProperties](cacheSize, nil, cacheTTL), + torrentsCache: expirable.NewLRU[string, qbittorrent.TorrentInfo](cacheSize, nil, cacheTTL), + pieceCache: btrsync.MapOf[pieceKey, int]{}, + } } -var errNotFound = fmt.Errorf("not found") +func (f *cacheClient) getInfo(ctx context.Context, hash string) (*qbittorrent.TorrentInfo, error) { + if v, ok := f.torrentsCache.Get(hash); ok { + return &v, nil + } + + infos, err := f.qb.Torrent().GetTorrents(ctx, &qbittorrent.TorrentOption{ + Hashes: []string{hash}, + }) + if err != nil { + return nil, fmt.Errorf("error to check torrent existence: %w", err) + } + + if len(infos) == 0 { + return nil, nil + } + + if len(infos) > 1 { + return nil, fmt.Errorf("multiple torrents with the same hash") + } + + f.torrentsCache.Add(hash, *infos[0]) + + return infos[0], nil +} func (f *cacheClient) getProperties(ctx context.Context, hash string) (*qbittorrent.TorrentProperties, error) { + if v, ok := f.propertiesCache.Get(hash); ok { + return &v, nil + } + info, err := f.qb.Torrent().GetProperties(ctx, hash) if err != nil { return nil, err } + f.propertiesCache.Add(hash, *info) return info, nil } @@ -53,11 +104,20 @@ func (f *cacheClient) getContent(ctx context.Context, hash string, contentIndex } func (f *cacheClient) isPieceComplete(ctx context.Context, hash string, pieceIndex int) (bool, error) { + cachedPieceState, ok := f.pieceCache.Load(pieceKey{hash: hash, index: pieceIndex}) + if ok && cachedPieceState == 2 { + return true, nil + } + completion, err := f.qb.Torrent().GetPiecesStates(ctx, hash) if err != nil { return false, err } + for i, v := range completion { + f.pieceCache.Store(pieceKey{hash: hash, index: i}, v) + } + if completion[pieceIndex] == 2 { return true, nil } diff --git a/src/sources/qbittorrent/daemon.go b/src/sources/qbittorrent/daemon.go index 41fe7fa..777fe22 100644 --- a/src/sources/qbittorrent/daemon.go +++ b/src/sources/qbittorrent/daemon.go @@ -20,8 +20,11 @@ import ( "github.com/anacrolix/torrent/types/infohash" infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "github.com/royalcat/ctxio" + "go.opentelemetry.io/otel" ) +var trace = otel.Tracer("git.kmsign.ru/royalcat/tstor/src/sources/qbittorrent") + type Daemon struct { proc *os.Process qb qbittorrent.Client @@ -130,11 +133,14 @@ func (d *Daemon) Close(ctx context.Context) error { return nil } -func (d *Daemon) torrentPath(ih infohash.T) (string, error) { - return filepath.Abs(path.Join(d.dataDir, ih.HexString())) +func torrentDataPath(dataDir string, ih string) (string, error) { + return filepath.Abs(path.Join(dataDir, ih)) } -func (fs *Daemon) TorrentFS(ctx context.Context, file vfs.File) (vfs.Filesystem, error) { +func (fs *Daemon) GetTorrentFS(ctx context.Context, file vfs.File) (vfs.Filesystem, error) { + ctx, span := trace.Start(ctx, "GetTorrentFS") + defer span.End() + log := fs.log.With(slog.String("file", file.Name())) ih, err := readInfoHash(ctx, file) @@ -143,7 +149,7 @@ func (fs *Daemon) TorrentFS(ctx context.Context, file vfs.File) (vfs.Filesystem, } log = log.With(slog.String("infohash", ih.HexString())) - torrentPath, err := fs.torrentPath(ih) + torrentPath, err := torrentDataPath(fs.dataDir, ih.HexString()) if err != nil { return nil, fmt.Errorf("error getting torrent path: %w", err) } @@ -160,18 +166,18 @@ func (fs *Daemon) TorrentFS(ctx context.Context, file vfs.File) (vfs.Filesystem, } func (d *Daemon) syncTorrentState(ctx context.Context, file vfs.File, ih metainfo.Hash, torrentPath string) error { + ctx, span := trace.Start(ctx, "syncTorrentState") + defer span.End() log := d.log.With(slog.String("file", file.Name()), slog.String("infohash", ih.HexString())) - existing, err := d.qb.Torrent().GetTorrents(ctx, &qbittorrent.TorrentOption{ - Hashes: []string{ih.HexString()}, - }) + info, err := d.client.getInfo(ctx, ih.HexString()) if err != nil { - return fmt.Errorf("error to check torrent existence: %w", err) + return err } log = log.With(slog.String("torrentPath", torrentPath)) - if len(existing) == 0 { + if info == nil { _, err := file.Seek(0, io.SeekStart) if err != nil { return err @@ -195,7 +201,7 @@ func (d *Daemon) syncTorrentState(ctx context.Context, file vfs.File, ih metainf return err } for { - _, err := d.qb.Torrent().GetProperties(ctx, ih.HexString()) + _, err := d.client.getProperties(ctx, ih.HexString()) if err == nil { break } @@ -211,9 +217,9 @@ func (d *Daemon) syncTorrentState(ctx context.Context, file vfs.File, ih metainf } return nil - } else if len(existing) == 1 { + } else { // info := existing[0] - props, err := d.qb.Torrent().GetProperties(ctx, ih.HexString()) + props, err := d.client.getProperties(ctx, ih.HexString()) if err != nil { return err } @@ -228,9 +234,6 @@ func (d *Daemon) syncTorrentState(ctx context.Context, file vfs.File, ih metainf return nil } - - return fmt.Errorf("multiple torrents with the same infohash") - } // TODO caching diff --git a/src/sources/qbittorrent/fs.go b/src/sources/qbittorrent/fs.go index 7df8f8c..f2517ce 100644 --- a/src/sources/qbittorrent/fs.go +++ b/src/sources/qbittorrent/fs.go @@ -2,44 +2,62 @@ package qbittorrent import ( "context" + "errors" "fmt" "io" "io/fs" + "log/slog" "os" "path" "strings" + "sync" "time" "git.kmsign.ru/royalcat/tstor/pkg/qbittorrent" + "git.kmsign.ru/royalcat/tstor/pkg/rlog" "git.kmsign.ru/royalcat/tstor/src/vfs" ) type FS struct { + mu sync.Mutex client *cacheClient name string hash string - dataDir string + dataDir string // directory where torrent files are stored - content map[string]*qbittorrent.TorrentContent - files map[string]fs.FileInfo + entries map[string]fileEntry + + log *rlog.Logger vfs.FilesystemPrototype } +type fileEntry struct { + fs.FileInfo + Content *qbittorrent.TorrentContent +} + var _ vfs.Filesystem = (*FS)(nil) func newTorrentFS(ctx context.Context, client *cacheClient, name string, hash string, dataDir string) (*FS, error) { + ctx, span := trace.Start(ctx, "newTorrentFS") + defer span.End() + cnts, err := client.listContent(ctx, hash) if err != nil { return nil, fmt.Errorf("failed to list content for hash %s: %w", hash, err) } - content := make(map[string]*qbittorrent.TorrentContent, len(cnts)) - files := make(map[string]fs.FileInfo, len(cnts)) + entries := make(map[string]fileEntry, len(cnts)) for _, cnt := range cnts { - path := vfs.AbsPath(cnt.Name) - files[path] = vfs.NewFileInfo(cnt.Name, cnt.Size) - content[path] = cnt + if cnt.Priority == qbittorrent.PriorityDoNotDownload { + continue + } + + entries[vfs.AbsPath(cnt.Name)] = fileEntry{ + Content: cnt, + FileInfo: vfs.NewFileInfo(cnt.Name, cnt.Size), + } } return &FS{ @@ -49,8 +67,9 @@ func newTorrentFS(ctx context.Context, client *cacheClient, name string, hash st dataDir: dataDir, - content: content, - files: files, + entries: entries, + + log: rlog.Component("qbittorrent", "fs"), FilesystemPrototype: vfs.FilesystemPrototype(name), }, nil @@ -62,12 +81,11 @@ func (f *FS) Open(ctx context.Context, name string) (vfs.File, error) { return vfs.NewDirFile(name), nil } - cnt, ok := f.content[name] - if ok { - return openFile(ctx, f.client, f.dataDir, f.hash, cnt) + if entry, ok := f.entries[name]; ok { + return openFile(ctx, f.client, f.dataDir, f.hash, entry.Content) } - for p := range f.content { + for p := range f.entries { if strings.HasPrefix(p, name) { return vfs.NewDirFile(name), nil } @@ -77,22 +95,79 @@ func (f *FS) Open(ctx context.Context, name string) (vfs.File, error) { } // ReadDir implements vfs.Filesystem. -func (fs *FS) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { - return vfs.ListDirFromInfo(fs.files, name) +func (f *FS) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { + infos := make(map[string]fs.FileInfo, len(f.entries)) + for k, v := range f.entries { + infos[k] = v.FileInfo + } + + return vfs.ListDirFromInfo(infos, name) } // Stat implements vfs.Filesystem. func (f *FS) Stat(ctx context.Context, name string) (fs.FileInfo, error) { - info, ok := f.files[name] - if !ok { - return nil, vfs.ErrNotExist + name = vfs.AbsPath(path.Clean(name)) + + if vfs.IsRoot(name) { + return vfs.NewDirInfo(f.name), nil } - return info, nil + + if entry, ok := f.entries[name]; ok { + return entry.FileInfo, nil + } + + for p := range f.entries { + if strings.HasPrefix(p, name) { + return vfs.NewDirInfo(name), nil + } + } + + return nil, vfs.ErrNotExist } // Unlink implements vfs.Filesystem. func (f *FS) Unlink(ctx context.Context, filename string) error { - return vfs.ErrNotImplemented + filename = vfs.AbsPath(path.Clean(filename)) + + // we cannot delete a torrent itself, cause it will be added on next source scan and all delited files will be restored + + if entry, ok := f.entries[filename]; ok { + return f.removeFile(ctx, f.hash, entry.Content) + } + + for p, entry := range f.entries { + if strings.HasPrefix(p, filename) { + return f.removeFile(ctx, f.hash, entry.Content) + } + } + + return vfs.ErrNotExist +} + +func (f *FS) removeFile(ctx context.Context, hash string, content *qbittorrent.TorrentContent) error { + log := f.log.With(slog.String("hash", hash), slog.String("file", content.Name)) + + f.mu.Lock() + defer f.mu.Unlock() + + fpath := vfs.AbsPath(content.Name) + + if _, ok := f.entries[fpath]; !ok { + return fmt.Errorf("file %s is does not found", fpath) + } + delete(f.entries, fpath) + + err := f.client.qb.Torrent().SetFilePriority(ctx, f.hash, content.Index, qbittorrent.PriorityDoNotDownload) + if err != nil { + return fmt.Errorf("failed to set priority for torrent %s for file %s: %w", hash, content.Name, err) + } + + err = os.Remove(path.Join(f.dataDir, vfs.RelPath(content.Name))) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Warn(ctx, "failed to remove file", rlog.Error(err)) + return fmt.Errorf("failed to remove file %s: %w", content.Name, err) + } + return nil } func openFile(ctx context.Context, client *cacheClient, torrentDir string, hash string, content *qbittorrent.TorrentContent) (*File, error) { @@ -158,11 +233,73 @@ func (f *File) Name() string { return path.Base(f.filePath) } +func (f *File) canExpectSoon(ctx context.Context) (bool, error) { + info, err := f.client.getInfo(ctx, f.hash) + if err != nil { + return false, err + } + + return info.Completed == info.Size || info.State == qbittorrent.TorrentStateCheckingUP || info.State == qbittorrent.TorrentStateDownloading || info.State == qbittorrent.TorrentStateForcedDL, nil +} + +func (f *File) isRangeComplete(ctx context.Context, offset int64, size int) (bool, error) { + startPieceIndex := int(offset / int64(f.pieceSize)) + pieceCount := (size + f.pieceSize - 1) / f.pieceSize // rouding up + + for i := range pieceCount { + ok, err := f.client.isPieceComplete(ctx, f.hash, startPieceIndex+i) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + } + + return true, nil +} + +func (f *File) waitPieceAvailable(ctx context.Context, offset int64, size int) error { + complete, err := f.isRangeComplete(ctx, offset, size) + if err != nil { + return err + } + if complete { + return nil + } + + canExpectSoon, err := f.canExpectSoon(ctx) + if err != nil { + return err + } + if !canExpectSoon { + return fmt.Errorf("torrent is not downloading") + } + + const checkingInterval = 1 * time.Second + + ticker := time.NewTicker(checkingInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + complete, err := f.isRangeComplete(ctx, offset, size) + if err != nil { + return err + } + if complete { + return nil + } + } + } +} + // Read implements vfs.File. func (f *File) Read(ctx context.Context, p []byte) (n int, err error) { - pieceIndex := int(f.offset / int64(f.pieceSize)) - err = f.client.waitPieceToComplete(ctx, f.hash, pieceIndex) - if err != nil { + if err := f.waitPieceAvailable(ctx, f.offset, len(p)); err != nil { return 0, err } @@ -178,9 +315,7 @@ func (f *File) Read(ctx context.Context, p []byte) (n int, err error) { // ReadAt implements vfs.File. func (f *File) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) { - pieceIndex := int(off / int64(f.pieceSize)) - err = f.client.waitPieceToComplete(ctx, f.hash, pieceIndex) - if err != nil { + if err := f.waitPieceAvailable(ctx, f.offset, len(p)); err != nil { return 0, err } diff --git a/src/sources/storage.go b/src/sources/storage.go index 1420c7d..fe3b780 100644 --- a/src/sources/storage.go +++ b/src/sources/storage.go @@ -1,6 +1,8 @@ package sources import ( + "context" + "git.kmsign.ru/royalcat/tstor/src/sources/qbittorrent" "git.kmsign.ru/royalcat/tstor/src/sources/ytdlp" "git.kmsign.ru/royalcat/tstor/src/vfs" @@ -8,7 +10,13 @@ import ( func NewHostedFS(sourceFS vfs.Filesystem, tsrv *qbittorrent.Daemon, ytdlpsrv *ytdlp.Daemon) vfs.Filesystem { factories := map[string]vfs.FsFactory{ - ".torrent": tsrv.TorrentFS, + ".torrent": func(ctx context.Context, f vfs.File) (vfs.Filesystem, error) { + tfs, err := tsrv.GetTorrentFS(ctx, f) + if err != nil { + return nil, err + } + return vfs.NewResolveFS(tfs, vfs.ArchiveFactories), nil + }, ".ts-ytdlp": ytdlpsrv.BuildFS, } diff --git a/src/vfs/archive.go b/src/vfs/archive.go index 47e69ec..c80c8b6 100644 --- a/src/vfs/archive.go +++ b/src/vfs/archive.go @@ -282,16 +282,17 @@ func (d *archiveFile) ReadAt(ctx context.Context, p []byte, off int64) (n int, e } n, err = d.buffer.ReadAt(p, off) if err != nil && err != io.EOF { - return n, fmt.Errorf("failed to read from buffer: %w", err) + return n, fmt.Errorf("failed to readAt from buffer: %w", err) } return n, err } func (d *archiveFile) Close(ctx context.Context) error { - d.m.Lock() - defer d.m.Unlock() + return nil + // d.m.Lock() + // defer d.m.Unlock() - return d.buffer.Close() + // return d.buffer.Close() } type archiveFileReaderFactory func(ctx context.Context) (ctxio.ReadCloser, error) diff --git a/src/vfs/hash.go b/src/vfs/hash.go index 3bee0c2..092f373 100644 --- a/src/vfs/hash.go +++ b/src/vfs/hash.go @@ -15,6 +15,9 @@ var ErrOsHashLen = errors.New("oshash: buffer length must be a multiple of 8") type Hash string func FileHash(ctx context.Context, f File) (Hash, error) { + ctx, span := tracer.Start(ctx, "FileHash") + defer span.End() + _, err := f.Seek(0, io.SeekStart) if err != nil { return "", fmt.Errorf("error seeking file: %w", err) diff --git a/src/vfs/log.go b/src/vfs/log.go index 3f203bc..4491c5b 100644 --- a/src/vfs/log.go +++ b/src/vfs/log.go @@ -2,7 +2,9 @@ package vfs import ( "context" + "errors" "fmt" + "io" "io/fs" "log/slog" "reflect" @@ -34,7 +36,7 @@ type LogFS struct { } func isLoggableError(err error) bool { - return err != nil // && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, io.EOF) + return err != nil && !errors.Is(err, io.EOF) // && !errors.Is(err, fs.ErrNotExist) } var _ Filesystem = (*LogFS)(nil) @@ -169,10 +171,11 @@ func (lfs *LogFS) Stat(ctx context.Context, filename string) (info fs.FileInfo, } span.End() }() + log := lfs.log.With(slog.String("filename", filename)) info, err = lfs.fs.Stat(ctx, filename) if isLoggableError(err) { - lfs.log.Error(ctx, "Failed to stat", rlog.Error(err)) + log.Error(ctx, "Failed to stat", rlog.Error(err)) } return info, err } @@ -190,10 +193,11 @@ func (fs *LogFS) Unlink(ctx context.Context, filename string) (err error) { } span.End() }() + log := fs.log.With(slog.String("filename", filename)) err = fs.fs.Unlink(ctx, filename) if isLoggableError(err) { - fs.log.Error(ctx, "Failed to stat", rlog.Error(err)) + log.Error(ctx, "Failed to unlink", rlog.Error(err)) } return err } diff --git a/src/vfs/resolver.go b/src/vfs/resolver.go index 3c697b6..5b87cdd 100644 --- a/src/vfs/resolver.go +++ b/src/vfs/resolver.go @@ -120,7 +120,7 @@ func (r *ResolverFS) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, e return nil, err } nestedfs, err := r.resolver.nestedFs(ctx, filepath, file) - if errors.Is(err, context.DeadlineExceeded) { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { return nil, err } if err != nil { diff --git a/src/vfs/utils.go b/src/vfs/utils.go index aed4003..739cbad 100644 --- a/src/vfs/utils.go +++ b/src/vfs/utils.go @@ -24,6 +24,10 @@ func trimRelPath(p, t string) string { // return path.Clean(Separator + strings.ReplaceAll(p, "\\", "/")) // } +func RelPath(p string) string { + return strings.TrimLeft(p, Separator) +} + func AbsPath(p string) string { if p == "" || p[0] != '/' { return Separator + p