From 5591f145a953c02df32085f92760ced42cc8f40d Mon Sep 17 00:00:00 2001 From: royalcat Date: Wed, 17 Apr 2024 11:36:14 +0300 Subject: [PATCH] no cache archive reader --- cmd/tstor/main.go | 31 ++++--- go.mod | 44 +++++----- go.sum | 90 +++++++++---------- pkg/ctxio/disk.go | 70 +++++++++++++++ pkg/ctxio/io.go | 6 +- pkg/ctxio/reader.go | 41 +++++++++ pkg/ctxio/readerat.go | 47 ++++++++++ pkg/ctxio/seeker.go | 2 +- pkg/rlog/rlog.go | 116 +++++++++++++++++++------ src/export/nfs/kvhandler.go | 108 ++++++++++++++--------- src/host/datastorage/storage.go | 5 +- src/host/service/service.go | 54 +++++++----- src/host/vfs/archive.go | 147 ++++++++++++++++++++------------ src/host/vfs/log.go | 32 ++++--- src/host/vfs/resolver.go | 8 +- src/telemetry/setup.go | 50 +++++++---- 16 files changed, 579 insertions(+), 272 deletions(-) create mode 100644 pkg/ctxio/disk.go create mode 100644 pkg/ctxio/readerat.go diff --git a/cmd/tstor/main.go b/cmd/tstor/main.go index b430181..d039567 100644 --- a/cmd/tstor/main.go +++ b/cmd/tstor/main.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "log/slog" "net" nethttp "net/http" @@ -79,12 +80,13 @@ func run(configPath string) error { defer client.Shutdown(ctx) } - log := rlog.ComponentLog("run") + log := rlog.Component("run") + ctx := context.Background() // TODO make optional err = syscall.Setpriority(syscall.PRIO_PGRP, 0, 19) if err != nil { - log.Error("set priority failed", "error", err) + log.Error(ctx, "set priority failed", rlog.Error(err)) } if err := os.MkdirAll(conf.TorrentClient.MetadataFolder, 0744); err != nil { @@ -186,18 +188,19 @@ func run(configPath string) error { if conf.Mounts.WebDAV.Enabled { go func() { if err := webdav.NewWebDAVServer(sfs, conf.Mounts.WebDAV.Port, conf.Mounts.WebDAV.User, conf.Mounts.WebDAV.Pass); err != nil { - log.Error("error starting webDAV", "error", err) + log.Error(ctx, "error starting webDAV", rlog.Error(err)) } - log.Warn("webDAV configuration not found!") + log.Warn(ctx, "webDAV configuration not found!") }() } if conf.Mounts.HttpFs.Enabled { go func() { httpfs := httpfs.NewHTTPFS(sfs) - err = nethttp.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port), nethttp.FileServer(httpfs)) + addr := fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port) + err = nethttp.ListenAndServe(addr, nethttp.FileServer(httpfs)) if err != nil { - log.Error("error starting HTTPFS", "error", err) + log.Error(ctx, "error starting HTTPFS", rlog.Error(err)) } // r := gin.New() @@ -206,7 +209,7 @@ func run(configPath string) error { // c.FileFromFS(path, httpfs) // }) - log.Info("starting HTTPFS", "host", fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port)) + log.Info(ctx, "starting HTTPFS", slog.String("address", addr)) // if err := r.Run(fmt.Sprintf("0.0.0.0:%d", conf.Mounts.HttpFs.Port)); err != nil { // log.Error().Err(err).Msg("error starting HTTPFS") // } @@ -215,22 +218,22 @@ func run(configPath string) error { if conf.Mounts.NFS.Enabled { go func() { - log := log.With("component", "NFS") + log := log.WithComponent("NFS") listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", conf.Mounts.NFS.Port)) if err != nil { - log.Error("failed to start TCP listener", err) + log.Error(ctx, "failed to start TCP listener", rlog.Error(err)) return } - log.Info("starting NFS server", "host", listener.Addr().String()) + log.Info(ctx, "starting NFS server", slog.String("address", listener.Addr().String())) handler, err := nfs.NewNFSv3Handler(sfs) if err != nil { - log.Error("failed to create NFS handler", "error", err) + log.Error(ctx, "failed to create NFS handler", rlog.Error(err)) return } err = wnfs.Serve(listener, handler) if err != nil { - log.Error("error serving nfs", "error", err) + log.Error(ctx, "error serving nfs", rlog.Error(err)) return } }() @@ -239,7 +242,7 @@ func run(configPath string) error { go func() { err := webdav.NewDirServer(conf.SourceDir, 36912, conf.Mounts.WebDAV.User, conf.Mounts.WebDAV.Pass) if err != nil { - log.Error("error starting webDAV", "error", err) + log.Error(ctx, "error starting webDAV", rlog.Error(err)) } }() @@ -248,7 +251,7 @@ func run(configPath string) error { err := delivery.New(nil, service.NewStats(), ts, sfs, logFilename, conf) if err != nil { - log.Error("error initializing HTTP server", "error", err) + log.Error(ctx, "error initializing HTTP server", rlog.Error(err)) } }() diff --git a/go.mod b/go.mod index 5ea10df..7d9e013 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/go-git/go-billy/v5 v5.5.0 github.com/gofrs/uuid/v5 v5.0.0 - github.com/google/uuid v1.5.0 + github.com/google/uuid v1.6.0 github.com/grafana/otel-profiling-go v0.5.1 github.com/grafana/pyroscope-go v1.1.1 github.com/hashicorp/go-multierror v1.1.1 @@ -37,21 +37,21 @@ require ( github.com/samber/slog-multi v1.0.2 github.com/samber/slog-zerolog v1.0.0 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.0 github.com/vektah/gqlparser/v2 v2.5.11 github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 github.com/willscott/memphis v0.0.0-20210922141505-529d4987ab7e - go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel v1.25.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.25.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 - go.opentelemetry.io/otel/exporters/prometheus v0.46.0 - go.opentelemetry.io/otel/sdk v1.24.0 - go.opentelemetry.io/otel/sdk/metric v1.24.0 - go.opentelemetry.io/otel/trace v1.24.0 + go.opentelemetry.io/otel/sdk v1.25.0 + go.opentelemetry.io/otel/sdk/metric v1.25.0 + go.opentelemetry.io/otel/trace v1.25.0 go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20231226003508-02704c960a9b - golang.org/x/net v0.19.0 - golang.org/x/sys v0.17.0 + golang.org/x/net v0.23.0 + golang.org/x/sys v0.18.0 ) require ( @@ -74,13 +74,12 @@ require ( github.com/andybalholm/brotli v1.0.6 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d // indirect - github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.2 // indirect github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/windows v1.0.1 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect github.com/bytedance/sonic v1.9.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect @@ -101,9 +100,9 @@ require ( github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.1.2 // indirect + github.com/golang/glog v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect @@ -119,7 +118,6 @@ require ( github.com/leodido/go-urn v1.2.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -151,10 +149,6 @@ require ( github.com/polydawn/go-timeless-api v0.0.0-20220821201550-b93919e12c56 // indirect github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect github.com/polydawn/rio v0.0.0-20220823181337-7c31ad9831a4 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect - github.com/prometheus/client_model v0.6.0 // indirect - github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -171,20 +165,20 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib v1.21.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.25.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/sync v0.5.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect - google.golang.org/grpc v1.61.1 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect + google.golang.org/grpc v1.63.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.22.3 // indirect modernc.org/mathutil v1.5.0 // indirect diff --git a/go.sum b/go.sum index 227ad50..8356481 100644 --- a/go.sum +++ b/go.sum @@ -119,7 +119,6 @@ github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d h1:2qVb9bs github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d/go.mod h1:iAr8OjJGLnLmVUr9MZ/rz4PWUy6Ouc2JLYuMArmvAJM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/billziss-gh/cgofuse v1.5.0 h1:kH516I/s+Ab4diL/Y/ayFeUjjA8ey+JK12xDfBf4HEs= github.com/billziss-gh/cgofuse v1.5.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM= @@ -139,8 +138,8 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -253,8 +252,8 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= @@ -277,8 +276,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -307,8 +306,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -399,8 +398,6 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -505,27 +502,19 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= -github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/ravilushqa/otelgqlgen v0.15.0 h1:U85nrlweMXTGaMChUViYM39/MXBZVeVVlpuHq+6eECQ= @@ -593,8 +582,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -640,25 +630,25 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib v1.21.1 h1:/U05KZ31iqMqAowhtW10cDPAViNY0tnpAacUgYBmuj8= go.opentelemetry.io/contrib v1.21.1/go.mod h1:usW9bPlrjHiJFbK0a6yK/M5wNHs3nLmtrT3vzhoD3co= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel v1.25.0 h1:gldB5FfhRl7OJQbUHt/8s0a7cE8fbsPAtdpRaApKy4k= +go.opentelemetry.io/otel v1.25.0/go.mod h1:Wa2ds5NOXEMkCmUou1WA7ZBfLTHWIsp034OVD7AO+Vg= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.25.0 h1:Wc4hZuYXhVqq+TfRXLXlmNIL/awOanGx8ssq3ciDQxc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.25.0/go.mod h1:BydOvapRqVEc0DVz27qWBX2jq45Ca5TI9mhZBDIdweY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM= -go.opentelemetry.io/otel/exporters/prometheus v0.46.0 h1:I8WIFXR351FoLJYuloU4EgXbtNX2URfU/85pUPheIEQ= -go.opentelemetry.io/otel/exporters/prometheus v0.46.0/go.mod h1:ztwVUHe5DTR/1v7PeuGRnU5Bbd4QKYwApWmuutKsJSs= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/metric v1.25.0 h1:LUKbS7ArpFL/I2jJHdJcqMGxkRdxpPHE0VU/D4NuEwA= +go.opentelemetry.io/otel/metric v1.25.0/go.mod h1:rkDLUSd2lC5lq2dFNrX9LGAbINP5B7WBkC78RXCpH5s= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8= -go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6Ek5IIrmwTLqMrrOE0= +go.opentelemetry.io/otel/sdk v1.25.0 h1:PDryEJPC8YJZQSyLY5eqLeafHtG+X7FWnf3aXMtxbqo= +go.opentelemetry.io/otel/sdk v1.25.0/go.mod h1:oFgzCM2zdsxKzz6zwpTZYLLQsFwc+K0daArPdIhuxkw= +go.opentelemetry.io/otel/sdk/metric v1.25.0 h1:7CiHOy08LbrxMAp4vWpbiPcklunUshVpAvGBrdDRlGw= +go.opentelemetry.io/otel/sdk/metric v1.25.0/go.mod h1:LzwoKptdbBBdYfvtGCzGwk6GWMA3aUzBOwtQpR6Nz7o= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel/trace v1.25.0 h1:tqukZGLwQYRIFtSQM2u2+yfMVTgGVeqRLPUYx1Dq6RM= +go.opentelemetry.io/otel/trace v1.25.0/go.mod h1:hCCs70XM/ljO+BeQkyFnbK28SBIJ/Emuha+ccrCRT7I= go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -682,8 +672,8 @@ golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220516162934-403b01795ae8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -751,8 +741,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -766,8 +756,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -817,8 +807,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -904,12 +894,12 @@ google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 h1:YJ5pD9rF8o9Qtta0Cmy9rdBwkSjrTCT6XTiUQVOtIos= -google.golang.org/genproto v0.0.0-20231212172506-995d672761c0/go.mod h1:l/k7rMz0vFTBPy+tFSGvXEd3z+BcoG1k7EHbqm+YBsY= -google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM= -google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -920,8 +910,8 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY= -google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/grpc v1.63.0 h1:WjKe+dnvABXyPJMD7KDNLxtoGk5tgk+YFWN6cBWjZE8= +google.golang.org/grpc v1.63.0/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -934,8 +924,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/ctxio/disk.go b/pkg/ctxio/disk.go new file mode 100644 index 0000000..cddd6a9 --- /dev/null +++ b/pkg/ctxio/disk.go @@ -0,0 +1,70 @@ +package ctxio + +import ( + "context" + "io" + "os" + "sync" +) + +type DiskCacheReader struct { + m sync.Mutex + + fo int64 + fr *os.File + to int64 + tr Reader +} + +var _ ReaderAt = (*DiskCacheReader)(nil) +var _ Reader = (*DiskCacheReader)(nil) +var _ Closer = (*DiskCacheReader)(nil) + +func NewDiskCacheReader(r Reader) (*DiskCacheReader, error) { + tempDir, err := os.MkdirTemp("/tmp", "tstor") + if err != nil { + return nil, err + } + fr, err := os.CreateTemp(tempDir, "dtb_tmp") + if err != nil { + return nil, err + } + + tr := TeeReader(r, WrapIoWriter(fr)) + return &DiskCacheReader{fr: fr, tr: tr}, nil +} + +func (dtr *DiskCacheReader) ReadAt(ctx context.Context, p []byte, off int64) (int, error) { + dtr.m.Lock() + defer dtr.m.Unlock() + tb := off + int64(len(p)) + + if tb > dtr.fo { + w, err := CopyN(ctx, Discard, dtr.tr, tb-dtr.fo) + dtr.to += w + if err != nil && err != io.EOF { + return 0, err + } + } + + n, err := dtr.fr.ReadAt(p, off) + dtr.fo += int64(n) + return n, err +} + +func (dtr *DiskCacheReader) Read(ctx context.Context, p []byte) (n int, err error) { + dtr.m.Lock() + defer dtr.m.Unlock() + // use directly tee reader here + n, err = dtr.tr.Read(ctx, p) + dtr.to += int64(n) + return +} + +func (dtr *DiskCacheReader) Close(ctx context.Context) error { + if err := dtr.fr.Close(); err != nil { + return err + } + + return os.Remove(dtr.fr.Name()) +} diff --git a/pkg/ctxio/io.go b/pkg/ctxio/io.go index fa0c561..e0a7158 100644 --- a/pkg/ctxio/io.go +++ b/pkg/ctxio/io.go @@ -9,9 +9,9 @@ import ( // Seek whence values. const ( - SeekStart = 0 // seek relative to the origin of the file - SeekCurrent = 1 // seek relative to the current offset - SeekEnd = 2 // seek relative to the end + SeekStart = io.SeekStart // seek relative to the origin of the file + SeekCurrent = io.SeekCurrent // seek relative to the current offset + SeekEnd = io.SeekEnd // seek relative to the end ) // ErrShortWrite means that a write accepted fewer bytes than requested diff --git a/pkg/ctxio/reader.go b/pkg/ctxio/reader.go index 8d2e4c7..3ec11b6 100644 --- a/pkg/ctxio/reader.go +++ b/pkg/ctxio/reader.go @@ -62,3 +62,44 @@ func (c *wrapReader) Read(ctx context.Context, p []byte) (n int, err error) { } return c.r.Read(p) } + +func WrapIoWriter(w io.Writer) Writer { + return &wrapWriter{w: w} +} + +type wrapWriter struct { + w io.Writer +} + +var _ Writer = (*wrapWriter)(nil) + +// Write implements Writer. +func (c *wrapWriter) Write(ctx context.Context, p []byte) (n int, err error) { + if ctx.Err() != nil { + return 0, ctx.Err() + } + return c.w.Write(p) +} + +func WrapIoReadCloser(r io.ReadCloser) ReadCloser { + return &wrapReadCloser{r: r} +} + +type wrapReadCloser struct { + r io.ReadCloser +} + +var _ Reader = (*wrapReadCloser)(nil) + +// Read implements Reader. +func (c *wrapReadCloser) Read(ctx context.Context, p []byte) (n int, err error) { + if ctx.Err() != nil { + return 0, ctx.Err() + } + return c.r.Read(p) +} + +// Close implements ReadCloser. +func (c *wrapReadCloser) Close(ctx context.Context) error { + return c.r.Close() +} diff --git a/pkg/ctxio/readerat.go b/pkg/ctxio/readerat.go new file mode 100644 index 0000000..af2d156 --- /dev/null +++ b/pkg/ctxio/readerat.go @@ -0,0 +1,47 @@ +package ctxio + +import ( + "context" + "sync" +) + +type ReaderReaderAtWrapper struct { + mu sync.Mutex + rat ReaderAt + offset int64 +} + +func NewReaderReaderAtWrapper(rat ReaderAt) *ReaderReaderAtWrapper { + return &ReaderReaderAtWrapper{ + rat: rat, + } +} + +var _ Reader = (*ReaderReaderAtWrapper)(nil) +var _ ReaderAt = (*ReaderReaderAtWrapper)(nil) +var _ Closer = (*ReaderReaderAtWrapper)(nil) + +// Read implements Reader. +func (r *ReaderReaderAtWrapper) Read(ctx context.Context, p []byte) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + n, err = r.rat.ReadAt(ctx, p, r.offset) + r.offset += int64(n) + return n, err +} + +// ReadAt implements ReaderAt. +func (r *ReaderReaderAtWrapper) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) { + return r.rat.ReadAt(ctx, p, off) +} + +// Close implements Closer. +func (r *ReaderReaderAtWrapper) Close(ctx context.Context) (err error) { + if c, ok := r.rat.(Closer); ok { + err = c.Close(ctx) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/ctxio/seeker.go b/pkg/ctxio/seeker.go index a482477..9a696c9 100644 --- a/pkg/ctxio/seeker.go +++ b/pkg/ctxio/seeker.go @@ -16,7 +16,7 @@ type ioSeekerWrapper struct { r ReaderAt } -func IoReadSeekerWrapper(ctx context.Context, r ReaderAt, size int64) io.ReadSeeker { +func WrapIoReadSeeker(ctx context.Context, r ReaderAt, size int64) io.ReadSeeker { return &ioSeekerWrapper{ ctx: ctx, r: r, diff --git a/pkg/rlog/rlog.go b/pkg/rlog/rlog.go index cbe6b75..bcf21b2 100644 --- a/pkg/rlog/rlog.go +++ b/pkg/rlog/rlog.go @@ -1,24 +1,26 @@ package rlog import ( + "context" "log/slog" "os" + "runtime" + "strings" + "time" "github.com/rs/zerolog" slogmulti "github.com/samber/slog-multi" slogzerolog "github.com/samber/slog-zerolog" ) -const errKey = "error" -const labelGroupKey = "labelGroup" - -var zl = zerolog.New(&zerolog.ConsoleWriter{Out: os.Stderr}) - -var handlers = []slog.Handler{ - slogzerolog.Option{Logger: &zl}.NewZerologHandler(), -} - -var defaultLogger = slog.New(slogmulti.Fanout(handlers...)) +var ( + zl = zerolog.New(&zerolog.ConsoleWriter{Out: os.Stderr}) + handlers = []slog.Handler{ + slogzerolog.Option{Logger: &zl}.NewZerologHandler(), + } + handler = slogmulti.Fanout(handlers...) + defaultLogger = slog.New(handler) +) func init() { slog.SetDefault(defaultLogger) @@ -26,41 +28,107 @@ func init() { func AddHandler(nh slog.Handler) { handlers = append(handlers, nh) - defaultLogger = slog.New(slogmulti.Fanout(handlers...)) + handler = slogmulti.Fanout(handlers...) + defaultLogger = slog.New(handler) slog.SetDefault(defaultLogger) } -func ComponentLog(name string) *slog.Logger { - return defaultLogger.With(slog.String("component", name)) +type Logger struct { + handler slog.Handler + component []string } -func FunctionLog(log *slog.Logger, name string) *slog.Logger { - return log.With(slog.String("function", name)) +const functionKey = "function" + +func (l *Logger) log(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) { + var pcs [1]uintptr + runtime.Callers(3, pcs[:]) + pc := pcs[0] + f := runtime.FuncForPC(pc) + attrs = append(attrs, slog.String(functionKey, f.Name())) + + r := slog.NewRecord(time.Now(), level, msg, pc) + r.AddAttrs(attrs...) + if ctx == nil { + ctx = context.Background() + } + _ = l.handler.Handle(ctx, r) } -func EndpointLog(log *slog.Logger, name string) *slog.Logger { - return log.With(slog.String("endpoint", name)) +func (l *Logger) Debug(ctx context.Context, msg string, attrs ...slog.Attr) { + l.log(ctx, slog.LevelDebug, msg, attrs...) } -func Err(err error) slog.Attr { - return slog.Attr{Key: errKey, Value: fmtErr(err)} +func (l *Logger) Info(ctx context.Context, msg string, attrs ...slog.Attr) { + l.log(ctx, slog.LevelInfo, msg, attrs...) } -func Label(args ...any) slog.Attr { - return slog.Group(labelGroupKey, args...) +func (l *Logger) Warn(ctx context.Context, msg string, attrs ...slog.Attr) { + l.log(ctx, slog.LevelWarn, msg, attrs...) } -// fmtErr returns a slog.GroupValue with keys "msg" and "trace". If the error +func (l *Logger) Error(ctx context.Context, msg string, attrs ...slog.Attr) { + l.log(ctx, slog.LevelError, msg, attrs...) +} + +const componentKey = "component" +const componentSep = "." + +func (log *Logger) WithComponent(name string) *Logger { + c := append(log.component, name) + return &Logger{ + handler: log.handler.WithAttrs([]slog.Attr{ + slog.String(componentKey, strings.Join(c, componentSep)), + }), + component: c, + } +} + +func (l *Logger) With(attrs ...slog.Attr) *Logger { + return &Logger{ + handler: l.handler.WithAttrs(attrs), + component: l.component, + } +} + +const endpointKey = "endpoint" + +func (l *Logger) WithEndpoint(name string) *Logger { + return &Logger{ + handler: l.handler.WithAttrs([]slog.Attr{ + slog.String(endpointKey, name), + }), + component: l.component, + } +} + +const errKey = "error" + +func Error(err error) slog.Attr { + return slog.Attr{Key: errKey, Value: errValue(err)} +} + +// errValue returns a slog.GroupValue with keys "msg" and "trace". If the error // does not implement interface { StackTrace() errors.StackTrace }, the "trace" // key is omitted. -func fmtErr(err error) slog.Value { +func errValue(err error) slog.Value { if err == nil { return slog.AnyValue(nil) } var groupValues []slog.Attr - groupValues = append(groupValues, slog.String("msg", err.Error())) + groupValues = append(groupValues, + slog.String("msg", err.Error()), + slog.Any("value", err), + ) return slog.GroupValue(groupValues...) } + +func Component(name string) *Logger { + return &Logger{ + handler: handler, + component: []string{name}, + } +} diff --git a/src/export/nfs/kvhandler.go b/src/export/nfs/kvhandler.go index c814405..b3731fa 100644 --- a/src/export/nfs/kvhandler.go +++ b/src/export/nfs/kvhandler.go @@ -4,8 +4,8 @@ import ( "context" "fmt" "path" - "slices" - "time" + "strings" + "sync" "git.kmsign.ru/royalcat/tstor/pkg/go-nfs" "git.kmsign.ru/royalcat/tstor/src/config" @@ -14,29 +14,51 @@ import ( "github.com/royalcat/kv" ) -const lifetime = time.Hour * 24 +type handle []string + +const sep = "\",\"" + +func (p handle) String() string { + return strings.Join(p, sep) +} + +// MarshalBinary implements kv.Binary. +func (p handle) MarshalBinary() (data []byte, err error) { + return []byte(strings.Join(p, sep)), nil +} + +// UnmarshalBinary implements kv.Binary. +func (p *handle) UnmarshalBinary(data []byte) error { + path := strings.Split(string(data), sep) + *p = path + return nil +} + +var _ kv.Binary = (*handle)(nil) + +func bytesToPath(path []string) string { + return strings.Join(path, sep) +} // NewKvHandler provides a basic to/from-file handle cache that can be tuned with a smaller cache of active directory listings. func NewKvHandler(h nfs.Handler, fs nfs.Filesystem) (nfs.Handler, error) { - activeHandles, err := kv.NewBadgerKVMarhsler[uuid.UUID, []string](path.Join(config.Config.Mounts.NFS.CachePath, "handlers")) + activeHandles, err := kv.NewBadgerKVMarhsler[uuid.UUID, handle](path.Join(config.Config.Mounts.NFS.CachePath, "handlers")) if err != nil { return nil, err } - // if s, ok := activeHandles.(kv.BadgerStore); ok { - // db := s.BadgerDB() - // enable with managed database - // go func() { - // for n := range time.NewTimer(lifetime / 2).C { - // db.SetDiscardTs(uint64(n.Add(-lifetime).Unix())) - // } - // }() - // } + reverseCache := map[string]uuid.UUID{} + + activeHandles.Range(context.Background(), func(k uuid.UUID, v handle) bool { + reverseCache[v.String()] = k + return true + }) return &CachingHandler{ Handler: h, fs: fs, activeHandles: activeHandles, + reverseCache: reverseCache, }, nil } @@ -44,32 +66,36 @@ func NewKvHandler(h nfs.Handler, fs nfs.Filesystem) (nfs.Handler, error) { type CachingHandler struct { nfs.Handler - fs nfs.Filesystem - activeHandles kv.Store[uuid.UUID, []string] + mu sync.RWMutex + + fs nfs.Filesystem + + activeHandles kv.Store[uuid.UUID, handle] + reverseCache map[string]uuid.UUID } // ToHandle takes a file and represents it with an opaque handle to reference it. // In stateless nfs (when it's serving a unix fs) this can be the device + inode // but we can generalize with a stateful local cache of handed out IDs. func (c *CachingHandler) ToHandle(_ nfs.Filesystem, path []string) []byte { - ctx := context.Background() - var id uuid.UUID - c.activeHandles.Range(ctx, func(k uuid.UUID, v []string) bool { - if slices.Equal(path, v) { - id = k - return false - } - return true - }) + + cacheKey := handle(path).String() + + if cacheId, ok := c.reverseCache[cacheKey]; ok { + id = cacheId + } if id != uuid.Nil { return id[:] } - id = uuid.New() + c.mu.Lock() + defer c.mu.Unlock() + id = uuid.New() + c.reverseCache[cacheKey] = id c.activeHandles.Set(ctx, id, path) return id[:] @@ -77,11 +103,14 @@ func (c *CachingHandler) ToHandle(_ nfs.Filesystem, path []string) []byte { // FromHandle converts from an opaque handle to the file it represents func (c *CachingHandler) FromHandle(fh []byte) (nfs.Filesystem, []string, error) { + c.mu.Lock() + defer c.mu.Unlock() + ctx := context.Background() id, err := uuid.FromBytes(fh) if err != nil { - return nil, []string{}, err + return nil, nil, err } paths, found, err := c.activeHandles.Get(ctx, id) @@ -93,7 +122,7 @@ func (c *CachingHandler) FromHandle(fh []byte) (nfs.Filesystem, []string, error) return c.fs, paths, nil } - return nil, []string{}, &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale} + return nil, nil, &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale} } func (c *CachingHandler) InvalidateHandle(fs nfs.Filesystem, handle []byte) error { @@ -103,8 +132,7 @@ func (c *CachingHandler) InvalidateHandle(fs nfs.Filesystem, handle []byte) erro if err != nil { return err } - c.activeHandles.Delete(ctx, id) - return nil + return c.activeHandles.Delete(ctx, id) } const maxInt = int(^uint(0) >> 1) @@ -114,14 +142,14 @@ func (c *CachingHandler) HandleLimit() int { return maxInt } -func hasPrefix(path, prefix []string) bool { - if len(prefix) > len(path) { - return false - } - for i, e := range prefix { - if path[i] != e { - return false - } - } - return true -} +// func hasPrefix(path, prefix []string) bool { +// if len(prefix) > len(path) { +// return false +// } +// for i, e := range prefix { +// if path[i] != e { +// return false +// } +// } +// return true +// } diff --git a/src/host/datastorage/storage.go b/src/host/datastorage/storage.go index 1145f86..dc4406f 100644 --- a/src/host/datastorage/storage.go +++ b/src/host/datastorage/storage.go @@ -12,7 +12,6 @@ import ( "path/filepath" "slices" - "git.kmsign.ru/royalcat/tstor/pkg/rlog" "git.kmsign.ru/royalcat/tstor/src/host/controller" "github.com/anacrolix/torrent" "github.com/anacrolix/torrent/metainfo" @@ -223,7 +222,7 @@ func (s *DataStorage) Dedupe(ctx context.Context) (uint64, error) { ctx, span := tracer.Start(ctx, fmt.Sprintf("Dedupe")) defer span.End() - log := rlog.FunctionLog(s.log, "Dedupe") + log := s.log sizeMap := map[int64][]string{} err := s.iterFiles(ctx, func(ctx context.Context, path string, info fs.FileInfo) error { @@ -314,7 +313,7 @@ func (s *DataStorage) dedupeFiles(ctx context.Context, paths []string) (deduped span.End() }() - log := rlog.FunctionLog(s.log, "dedupeFiles") + log := s.log srcF, err := os.Open(paths[0]) if err != nil { diff --git a/src/host/service/service.go b/src/host/service/service.go index a5809c7..8186ab3 100644 --- a/src/host/service/service.go +++ b/src/host/service/service.go @@ -56,7 +56,7 @@ type Service struct { dirsAquire kv.Store[string, DirAquire] - log *slog.Logger + log *rlog.Logger } func NewService(sourceDir string, cfg config.TorrentClient, c *torrent.Client, @@ -68,7 +68,7 @@ func NewService(sourceDir string, cfg config.TorrentClient, c *torrent.Client, } s := &Service{ - log: slog.With("component", "torrent-service"), + log: rlog.Component("torrent-service"), c: c, DefaultPriority: types.PiecePriorityNone, excludedFiles: excludedFiles, @@ -86,7 +86,7 @@ func NewService(sourceDir string, cfg config.TorrentClient, c *torrent.Client, ctx := context.Background() err := s.loadTorrentFiles(ctx) if err != nil { - s.log.Error("initial torrent load failed", "error", err) + s.log.Error(ctx, "initial torrent load failed", rlog.Error(err)) } close(s.torrentLoaded) }() @@ -105,8 +105,7 @@ func (s *Service) Close() error { func (s *Service) LoadTorrent(ctx context.Context, f vfs.File) (*torrent.Torrent, error) { ctx, span := tracer.Start(ctx, "LoadTorrent") defer span.End() - - log := rlog.FunctionLog(s.log, "LoadTorrent") + log := s.log defer f.Close(ctx) @@ -126,7 +125,7 @@ func (s *Service) LoadTorrent(ctx context.Context, f vfs.File) (*torrent.Torrent if !ok { span.AddEvent("torrent not found, loading from file") - log.InfoContext(ctx, "torrent not found, loading from file") + log.Info(ctx, "torrent not found, loading from file") spec, err := torrent.TorrentSpecFromMetaInfoErr(mi) if err != nil { @@ -135,12 +134,12 @@ func (s *Service) LoadTorrent(ctx context.Context, f vfs.File) (*torrent.Torrent infoBytes := spec.InfoBytes if !isValidInfoHashBytes(infoBytes) { - log.WarnContext(ctx, "info loaded from spec not valid") + log.Warn(ctx, "info loaded from spec not valid") infoBytes = nil } if len(infoBytes) == 0 { - log.InfoContext(ctx, "no info loaded from file, try to load from cache") + log.Info(ctx, "no info loaded from file, try to load from cache") infoBytes, err = s.infoBytes.GetBytes(spec.InfoHash) if err != nil && err != store.ErrNotFound { return nil, fmt.Errorf("get info bytes from database: %w", err) @@ -164,7 +163,10 @@ func (s *Service) LoadTorrent(ctx context.Context, f vfs.File) (*torrent.Torrent case <-t.GotInfo(): err := s.infoBytes.Set(t.InfoHash(), t.Metainfo()) if err != nil { - s.log.Error("error setting info bytes for torrent %s: %s", t.Name(), err.Error()) + log.Error(ctx, "error setting info bytes for torrent", + slog.String("torrent-name", t.Name()), + rlog.Error(err), + ) } } span.AddEvent("got info") @@ -190,7 +192,10 @@ func (s *Service) LoadTorrent(ctx context.Context, f vfs.File) (*torrent.Torrent } func (s *Service) checkTorrentCompatable(ctx context.Context, ih infohash.T, info metainfo.Info) (compatable bool, tryLater bool, err error) { - log := s.log.With("new-name", info.BestName(), "new-infohash", ih.String()) + log := s.log.With( + slog.String("new-name", info.BestName()), + slog.String("new-infohash", ih.String()), + ) name := info.BestName() @@ -207,12 +212,12 @@ func (s *Service) checkTorrentCompatable(ctx context.Context, ih infohash.T, inf return false, false, err } - log.Debug("acquiring was not found, so created") + log.Debug(ctx, "acquiring was not found, so created") return true, false, nil } if slices.Contains(aq.Hashes, ih) { - log.Debug("hash already know to be compatable") + log.Debug(ctx, "hash already know to be compatable") return true, false, nil } @@ -226,30 +231,30 @@ func (s *Service) checkTorrentCompatable(ctx context.Context, ih infohash.T, inf existingFiles := slices.Clone(existingInfo.Files) newFiles := slices.Clone(info.Files) - if !s.checkTorrentFilesCompatable(aq, existingFiles, newFiles) { + if !s.checkTorrentFilesCompatable(ctx, aq, existingFiles, newFiles) { return false, false, nil } aq.Hashes = slicesUnique(append(aq.Hashes, ih)) err = s.dirsAquire.Set(ctx, aq.Name, aq) if err != nil { - log.Warn("torrent not compatible") + log.Warn(ctx, "torrent not compatible") return false, false, err } } if slices.Contains(aq.Hashes, ih) { - log.Debug("hash is compatable") + log.Debug(ctx, "hash is compatable") return true, false, nil } - log.Debug("torrent with same name not found, try later") + log.Debug(ctx, "torrent with same name not found, try later") return false, true, nil } -func (s *Service) checkTorrentFilesCompatable(aq DirAquire, existingFiles, newFiles []metainfo.FileInfo) bool { - log := s.log.With("name", aq.Name) +func (s *Service) checkTorrentFilesCompatable(ctx context.Context, aq DirAquire, existingFiles, newFiles []metainfo.FileInfo) bool { + log := s.log.With(slog.String("name", aq.Name)) pathCmp := func(a, b metainfo.FileInfo) int { return slices.Compare(a.BestPath(), b.BestPath()) @@ -286,7 +291,11 @@ func (s *Service) checkTorrentFilesCompatable(aq DirAquire, existingFiles, newFi for _, e := range existingFiles { if e.Path == n.Path && e.Length != n.Length { - log.Warn("torrents not compatible, has files with different length", "path", n.Path, "existing-length", e.Length, "new-length", e.Length) + log.Warn(ctx, "torrents not compatible, has files with different length", + slog.String("path", n.Path), + slog.Int64("existing-length", e.Length), + slog.Int64("new-length", e.Length), + ) return false } } @@ -343,8 +352,7 @@ func (s *Service) loadTorrentFiles(ctx context.Context) error { attribute.Int("workers", loadWorkers), )) defer span.End() - - log := rlog.FunctionLog(s.log, "loadTorrentFiles") + log := s.log loaderPaths := make(chan string) wg := sync.WaitGroup{} @@ -359,14 +367,14 @@ func (s *Service) loadTorrentFiles(ctx context.Context) error { for path := range loaderPaths { file, err := vfs.NewLazyOsFile(path) if err != nil { - log.Error("error opening torrent file", "filename", path, rlog.Err(err)) + log.Error(ctx, "error opening torrent file", slog.String("filename", path), rlog.Error(err)) continue } defer file.Close(ctx) _, err = s.LoadTorrent(ctx, file) if err != nil { - s.log.Error("failed adding torrent", "error", err) + log.Error(ctx, "failed adding torrent", rlog.Error(err)) } } wg.Done() diff --git a/src/host/vfs/archive.go b/src/host/vfs/archive.go index 4717ec8..002cd17 100644 --- a/src/host/vfs/archive.go +++ b/src/host/vfs/archive.go @@ -168,27 +168,34 @@ func (a *ArchiveFS) Type() fs.FileMode { var _ File = (*archiveFile)(nil) func NewArchiveFile(name string, size int64, af archiveFileReaderFactory) *archiveFile { + + readerat := newReaderAtNoCache(func(ctx context.Context) (ctxio.ReadCloser, error) { + rc, err := af(ctx) + if err != nil { + return nil, err + } + return ctxio.WrapIoReadCloser(rc), nil + }) + return &archiveFile{ name: name, size: size, af: af, - buffer: ctxio.NewFileBuffer(nil), + reader: ctxio.NewReaderReaderAtWrapper(readerat), } } -const readahead = 1024 * 16 - type archiveFile struct { name string size int64 af archiveFileReaderFactory - m sync.Mutex - - offset int64 - readen int64 - buffer *ctxio.FileBuffer + reader interface { + ctxio.Reader + ctxio.ReaderAt + ctxio.Closer + } } // Name implements File. @@ -214,55 +221,15 @@ func (d *archiveFile) IsDir() bool { } func (d *archiveFile) Close(ctx context.Context) error { - return d.buffer.Close(ctx) -} - -func (d *archiveFile) loadMore(ctx context.Context, to int64) error { - d.m.Lock() - defer d.m.Unlock() - - if to < d.readen { - return nil - } - - reader, err := d.af(ctx) - if err != nil { - return fmt.Errorf("failed to get file reader: %w", err) - } - _, err = d.buffer.Seek(0, io.SeekStart) - if err != nil { - return fmt.Errorf("failed to seek to start of the file: %w", err) - } - d.readen, err = ctxio.CopyN(ctx, d.buffer, ctxio.WrapIoReader(reader), to+readahead) - if err != nil && err != io.EOF { - return fmt.Errorf("error copying from archive file reader: %w", err) - } - - return nil + return d.reader.Close(ctx) } func (d *archiveFile) Read(ctx context.Context, p []byte) (n int, err error) { - err = d.loadMore(ctx, d.offset+int64(len(p))) - if err != nil { - return 0, fmt.Errorf("failed to load more from archive file: %w", err) - } - n, err = d.buffer.Read(ctx, p) - if err != nil && err != io.EOF { - return n, fmt.Errorf("failed to read from buffer: %w", err) - } - return n, nil + return d.reader.Read(ctx, p) } func (d *archiveFile) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) { - err = d.loadMore(ctx, off+int64(len(p))) - if err != nil { - return 0, fmt.Errorf("failed to load more from archive file: %w", err) - } - n, err = d.buffer.ReadAt(ctx, p, off) - if err != nil && err != io.EOF { - return n, fmt.Errorf("failed to read from buffer: %w", err) - } - return n, nil + return d.reader.ReadAt(ctx, p, off) } type archiveFileReaderFactory func(ctx context.Context) (io.ReadCloser, error) @@ -287,7 +254,6 @@ func ZipLoader(ctx context.Context, ctxreader ctxio.ReaderAt, size int64) (map[s i := i af := func(ctx context.Context) (io.ReadCloser, error) { reader := ctxio.IoReaderAt(ctx, ctxreader) - zr, err := zip.NewReader(reader, size) if err != nil { return nil, err @@ -349,7 +315,7 @@ func SevenZipLoader(ctx context.Context, ctxreader ctxio.ReaderAt, size int64) ( var _ archiveLoader = RarLoader func RarLoader(ctx context.Context, ctxreader ctxio.ReaderAt, size int64) (map[string]*archiveFile, error) { - reader := ctxio.IoReadSeekerWrapper(ctx, ctxreader, size) + reader := ctxio.WrapIoReadSeeker(ctx, ctxreader, size) r, err := rardecode.NewReader(reader) if err != nil { @@ -368,7 +334,7 @@ func RarLoader(ctx context.Context, ctxreader ctxio.ReaderAt, size int64) (map[s name := header.Name af := func(ctx context.Context) (io.ReadCloser, error) { - reader := ctxio.IoReadSeekerWrapper(ctx, ctxreader, size) + reader := ctxio.WrapIoReadSeeker(ctx, ctxreader, size) r, err := rardecode.NewReader(reader) if err != nil { return nil, err @@ -390,3 +356,76 @@ func RarLoader(ctx context.Context, ctxreader ctxio.ReaderAt, size int64) (map[s return out, nil } + +func newReaderAtNoCache(newReader func(ctx context.Context) (ctxio.ReadCloser, error)) *readerAtNoCache { + return &readerAtNoCache{ + newReader: newReader, + } +} + +type readerAtNoCache struct { + newReader func(ctx context.Context) (ctxio.ReadCloser, error) + + mu sync.Mutex + readerCtx context.Context + r ctxio.ReadCloser + nread int + isClosed bool +} + +func (f *readerAtNoCache) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.isClosed { + return 0, fs.ErrClosed + } + + if f.r == nil || int64(f.nread) > off || f.readerCtx.Err() == context.DeadlineExceeded || f.readerCtx.Err() == context.Canceled { + err = f.recreateReader(ctx) + if err != nil { + return 0, err + } + } + + if f.nread != int(off) { + _, err = ctxio.CopyN(ctx, ctxio.Discard, f.r, off-int64(f.nread)) + if err != nil { + return 0, err + } + + } + + n, err = f.r.Read(ctx, p) + f.nread += n + if err != nil { + return n, err + } + return n, err +} + +func (f *readerAtNoCache) recreateReader(ctx context.Context) (err error) { + if f.r != nil { + err = f.r.Close(ctx) + if err != nil { + return err + } + } + + f.r, err = f.newReader(ctx) + if err != nil { + return err + } + f.readerCtx = ctx + f.nread = 0 + return nil +} + +func (f *readerAtNoCache) Close(ctx context.Context) error { + f.mu.Lock() + defer f.mu.Unlock() + + f.isClosed = true + + return f.r.Close(ctx) +} diff --git a/src/host/vfs/log.go b/src/host/vfs/log.go index c5519ed..e229c73 100644 --- a/src/host/vfs/log.go +++ b/src/host/vfs/log.go @@ -15,7 +15,7 @@ import ( type LogFS struct { fs Filesystem - log *slog.Logger + log *rlog.Logger timeout time.Duration readTimeout time.Duration @@ -30,7 +30,7 @@ var _ Filesystem = (*LogFS)(nil) func WrapLogFS(vfs Filesystem) *LogFS { return &LogFS{ fs: vfs, - log: rlog.ComponentLog("fs"), + log: rlog.Component("logfs"), timeout: time.Minute * 3, readTimeout: time.Minute, } @@ -102,7 +102,7 @@ func (fs *LogFS) Open(ctx context.Context, filename string) (file File, err erro file, err = fs.fs.Open(ctx, filename) if isLoggableError(err) { - fs.log.With("filename", filename).Error("Failed to open file") + fs.log.Error(ctx, "Failed to open file") } file = WrapLogFile(file, filename, fs.log, fs.readTimeout) return file, err @@ -113,7 +113,10 @@ func (fs *LogFS) ReadDir(ctx context.Context, path string) (entries []fs.DirEntr ctx, cancel := context.WithTimeout(ctx, fs.timeout) defer cancel() ctx, span := tracer.Start(ctx, "ReadDir", - fs.traceAttrs(attribute.String("path", path)), + fs.traceAttrs( + attribute.String("path", path), + attribute.String("fs-type", reflect.TypeOf(fs.fs).Name()), + ), ) defer func() { if err != nil { @@ -124,7 +127,7 @@ func (fs *LogFS) ReadDir(ctx context.Context, path string) (entries []fs.DirEntr entries, err = fs.fs.ReadDir(ctx, path) if isLoggableError(err) { - fs.log.ErrorContext(ctx, "Failed to read dir", "path", path, "error", err.Error(), "fs-type", reflect.TypeOf(fs.fs).Name()) + fs.log.Error(ctx, "Failed to read dir", rlog.Error(err)) } return entries, err } @@ -145,7 +148,7 @@ func (lfs *LogFS) Stat(ctx context.Context, filename string) (info fs.FileInfo, info, err = lfs.fs.Stat(ctx, filename) if isLoggableError(err) { - lfs.log.Error("Failed to stat", "filename", filename, "error", err) + lfs.log.Error(ctx, "Failed to stat", rlog.Error(err)) } return info, err } @@ -166,7 +169,7 @@ func (fs *LogFS) Unlink(ctx context.Context, filename string) (err error) { err = fs.fs.Unlink(ctx, filename) if isLoggableError(err) { - fs.log.Error("Failed to stat", "filename", filename, "error", err) + fs.log.Error(ctx, "Failed to stat", rlog.Error(err)) } return err } @@ -175,7 +178,7 @@ type LogFile struct { filename string f File - log *slog.Logger + log *rlog.Logger timeout time.Duration } @@ -191,11 +194,11 @@ func (f *LogFile) Type() fs.FileMode { var _ File = (*LogFile)(nil) -func WrapLogFile(f File, filename string, log *slog.Logger, timeout time.Duration) *LogFile { +func WrapLogFile(f File, filename string, log *rlog.Logger, timeout time.Duration) *LogFile { return &LogFile{ filename: filename, f: f, - log: log.With("filename", filename), + log: log.With(slog.String("filename", filename)), timeout: timeout, } } @@ -216,7 +219,7 @@ func (f *LogFile) Close(ctx context.Context) (err error) { err = f.f.Close(ctx) if isLoggableError(err) { - f.log.ErrorContext(ctx, "Failed to close", "error", err) + f.log.Error(ctx, "Failed to close", rlog.Error(err)) } return err } @@ -246,7 +249,7 @@ func (f *LogFile) Read(ctx context.Context, p []byte) (n int, err error) { n, err = f.f.Read(ctx, p) if isLoggableError(err) { - f.log.Error("Failed to read", "error", err) + f.log.Error(ctx, "Failed to read", rlog.Error(err)) } return n, err } @@ -259,6 +262,7 @@ func (f *LogFile) ReadAt(ctx context.Context, p []byte, off int64) (n int, err e trace.WithAttributes( attribute.String("filename", f.filename), attribute.Int("length", len(p)), + attribute.Int64("offset", off), ), ) defer func() { @@ -271,7 +275,7 @@ func (f *LogFile) ReadAt(ctx context.Context, p []byte, off int64) (n int, err e n, err = f.f.ReadAt(ctx, p, off) if isLoggableError(err) { - f.log.Error("Failed to read", "offset", off, "error", err) + f.log.Error(ctx, "Failed to read") } return n, err } @@ -285,7 +289,7 @@ func (f *LogFile) Size() int64 { func (f *LogFile) Info() (fs.FileInfo, error) { info, err := f.f.Info() if isLoggableError(err) { - f.log.Error("Failed to info", "error", err) + f.log.Error(context.Background(), "Failed to info", rlog.Error(err)) } return info, err } diff --git a/src/host/vfs/resolver.go b/src/host/vfs/resolver.go index de028e5..c81a37b 100644 --- a/src/host/vfs/resolver.go +++ b/src/host/vfs/resolver.go @@ -23,14 +23,14 @@ type ResolverFS struct { rootFS Filesystem resolver *resolver - log *slog.Logger + log *rlog.Logger } func NewResolveFS(rootFs Filesystem, factories map[string]FsFactory) *ResolverFS { return &ResolverFS{ rootFS: rootFs, resolver: newResolver(factories), - log: rlog.ComponentLog("fs/resolverfs"), + log: rlog.Component("fs.resolverfs"), } } @@ -126,7 +126,9 @@ func (r *ResolverFS) ReadDir(ctx context.Context, dir string) ([]fs.DirEntry, er nestedfs, err := r.resolver.nestedFs(factoryCtx, filepath, file) if err != nil { if errors.Is(err, context.DeadlineExceeded) { - r.log.ErrorContext(ctx, "creating fs timed out", "filename", e.Name()) + r.log.Error(ctx, "creating fs timed out", + slog.String("filename", e.Name()), + ) return nil } diff --git a/src/telemetry/setup.go b/src/telemetry/setup.go index c83663a..914c0ee 100644 --- a/src/telemetry/setup.go +++ b/src/telemetry/setup.go @@ -15,8 +15,8 @@ import ( otelpyroscope "github.com/grafana/otel-profiling-go" "github.com/grafana/pyroscope-go" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" - "go.opentelemetry.io/otel/exporters/prometheus" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" @@ -24,7 +24,7 @@ import ( ) type Client struct { - log *slog.Logger + log *rlog.Logger tracerProvider *trace.TracerProvider metricProvider *metric.MeterProvider @@ -32,23 +32,22 @@ type Client struct { } func (client *Client) Shutdown(ctx context.Context) { - log := rlog.FunctionLog(client.log, "Shutdown") if client.metricProvider == nil { err := client.metricProvider.Shutdown(ctx) if err != nil { - log.Error("error shutting down metric provider", rlog.Err(err)) + client.log.Error(ctx, "error shutting down metric provider", rlog.Error(err)) } } if client.tracerProvider == nil { err := client.tracerProvider.Shutdown(ctx) if err != nil { - log.Error("error shutting down tracer provider", rlog.Err(err)) + client.log.Error(ctx, "error shutting down tracer provider", rlog.Error(err)) } } if client.loggerProvider == nil { err := client.loggerProvider.Shutdown(ctx) if err != nil { - log.Error("error shutting down logger provider", rlog.Err(err)) + client.log.Error(ctx, "error shutting down logger provider", rlog.Error(err)) } } } @@ -56,13 +55,13 @@ func (client *Client) Shutdown(ctx context.Context) { const appName = "tstor" func Setup(ctx context.Context, endpoint string) (*Client, error) { - log := rlog.ComponentLog("telemetry") + log := rlog.Component("telemetry") client := &Client{ log: log, } otel.SetErrorHandler(otel.ErrorHandlerFunc(func(cause error) { - log.Error("otel error", rlog.Err(cause)) + log.Error(context.Background(), "otel error", rlog.Error(cause)) })) hostName, _ := os.Hostname() @@ -79,16 +78,22 @@ func Setup(ctx context.Context, endpoint string) (*Client, error) { return nil, err } - metricExporter, err := prometheus.New(prometheus.WithNamespace(appName)) + meticExporter, err := otlpmetrichttp.New(ctx, + otlpmetrichttp.WithEndpoint(endpoint), + otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{ + Enabled: false, + }), + ) if err != nil { return nil, err } + client.metricProvider = metric.NewMeterProvider( - metric.WithReader(metricExporter), + metric.WithReader(metric.NewPeriodicReader(meticExporter)), metric.WithResource(r), ) otel.SetMeterProvider(client.metricProvider) - log.Info("prometheus metrics provider initialized") + log.Info(ctx, "prometheus metrics provider initialized") traceExporter, err := otlptracehttp.New(ctx, otlptracehttp.WithEndpoint(endpoint), @@ -105,7 +110,7 @@ func Setup(ctx context.Context, endpoint string) (*Client, error) { ) // otel.SetTracerProvider(client.tracerProvider) otel.SetTracerProvider(otelpyroscope.NewTracerProvider(client.tracerProvider)) - log.Info("otel tracing provider initialized") + log.Info(ctx, "otel tracing provider initialized") logExporter, err := otlplogs.NewExporter(ctx, otlplogs.WithClient( @@ -125,7 +130,7 @@ func Setup(ctx context.Context, endpoint string) (*Client, error) { Level: slog.LevelDebug, }), ) - client.log = slog.Default() + client.log = rlog.Component("telemetry-client") runtime.SetMutexProfileFraction(5) runtime.SetBlockProfileRate(5) @@ -135,7 +140,7 @@ func Setup(ctx context.Context, endpoint string) (*Client, error) { ServerAddress: "https://pyroscope.kmsign.ru", // you can disable logging by setting this to nil Logger: &pyroscopeLogger{ - log: rlog.ComponentLog("metrics.pyroscope"), + log: client.log.WithComponent("pyroscope"), }, ProfileTypes: []pyroscope.ProfileType{ // these profile types are enabled by default: @@ -160,22 +165,31 @@ func Setup(ctx context.Context, endpoint string) (*Client, error) { } type pyroscopeLogger struct { - log *slog.Logger + log *rlog.Logger } var _ pyroscope.Logger = (*pyroscopeLogger)(nil) // Debugf implements pyroscope.Logger. func (p *pyroscopeLogger) Debugf(msg string, args ...any) { - p.log.Debug(fmt.Sprintf(msg, args...)) + ctx := context.Background() + p.log.Debug(ctx, fmt.Sprintf(msg, args...)) } // Errorf implements pyroscope.Logger. func (p *pyroscopeLogger) Errorf(msg string, args ...any) { - p.log.Error(fmt.Sprintf(msg, args...)) + ctx := context.Background() + p.log.Error(ctx, fmt.Sprintf(msg, args...)) } // Infof implements pyroscope.Logger. func (p *pyroscopeLogger) Infof(msg string, args ...any) { - p.log.Info(fmt.Sprintf(msg, args...)) + ctx := context.Background() + p.log.Info(ctx, fmt.Sprintf(msg, args...)) +} + +func functionName() string { + var pcs [1]uintptr + runtime.Callers(1, pcs[:]) + return runtime.FuncForPC(pcs[0]).Name() }