From bf2dac5cf13340145214527bd9f9423d0b45caaa Mon Sep 17 00:00:00 2001
From: royalcat <k.adamovich20@gmail.com>
Date: Tue, 18 Jun 2024 22:34:28 +0300
Subject: [PATCH 1/3] remove nfs handle kv ttl
---
src/export/nfs/kvhandler.go | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/export/nfs/kvhandler.go b/src/export/nfs/kvhandler.go
index bc06f1f..d84c94b 100644
--- a/src/export/nfs/kvhandler.go
+++ b/src/export/nfs/kvhandler.go
@@ -7,7 +7,6 @@ import (
"path"
"strings"
"sync"
- "time"
"git.kmsign.ru/royalcat/tstor/pkg/go-nfs"
"git.kmsign.ru/royalcat/tstor/src/config"
@@ -51,7 +50,6 @@ var kvhandlerMeter = otel.Meter("git.kmsign.ru/royalcat/tstor/src/export/nfs.kvh
// NewKvHandler provides a basic to/from-file handle cache that can be tuned with a smaller cache of active directory listings.
func NewKvHandler(h nfs.Handler, fs nfs.Filesystem, config config.NFS) (nfs.Handler, error) {
opts := kvbadger.DefaultOptions(path.Join(config.CachePath, "handlers"))
- opts.DefaultTTL = time.Hour
opts.BadgerOptions.Logger = log.BadgerLogger("nfs", "kvhandler")
activeHandles, err := kvbadger.NewBagerKVBinaryKey[uuid.UUID, handle](opts)
From 2d72790c1a6d8eaec52c3583442703f83ce52131 Mon Sep 17 00:00:00 2001
From: royalcat <k.adamovich20@gmail.com>
Date: Wed, 19 Jun 2024 00:17:10 +0300
Subject: [PATCH 2/3] logfs opened files
---
cmd/tstor/main.go | 5 ++++-
src/export/nfs/kvhandler.go | 3 +--
src/vfs/fs.go | 3 ---
src/vfs/log.go | 42 ++++++++++++++++++++++++++++++++-----
4 files changed, 42 insertions(+), 11 deletions(-)
diff --git a/cmd/tstor/main.go b/cmd/tstor/main.go
index 3e7ad43..23e874e 100644
--- a/cmd/tstor/main.go
+++ b/cmd/tstor/main.go
@@ -109,7 +109,10 @@ func run(configPath string) error {
vfs.NewCtxBillyFs("/", ctxbilly.WrapFileSystem(sourceFs)),
tsrv, ytdlpsrv,
)
- sfs = vfs.WrapLogFS(sfs)
+ sfs, err = vfs.WrapLogFS(sfs)
+ if err != nil {
+ return err
+ }
if conf.Mounts.Fuse.Enabled {
mh := fuse.NewHandler(conf.Mounts.Fuse.AllowOther, conf.Mounts.Fuse.Path)
diff --git a/src/export/nfs/kvhandler.go b/src/export/nfs/kvhandler.go
index d84c94b..729a996 100644
--- a/src/export/nfs/kvhandler.go
+++ b/src/export/nfs/kvhandler.go
@@ -154,8 +154,7 @@ func (c *CachingHandler) InvalidateHandle(ctx context.Context, fs nfs.Filesystem
return c.activeHandles.Delete(ctx, id)
}
-// const maxInt = int(^uint(0) >> 1)
-const maxHandlers = 8129
+const maxHandlers = int(^uint(0) >> 1)
// HandleLimit exports how many file handles can be safely stored by this cache.
func (c *CachingHandler) HandleLimit() int {
diff --git a/src/vfs/fs.go b/src/vfs/fs.go
index 88cc267..79cb242 100644
--- a/src/vfs/fs.go
+++ b/src/vfs/fs.go
@@ -8,7 +8,6 @@ import (
"time"
"github.com/royalcat/ctxio"
- "go.opentelemetry.io/otel"
)
type File interface {
@@ -24,8 +23,6 @@ type File interface {
var ErrNotImplemented = errors.New("not implemented")
-var tracer = otel.Tracer("git.kmsign.ru/royalcat/tstor/src/vfs")
-
type Filesystem interface {
// Open opens the named file for reading. If successful, methods on the
// returned file can be used for reading; the associated file descriptor has
diff --git a/src/vfs/log.go b/src/vfs/log.go
index e229c73..ece02aa 100644
--- a/src/vfs/log.go
+++ b/src/vfs/log.go
@@ -3,19 +3,32 @@ package vfs
import (
"context"
"errors"
+ "fmt"
"io/fs"
"log/slog"
"reflect"
"time"
"git.kmsign.ru/royalcat/tstor/pkg/rlog"
+ "go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
)
+var (
+ meter = otel.Meter("git.kmsign.ru/royalcat/tstor/src/vfs")
+ tracer = otel.Tracer("git.kmsign.ru/royalcat/tstor/src/vfs")
+)
+
+type fsTelemetry struct {
+ openedFiles metric.Int64UpDownCounter
+}
+
type LogFS struct {
fs Filesystem
log *rlog.Logger
+ tel *fsTelemetry
timeout time.Duration
readTimeout time.Duration
@@ -27,13 +40,19 @@ func isLoggableError(err error) bool {
var _ Filesystem = (*LogFS)(nil)
-func WrapLogFS(vfs Filesystem) *LogFS {
+func WrapLogFS(vfs Filesystem) (*LogFS, error) {
+ openedFiles, err := meter.Int64UpDownCounter("vfs.opened_files")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create opened_files metric: %w", err)
+ }
+
return &LogFS{
fs: vfs,
log: rlog.Component("logfs"),
+ tel: &fsTelemetry{openedFiles: openedFiles},
timeout: time.Minute * 3,
readTimeout: time.Minute,
- }
+ }, nil
}
// ModTime implements Filesystem.
@@ -104,7 +123,12 @@ func (fs *LogFS) Open(ctx context.Context, filename string) (file File, err erro
if isLoggableError(err) {
fs.log.Error(ctx, "Failed to open file")
}
- file = WrapLogFile(file, filename, fs.log, fs.readTimeout)
+ file = wrapLogFile(file, filename, fs.log, fs.readTimeout, fs.tel)
+
+ if file != nil {
+ fs.tel.openedFiles.Add(ctx, 1)
+ }
+
return file, err
}
@@ -178,7 +202,9 @@ type LogFile struct {
filename string
f File
- log *rlog.Logger
+ log *rlog.Logger
+ tel *fsTelemetry
+
timeout time.Duration
}
@@ -194,11 +220,12 @@ func (f *LogFile) Type() fs.FileMode {
var _ File = (*LogFile)(nil)
-func WrapLogFile(f File, filename string, log *rlog.Logger, timeout time.Duration) *LogFile {
+func wrapLogFile(f File, filename string, log *rlog.Logger, timeout time.Duration, tel *fsTelemetry) *LogFile {
return &LogFile{
filename: filename,
f: f,
log: log.With(slog.String("filename", filename)),
+ tel: tel,
timeout: timeout,
}
}
@@ -221,6 +248,11 @@ func (f *LogFile) Close(ctx context.Context) (err error) {
if isLoggableError(err) {
f.log.Error(ctx, "Failed to close", rlog.Error(err))
}
+
+ if err != nil {
+ f.tel.openedFiles.Add(ctx, -1)
+ }
+
return err
}
From 77d19e08fc1c513983b9730212e67afd4cd49521 Mon Sep 17 00:00:00 2001
From: royalcat <k.adamovich20@gmail.com>
Date: Wed, 19 Jun 2024 00:17:43 +0300
Subject: [PATCH 3/3] fix nfs close file on read
---
pkg/go-nfs/nfs_onread.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/pkg/go-nfs/nfs_onread.go b/pkg/go-nfs/nfs_onread.go
index 2d9fd1e..107481a 100644
--- a/pkg/go-nfs/nfs_onread.go
+++ b/pkg/go-nfs/nfs_onread.go
@@ -52,6 +52,7 @@ func onRead(ctx context.Context, w *response, userHandle Handler) error {
}
return &NFSStatusError{NFSStatusAccess, err}
}
+ defer fh.Close(ctx)
resp := nfsReadResponse{}