package nfs import ( "context" "errors" "fmt" "path" "strings" "sync" "git.kmsign.ru/royalcat/tstor/pkg/go-nfs" "git.kmsign.ru/royalcat/tstor/src/config" "git.kmsign.ru/royalcat/tstor/src/logwrap" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" "github.com/google/uuid" "github.com/royalcat/kv" "github.com/royalcat/kv/kvbadger" ) type handle []string const sep = "\",\"" func (p handle) String() string { return strings.Join(p, sep) } // MarshalBinary implements kv.Binary. func (p handle) MarshalBinary() (data []byte, err error) { return []byte(strings.Join(p, sep)), nil } // UnmarshalBinary implements kv.Binary. func (p *handle) UnmarshalBinary(data []byte) error { path := strings.Split(string(data), sep) *p = path return nil } var _ kv.Binary = (*handle)(nil) func bytesToPath(path []string) string { return strings.Join(path, sep) } var kvhandlerMeter = otel.Meter("git.kmsign.ru/royalcat/tstor/src/export/nfs.kvhandler") // NewKvHandler provides a basic to/from-file handle cache that can be tuned with a smaller cache of active directory listings. func NewKvHandler(h nfs.Handler, fs nfs.Filesystem, config config.NFS) (nfs.Handler, error) { opts := kvbadger.DefaultOptions[handle](path.Join(config.CachePath, "handlers")) opts.Codec = kv.CodecBinary[handle, *handle]{} opts.BadgerOptions.Logger = logwrap.BadgerLogger("nfs", "kvhandler") activeHandles, err := kvbadger.NewBagerKVBinaryKey[uuid.UUID, handle](opts) if err != nil { return nil, err } reverseCache := map[string]uuid.UUID{} activeHandles.Range(context.Background(), func(k uuid.UUID, v handle) error { reverseCache[v.String()] = k return nil }) c := &CachingHandler{ Handler: h, fs: fs, activeHandles: activeHandles, reverseCache: reverseCache, } _, err = kvhandlerMeter.Int64ObservableGauge("nfs.activehandles", metric.WithInt64Callback(func(ctx context.Context, io metric.Int64Observer) error { io.Observe(int64(c.ActiveHandlers())) return nil }), ) if err != nil { return nil, err } return c, nil } // CachingHandler implements to/from handle via an LRU cache. type CachingHandler struct { nfs.Handler mu sync.RWMutex fs nfs.Filesystem activeHandles kv.Store[uuid.UUID, handle] reverseCache map[string]uuid.UUID } // ToHandle takes a file and represents it with an opaque handle to reference it. // In stateless nfs (when it's serving a unix fs) this can be the device + inode // but we can generalize with a stateful local cache of handed out IDs. func (c *CachingHandler) ToHandle(ctx context.Context, _ nfs.Filesystem, path []string) []byte { var id uuid.UUID cacheKey := handle(path).String() if cacheId, ok := c.reverseCache[cacheKey]; ok { id = cacheId } if id != uuid.Nil { return id[:] } c.mu.Lock() defer c.mu.Unlock() id = uuid.New() c.reverseCache[cacheKey] = id c.activeHandles.Set(ctx, id, path) return id[:] } // FromHandle converts from an opaque handle to the file it represents func (c *CachingHandler) FromHandle(ctx context.Context, fh []byte) (nfs.Filesystem, []string, error) { c.mu.Lock() defer c.mu.Unlock() id, err := uuid.FromBytes(fh) if err != nil { return nil, nil, err } paths, err := c.activeHandles.Get(ctx, id) if err != nil { if errors.Is(err, kv.ErrKeyNotFound) { return nil, nil, &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale} } return nil, nil, fmt.Errorf("kv error: %w", err) } return c.fs, paths, nil } func (c *CachingHandler) InvalidateHandle(ctx context.Context, fs nfs.Filesystem, handle []byte) error { //Remove from cache id, err := uuid.FromBytes(handle) if err != nil { return err } return c.activeHandles.Delete(ctx, id) } const maxHandlers = int(^uint(0) >> 1) // HandleLimit exports how many file handles can be safely stored by this cache. func (c *CachingHandler) HandleLimit() int { return maxHandlers } // HandleLimit exports how many file handles can be safely stored by this cache. func (c *CachingHandler) ActiveHandlers() int { c.mu.RLock() defer c.mu.RUnlock() return len(c.reverseCache) } // func hasPrefix(path, prefix []string) bool { // if len(prefix) > len(path) { // return false // } // for i, e := range prefix { // if path[i] != e { // return false // } // } // return true // }