tstor/src/export/nfs/kvhandler.go

package nfs

import (
	"context"
	"errors"
	"fmt"
	"path"
	"strings"
	"sync"

	"git.kmsign.ru/royalcat/tstor/pkg/go-nfs"
	"git.kmsign.ru/royalcat/tstor/src/config"
	"git.kmsign.ru/royalcat/tstor/src/logwrap"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/metric"

	"github.com/google/uuid"
	"github.com/royalcat/kv"
	"github.com/royalcat/kv/kvbadger"
)

type handle []string

const sep = "\",\""

func (p handle) String() string {
	return strings.Join(p, sep)
}

// MarshalBinary implements kv.Binary.
func (p handle) MarshalBinary() (data []byte, err error) {
	return []byte(strings.Join(p, sep)), nil
}

// UnmarshalBinary implements kv.Binary.
func (p *handle) UnmarshalBinary(data []byte) error {
	path := strings.Split(string(data), sep)
	*p = path
	return nil
}

var _ kv.Binary = (*handle)(nil)

func bytesToPath(path []string) string {
	return strings.Join(path, sep)
}

var kvhandlerMeter = otel.Meter("git.kmsign.ru/royalcat/tstor/src/export/nfs.kvhandler")

// NewKvHandler provides a basic to/from-file handle cache that can be tuned with a smaller cache of active directory listings.
func NewKvHandler(h nfs.Handler, fs nfs.Filesystem, config config.NFS) (nfs.Handler, error) {
	opts := kvbadger.DefaultOptions[handle](path.Join(config.CachePath, "handlers"))
	opts.Codec = kv.CodecBinary[handle, *handle]{}
	opts.BadgerOptions.Logger = logwrap.BadgerLogger("nfs", "kvhandler")

	activeHandles, err := kvbadger.NewBagerKVBinaryKey[uuid.UUID, handle](opts)
	if err != nil {
		return nil, err
	}

	reverseCache := map[string]uuid.UUID{}

	activeHandles.Range(context.Background(), func(k uuid.UUID, v handle) error {
		reverseCache[v.String()] = k
		return nil
	})

	c := &CachingHandler{
		Handler:       h,
		fs:            fs,
		activeHandles: activeHandles,
		reverseCache:  reverseCache,
	}

	_, err = kvhandlerMeter.Int64ObservableGauge("nfs.activehandles",
		metric.WithInt64Callback(func(ctx context.Context, io metric.Int64Observer) error {
			io.Observe(int64(c.ActiveHandlers()))
			return nil
		}),
	)
	if err != nil {
		return nil, err
	}

	return c, nil
}

// CachingHandler implements to/from handle via an LRU cache.
type CachingHandler struct {
	nfs.Handler

	mu sync.RWMutex

	fs nfs.Filesystem

	activeHandles kv.Store[uuid.UUID, handle]
	reverseCache  map[string]uuid.UUID
}

// ToHandle takes a file and represents it with an opaque handle to reference it.
// In stateless nfs (when it's serving a unix fs) this can be the device + inode
// but we can generalize with a stateful local cache of handed out IDs.
func (c *CachingHandler) ToHandle(ctx context.Context, _ nfs.Filesystem, path []string) []byte {
	var id uuid.UUID

	cacheKey := handle(path).String()

	if cacheId, ok := c.reverseCache[cacheKey]; ok {
		id = cacheId
	}

	if id != uuid.Nil {
		return id[:]
	}

	c.mu.Lock()
	defer c.mu.Unlock()

	id = uuid.New()
	c.reverseCache[cacheKey] = id
	c.activeHandles.Set(ctx, id, path)

	return id[:]
}

// FromHandle converts from an opaque handle to the file it represents
func (c *CachingHandler) FromHandle(ctx context.Context, fh []byte) (nfs.Filesystem, []string, error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	id, err := uuid.FromBytes(fh)
	if err != nil {
		return nil, nil, err
	}

	paths, err := c.activeHandles.Get(ctx, id)
	if err != nil {
		if errors.Is(err, kv.ErrKeyNotFound) {
			return nil, nil, &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale}
		}

		return nil, nil, fmt.Errorf("kv error: %w", err)
	}

	return c.fs, paths, nil

}

func (c *CachingHandler) InvalidateHandle(ctx context.Context, fs nfs.Filesystem, handle []byte) error {
	//Remove from cache
	id, err := uuid.FromBytes(handle)
	if err != nil {
		return err
	}
	return c.activeHandles.Delete(ctx, id)
}

const maxHandlers = int(^uint(0) >> 1)

// HandleLimit exports how many file handles can be safely stored by this cache.
func (c *CachingHandler) HandleLimit() int {
	return maxHandlers
}

// HandleLimit exports how many file handles can be safely stored by this cache.
func (c *CachingHandler) ActiveHandlers() int {
	c.mu.RLock()
	defer c.mu.RUnlock()

	return len(c.reverseCache)
}

// func hasPrefix(path, prefix []string) bool {
// 	if len(prefix) > len(path) {
// 		return false
// 	}
// 	for i, e := range prefix {
// 		if path[i] != e {
// 			return false
// 		}
// 	}
// 	return true
// }