433 lines
10 KiB
Go
433 lines
10 KiB
Go
package vfs
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"log/slog"
|
|
"path"
|
|
"reflect"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.kmsign.ru/royalcat/tstor/pkg/rlog"
|
|
"github.com/sourcegraph/conc/iter"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/trace"
|
|
"golang.org/x/exp/maps"
|
|
)
|
|
|
|
type ResolverFS struct {
|
|
rootFS Filesystem
|
|
resolver *Resolver
|
|
|
|
log *rlog.Logger
|
|
}
|
|
|
|
func NewResolveFS(rootFs Filesystem, factories map[string]FsFactory) *ResolverFS {
|
|
return &ResolverFS{
|
|
rootFS: rootFs,
|
|
resolver: NewResolver(factories),
|
|
log: rlog.Component("fs.resolverfs"),
|
|
}
|
|
}
|
|
|
|
// ModTime implements Filesystem.
|
|
func (r *ResolverFS) ModTime() time.Time {
|
|
return time.Time{}
|
|
}
|
|
|
|
// Mode implements Filesystem.
|
|
func (r *ResolverFS) Mode() fs.FileMode {
|
|
return fs.ModeDir
|
|
}
|
|
|
|
// Size implements Filesystem.
|
|
func (r *ResolverFS) Size() int64 {
|
|
return 0
|
|
}
|
|
|
|
// Sys implements Filesystem.
|
|
func (r *ResolverFS) Sys() any {
|
|
return nil
|
|
}
|
|
|
|
// FsName implements Filesystem.
|
|
func (r *ResolverFS) FsName() string {
|
|
return "resolverfs"
|
|
}
|
|
|
|
func (fs *ResolverFS) traceAttrs(add ...attribute.KeyValue) trace.SpanStartOption {
|
|
return trace.WithAttributes(append([]attribute.KeyValue{
|
|
attribute.String("fs", fs.FsName()),
|
|
}, add...)...)
|
|
}
|
|
|
|
func (r *ResolverFS) ResolvablesExtensions() []string {
|
|
return maps.Keys(r.resolver.factories)
|
|
}
|
|
|
|
// Open implements Filesystem.
|
|
func (r *ResolverFS) Open(ctx context.Context, filename string) (File, error) {
|
|
ctx, span := tracer.Start(ctx, "Open",
|
|
r.traceAttrs(attribute.String("filename", filename)),
|
|
)
|
|
defer span.End()
|
|
|
|
if path.Clean(filename) == Separator {
|
|
return NewDirFile(r.Name()), nil
|
|
}
|
|
|
|
fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, filename, r.rootFS.Open)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if nestedFs != nil {
|
|
return nestedFs.Open(ctx, nestedFsPath)
|
|
}
|
|
|
|
return r.rootFS.Open(ctx, fsPath)
|
|
}
|
|
|
|
// ReadDir implements Filesystem.
|
|
func (r *ResolverFS) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) {
|
|
log := r.log.With(slog.String("name", name))
|
|
ctx, span := tracer.Start(ctx, "ReadDir",
|
|
r.traceAttrs(attribute.String("name", name)),
|
|
)
|
|
defer span.End()
|
|
|
|
fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, name, r.rootFS.Open)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if nestedFs != nil {
|
|
return nestedFs.ReadDir(ctx, nestedFsPath)
|
|
}
|
|
|
|
entries, err := r.rootFS.ReadDir(ctx, fsPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out, err := iter.MapErr(entries, func(pe *fs.DirEntry) (fs.DirEntry, error) {
|
|
e := *pe
|
|
if r.resolver.IsNestedFs(e.Name()) {
|
|
filepath := path.Join("/", name, e.Name())
|
|
file, err := r.rootFS.Open(ctx, filepath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
nestedfs, err := r.resolver.nestedFs(ctx, filepath, file)
|
|
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
|
return nil, err
|
|
}
|
|
if err != nil {
|
|
log.Error(ctx, "error creating nested fs", rlog.Error(err))
|
|
return nil, fmt.Errorf("error creating nested fs: %w", err)
|
|
}
|
|
return nestedfs, nil
|
|
} else {
|
|
return e, nil
|
|
}
|
|
})
|
|
|
|
if err != nil {
|
|
log.Error(ctx, "error mapping entries", rlog.Error(err))
|
|
err = nil
|
|
}
|
|
|
|
out = slices.DeleteFunc(out, func(e fs.DirEntry) bool { return e == nil })
|
|
|
|
return out, err
|
|
}
|
|
|
|
// Stat implements Filesystem.
|
|
func (r *ResolverFS) Stat(ctx context.Context, filename string) (fs.FileInfo, error) {
|
|
ctx, span := tracer.Start(ctx, "Stat",
|
|
r.traceAttrs(attribute.String("filename", filename)),
|
|
)
|
|
defer span.End()
|
|
|
|
if IsRoot(filename) {
|
|
return r, nil
|
|
}
|
|
|
|
fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, filename, r.rootFS.Open)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
span.SetAttributes(attribute.String("fsPath", fsPath), attribute.String("nestedFsPath", nestedFsPath))
|
|
|
|
if nestedFs != nil {
|
|
span.AddEvent("calling nested fs")
|
|
return nestedFs.Stat(ctx, nestedFsPath)
|
|
}
|
|
|
|
return r.rootFS.Stat(ctx, fsPath)
|
|
}
|
|
|
|
// Unlink implements Filesystem.
|
|
func (r *ResolverFS) Unlink(ctx context.Context, filename string) error {
|
|
fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, filename, r.rootFS.Open)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if nestedFs != nil {
|
|
return nestedFs.Unlink(ctx, nestedFsPath)
|
|
}
|
|
|
|
return r.rootFS.Unlink(ctx, fsPath)
|
|
}
|
|
|
|
// Rename implements Filesystem.
|
|
func (r *ResolverFS) Rename(ctx context.Context, oldpath string, newpath string) error {
|
|
oldFsPath, oldNestedFs, oldNestedFsPath, err := r.resolver.ResolvePath(ctx, oldpath, r.rootFS.Open)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
newFsPath, newNestedFs, newNestedFsPath, err := r.resolver.ResolvePath(ctx, newpath, r.rootFS.Open)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if oldNestedFs == nil && newNestedFs == nil {
|
|
return r.rootFS.Rename(ctx, oldFsPath, newFsPath)
|
|
}
|
|
|
|
fmt.Println(oldNestedFs)
|
|
fmt.Println(newNestedFs)
|
|
|
|
if oldNestedFs == nil || newNestedFs == nil || oldNestedFs == newNestedFs {
|
|
return oldNestedFs.Rename(ctx, oldNestedFsPath, newNestedFsPath)
|
|
}
|
|
|
|
return fmt.Errorf("rename between different nested filesystems is not supported")
|
|
}
|
|
|
|
// Info implements Filesystem.
|
|
func (r *ResolverFS) Info() (fs.FileInfo, error) {
|
|
return r, nil
|
|
}
|
|
|
|
// IsDir implements Filesystem.
|
|
func (r *ResolverFS) IsDir() bool {
|
|
return true
|
|
}
|
|
|
|
// Name implements Filesystem.
|
|
func (r *ResolverFS) Name() string {
|
|
return r.rootFS.Name()
|
|
}
|
|
|
|
// Type implements Filesystem.
|
|
func (r *ResolverFS) Type() fs.FileMode {
|
|
return fs.ModeDir
|
|
}
|
|
|
|
var _ Filesystem = &ResolverFS{}
|
|
|
|
// It factory responsobility to close file
|
|
type FsFactory func(ctx context.Context, sourcePath string, f File) (Filesystem, error)
|
|
|
|
func NewResolver(factories map[string]FsFactory) *Resolver {
|
|
return &Resolver{
|
|
factories: factories,
|
|
fsmap: map[string]Filesystem{},
|
|
}
|
|
}
|
|
|
|
type Resolver struct {
|
|
m sync.Mutex
|
|
factories map[string]FsFactory
|
|
fsmap map[string]Filesystem // filesystem cache
|
|
// TODO: add fsmap clean
|
|
}
|
|
|
|
type openFile func(ctx context.Context, path string) (File, error)
|
|
|
|
func (r *Resolver) IsNestedFs(f string) bool {
|
|
for ext := range r.factories {
|
|
if strings.HasSuffix(f, ext) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (r *Resolver) nestedFs(ctx context.Context, fsPath string, file File) (Filesystem, error) {
|
|
if file.IsDir() {
|
|
return nil, file.Close(ctx)
|
|
}
|
|
|
|
r.m.Lock()
|
|
defer r.m.Unlock()
|
|
|
|
if nestedFs, ok := r.fsmap[fsPath]; ok {
|
|
return nestedFs, file.Close(ctx)
|
|
}
|
|
|
|
for ext, nestFactory := range r.factories {
|
|
if !strings.HasSuffix(fsPath, ext) {
|
|
continue
|
|
}
|
|
|
|
nestedFs, err := nestFactory(ctx, fsPath, file)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error calling nest factory: %s with error: %w", fsPath, err)
|
|
}
|
|
r.fsmap[fsPath] = nestedFs
|
|
|
|
return nestedFs, nil
|
|
|
|
}
|
|
return nil, file.Close(ctx)
|
|
}
|
|
|
|
// open requeue raw open, without resolver call
|
|
func (r *Resolver) ResolvePath(ctx context.Context, name string, rawOpen openFile) (fsPath string, nestedFs Filesystem, nestedFsPath string, err error) {
|
|
ctx, span := tracer.Start(ctx, "ResolvePath")
|
|
defer span.End()
|
|
|
|
name = path.Clean(name)
|
|
name = strings.TrimPrefix(name, Separator)
|
|
parts := strings.Split(name, Separator)
|
|
|
|
nestOn := -1
|
|
var nestFactory FsFactory
|
|
|
|
PARTS_LOOP:
|
|
for i, part := range parts {
|
|
for ext, factory := range r.factories {
|
|
if strings.HasSuffix(part, ext) {
|
|
nestOn = i + 1
|
|
nestFactory = factory
|
|
break PARTS_LOOP
|
|
}
|
|
}
|
|
}
|
|
|
|
if nestOn == -1 {
|
|
return AbsPath(name), nil, "", nil
|
|
}
|
|
|
|
fsPath = AbsPath(path.Join(parts[:nestOn]...))
|
|
|
|
nestedFsPath = AbsPath(path.Join(parts[nestOn:]...))
|
|
|
|
file, err := rawOpen(ctx, fsPath)
|
|
if err != nil {
|
|
return "", nil, "", fmt.Errorf("error opening filesystem file: %s with error: %w", fsPath, err)
|
|
}
|
|
// fileHash, err := FileHash(ctx, file)
|
|
// if err != nil {
|
|
// return "", nil, "", fmt.Errorf("error calculating file hash: %w", err)
|
|
// }
|
|
err = file.Close(ctx)
|
|
if err != nil {
|
|
return "", nil, "", fmt.Errorf("error closing file: %w", err)
|
|
}
|
|
|
|
// we dont need lock until now
|
|
// it must be before fsmap read to exclude race condition:
|
|
// read -> write
|
|
// read -> write
|
|
r.m.Lock()
|
|
defer r.m.Unlock()
|
|
|
|
if nestedFs, ok := r.fsmap[fsPath]; ok {
|
|
span.AddEvent("fs loaded from cache", trace.WithAttributes(attribute.String("nestedFs", reflect.TypeOf(nestedFs).Name())))
|
|
return fsPath, nestedFs, nestedFsPath, nil
|
|
} else {
|
|
ctx, span := tracer.Start(ctx, "CreateFS")
|
|
defer span.End()
|
|
|
|
fsFile, err := rawOpen(ctx, fsPath)
|
|
if err != nil {
|
|
return "", nil, "", fmt.Errorf("error opening filesystem file: %s with error: %w", fsPath, err)
|
|
}
|
|
// it is factory responsibility to close file handler then needed
|
|
|
|
nestedFs, err := nestFactory(ctx, name, fsFile)
|
|
if err != nil {
|
|
return "", nil, "", fmt.Errorf("error creating filesystem from file: %s with error: %w", fsPath, err)
|
|
}
|
|
r.fsmap[fsPath] = nestedFs
|
|
|
|
span.AddEvent("fs created", trace.WithAttributes(attribute.String("nestedFs", reflect.TypeOf(nestedFs).Name())))
|
|
|
|
return fsPath, nestedFs, nestedFsPath, nil
|
|
}
|
|
|
|
}
|
|
|
|
var ErrNotExist = fs.ErrNotExist
|
|
|
|
func GetFile[F File](m map[string]F, name string) (File, error) {
|
|
if name == Separator {
|
|
return NewDirFile(name), nil
|
|
}
|
|
|
|
f, ok := m[name]
|
|
if ok {
|
|
return f, nil
|
|
}
|
|
|
|
for p := range m {
|
|
if strings.HasPrefix(p, name) {
|
|
return NewDirFile(name), nil
|
|
}
|
|
}
|
|
|
|
return nil, ErrNotExist
|
|
}
|
|
|
|
func ListDirFromFiles[F File](m map[string]F, name string) ([]fs.DirEntry, error) {
|
|
out := make([]fs.DirEntry, 0, len(m))
|
|
name = AddTrailSlash(path.Clean(name))
|
|
for p, f := range m {
|
|
if strings.HasPrefix(p, name) {
|
|
parts := strings.Split(trimRelPath(p, name), Separator)
|
|
if len(parts) == 1 {
|
|
out = append(out, NewFileInfo(parts[0], f.Size()))
|
|
} else {
|
|
out = append(out, NewDirInfo(parts[0]))
|
|
}
|
|
|
|
}
|
|
}
|
|
slices.SortStableFunc(out, func(de1, de2 fs.DirEntry) int {
|
|
return strings.Compare(de1.Name(), de2.Name())
|
|
})
|
|
out = slices.CompactFunc(out, func(de1, de2 fs.DirEntry) bool {
|
|
return de1.Name() == de2.Name()
|
|
})
|
|
|
|
return out, nil
|
|
}
|
|
|
|
func ListDirFromInfo(m map[string]fs.FileInfo, name string) ([]fs.DirEntry, error) {
|
|
out := make([]fs.DirEntry, 0, len(m))
|
|
name = AddTrailSlash(path.Clean(name))
|
|
for p, f := range m {
|
|
if strings.HasPrefix(p, name) {
|
|
parts := strings.Split(trimRelPath(p, name), Separator)
|
|
if len(parts) == 1 {
|
|
out = append(out, NewFileInfo(parts[0], f.Size()))
|
|
} else {
|
|
out = append(out, NewDirInfo(parts[0]))
|
|
}
|
|
|
|
}
|
|
}
|
|
slices.SortStableFunc(out, func(de1, de2 fs.DirEntry) int {
|
|
return strings.Compare(de1.Name(), de2.Name())
|
|
})
|
|
out = slices.CompactFunc(out, func(de1, de2 fs.DirEntry) bool {
|
|
return de1.Name() == de2.Name()
|
|
})
|
|
|
|
return out, nil
|
|
}
|