package vfs import ( "context" "errors" "fmt" "io/fs" "log/slog" "path" "reflect" "slices" "strings" "sync" "time" "git.kmsign.ru/royalcat/tstor/pkg/rlog" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "golang.org/x/exp/maps" ) type ResolverFS struct { rootFS Filesystem resolver *Resolver log *rlog.Logger } func NewResolveFS(rootFs Filesystem, factories map[string]FsFactory) *ResolverFS { return &ResolverFS{ rootFS: rootFs, resolver: NewResolver(factories), log: rlog.Component("fs.resolverfs"), } } // ModTime implements Filesystem. func (r *ResolverFS) ModTime() time.Time { return time.Time{} } // Mode implements Filesystem. func (r *ResolverFS) Mode() fs.FileMode { return fs.ModeDir } // Size implements Filesystem. func (r *ResolverFS) Size() int64 { return 0 } // Sys implements Filesystem. func (r *ResolverFS) Sys() any { return nil } // FsName implements Filesystem. func (r *ResolverFS) FsName() string { return "resolverfs" } func (fs *ResolverFS) traceAttrs(add ...attribute.KeyValue) trace.SpanStartOption { return trace.WithAttributes(append([]attribute.KeyValue{ attribute.String("fs", fs.FsName()), }, add...)...) } func (r *ResolverFS) ResolvablesExtensions() []string { return maps.Keys(r.resolver.factories) } // Open implements Filesystem. func (r *ResolverFS) Open(ctx context.Context, filename string) (File, error) { ctx, span := tracer.Start(ctx, "Open", r.traceAttrs(attribute.String("filename", filename)), ) defer span.End() if path.Clean(filename) == Separator { return NewDirFile(r.Name()), nil } fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, filename, r.rootFS.Open) if err != nil { return nil, err } if nestedFs != nil { return nestedFs.Open(ctx, nestedFsPath) } return r.rootFS.Open(ctx, fsPath) } // ReadDir implements Filesystem. func (r *ResolverFS) ReadDir(ctx context.Context, dir string) ([]fs.DirEntry, error) { ctx, span := tracer.Start(ctx, "ReadDir", r.traceAttrs(attribute.String("name", dir)), ) defer span.End() fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, dir, r.rootFS.Open) if err != nil { return nil, err } if nestedFs != nil { return nestedFs.ReadDir(ctx, nestedFsPath) } entries, err := r.rootFS.ReadDir(ctx, fsPath) if err != nil { return nil, err } out := make([]fs.DirEntry, 0, len(entries)) for _, e := range entries { if r.resolver.IsNestedFs(e.Name()) { filepath := path.Join("/", dir, e.Name()) file, err := r.Open(ctx, filepath) if err != nil { return nil, err } defer file.Close(ctx) err = func() error { factoryCtx, cancel := subTimeout(ctx) defer cancel() nestedfs, err := r.resolver.NestedFs(factoryCtx, filepath, file) if err != nil { if errors.Is(err, context.DeadlineExceeded) { r.log.Error(ctx, "creating fs timed out", slog.String("filename", e.Name()), ) return nil } return err } out = append(out, nestedfs) return nil }() if err != nil { return nil, err } } else { out = append(out, e) } } return out, nil } // Stat implements Filesystem. func (r *ResolverFS) Stat(ctx context.Context, filename string) (fs.FileInfo, error) { ctx, span := tracer.Start(ctx, "Stat", r.traceAttrs(attribute.String("filename", filename)), ) defer span.End() if IsRoot(filename) { return r, nil } fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, filename, r.rootFS.Open) if err != nil { return nil, err } span.SetAttributes(attribute.String("fsPath", fsPath), attribute.String("nestedFsPath", nestedFsPath)) if nestedFs != nil { span.AddEvent("calling nested fs") return nestedFs.Stat(ctx, nestedFsPath) } return r.rootFS.Stat(ctx, fsPath) } // Unlink implements Filesystem. func (r *ResolverFS) Unlink(ctx context.Context, filename string) error { fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, filename, r.rootFS.Open) if err != nil { return err } if nestedFs != nil { return nestedFs.Unlink(ctx, nestedFsPath) } return r.rootFS.Unlink(ctx, fsPath) } // Info implements Filesystem. func (r *ResolverFS) Info() (fs.FileInfo, error) { return r, nil } // IsDir implements Filesystem. func (r *ResolverFS) IsDir() bool { return true } // Name implements Filesystem. func (r *ResolverFS) Name() string { return r.rootFS.Name() } // Type implements Filesystem. func (r *ResolverFS) Type() fs.FileMode { return fs.ModeDir } var _ Filesystem = &ResolverFS{} type FsFactory func(ctx context.Context, f File) (Filesystem, error) func NewResolver(factories map[string]FsFactory) *Resolver { return &Resolver{ factories: factories, fsmap: map[string]Filesystem{}, } } type Resolver struct { m sync.Mutex factories map[string]FsFactory fsmap map[string]Filesystem // filesystem cache // TODO: add fsmap clean } type openFile func(ctx context.Context, path string) (File, error) func (r *Resolver) IsNestedFs(f string) bool { for ext := range r.factories { if strings.HasSuffix(f, ext) { return true } } return false } func (r *Resolver) NestedFs(ctx context.Context, fsPath string, file File) (Filesystem, error) { for ext, nestFactory := range r.factories { if !strings.HasSuffix(fsPath, ext) { continue } if nestedFs, ok := r.fsmap[fsPath]; ok { return nestedFs, nil } nestedFs, err := nestFactory(ctx, file) if err != nil { return nil, fmt.Errorf("error creating filesystem from file: %s with error: %w", fsPath, err) } r.fsmap[fsPath] = nestedFs return nestedFs, nil } return nil, nil } // open requeue raw open, without resolver call func (r *Resolver) ResolvePath(ctx context.Context, name string, rawOpen openFile) (fsPath string, nestedFs Filesystem, nestedFsPath string, err error) { ctx, span := tracer.Start(ctx, "ResolvePath") defer span.End() name = path.Clean(name) name = strings.TrimPrefix(name, Separator) parts := strings.Split(name, Separator) nestOn := -1 var nestFactory FsFactory PARTS_LOOP: for i, part := range parts { for ext, factory := range r.factories { if strings.HasSuffix(part, ext) { nestOn = i + 1 nestFactory = factory break PARTS_LOOP } } } if nestOn == -1 { return AbsPath(name), nil, "", nil } fsPath = AbsPath(path.Join(parts[:nestOn]...)) nestedFsPath = AbsPath(path.Join(parts[nestOn:]...)) // we dont need lock until now // it must be before fsmap read to exclude race condition: // read -> write // read -> write r.m.Lock() defer r.m.Unlock() if nestedFs, ok := r.fsmap[fsPath]; ok { span.AddEvent("fs loaded from cache", trace.WithAttributes(attribute.String("nestedFs", reflect.TypeOf(nestedFs).Name()))) return fsPath, nestedFs, nestedFsPath, nil } else { ctx, span := tracer.Start(ctx, "CreateFS") defer span.End() fsFile, err := rawOpen(ctx, fsPath) if err != nil { return "", nil, "", fmt.Errorf("error opening filesystem file: %s with error: %w", fsPath, err) } nestedFs, err := nestFactory(ctx, fsFile) if err != nil { return "", nil, "", fmt.Errorf("error creating filesystem from file: %s with error: %w", fsPath, err) } r.fsmap[fsPath] = nestedFs span.AddEvent("fs created", trace.WithAttributes(attribute.String("nestedFs", reflect.TypeOf(nestedFs).Name()))) return fsPath, nestedFs, nestedFsPath, nil } } var ErrNotExist = fs.ErrNotExist func GetFile[F File](m map[string]F, name string) (File, error) { if name == Separator { return NewDirFile(name), nil } f, ok := m[name] if ok { return f, nil } for p := range m { if strings.HasPrefix(p, name) { return NewDirFile(name), nil } } return nil, ErrNotExist } func ListDirFromFiles[F File](m map[string]F, name string) ([]fs.DirEntry, error) { out := make([]fs.DirEntry, 0, len(m)) name = AddTrailSlash(path.Clean(name)) for p, f := range m { if strings.HasPrefix(p, name) { parts := strings.Split(trimRelPath(p, name), Separator) if len(parts) == 1 { out = append(out, NewFileInfo(parts[0], f.Size())) } else { out = append(out, NewDirInfo(parts[0])) } } } slices.SortStableFunc(out, func(de1, de2 fs.DirEntry) int { return strings.Compare(de1.Name(), de2.Name()) }) out = slices.CompactFunc(out, func(de1, de2 fs.DirEntry) bool { return de1.Name() == de2.Name() }) return out, nil }