package vfs import ( "context" "errors" "fmt" "io/fs" "log/slog" "path" "reflect" "slices" "strings" "sync" "time" "git.kmsign.ru/royalcat/tstor/pkg/rlog" "github.com/sourcegraph/conc/iter" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "golang.org/x/exp/maps" ) type ResolverFS struct { rootFS Filesystem resolver *Resolver log *rlog.Logger } func NewResolveFS(rootFs Filesystem, factories map[string]FsFactory) *ResolverFS { return &ResolverFS{ rootFS: rootFs, resolver: NewResolver(factories), log: rlog.Component("fs.resolverfs"), } } // ModTime implements Filesystem. func (r *ResolverFS) ModTime() time.Time { return time.Time{} } // Mode implements Filesystem. func (r *ResolverFS) Mode() fs.FileMode { return fs.ModeDir } // Size implements Filesystem. func (r *ResolverFS) Size() int64 { return 0 } // Sys implements Filesystem. func (r *ResolverFS) Sys() any { return nil } // FsName implements Filesystem. func (r *ResolverFS) FsName() string { return "resolverfs" } func (fs *ResolverFS) traceAttrs(add ...attribute.KeyValue) trace.SpanStartOption { return trace.WithAttributes(append([]attribute.KeyValue{ attribute.String("fs", fs.FsName()), }, add...)...) } func (r *ResolverFS) ResolvablesExtensions() []string { return maps.Keys(r.resolver.factories) } // Open implements Filesystem. func (r *ResolverFS) Open(ctx context.Context, filename string) (File, error) { ctx, span := tracer.Start(ctx, "Open", r.traceAttrs(attribute.String("filename", filename)), ) defer span.End() if path.Clean(filename) == Separator { return NewDirFile(r.Name()), nil } fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, filename, r.rootFS.Open) if err != nil { return nil, err } if nestedFs != nil { return nestedFs.Open(ctx, nestedFsPath) } return r.rootFS.Open(ctx, fsPath) } // ReadDir implements Filesystem. func (r *ResolverFS) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { log := r.log.With(slog.String("name", name)) ctx, span := tracer.Start(ctx, "ReadDir", r.traceAttrs(attribute.String("name", name)), ) defer span.End() fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, name, r.rootFS.Open) if err != nil { return nil, err } if nestedFs != nil { return nestedFs.ReadDir(ctx, nestedFsPath) } entries, err := r.rootFS.ReadDir(ctx, fsPath) if err != nil { return nil, err } out, err := iter.MapErr(entries, func(pe *fs.DirEntry) (fs.DirEntry, error) { e := *pe if r.resolver.IsNestedFs(e.Name()) { filepath := path.Join("/", name, e.Name()) file, err := r.rootFS.Open(ctx, filepath) if err != nil { return nil, err } nestedfs, err := r.resolver.nestedFs(ctx, filepath, file) if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { return nil, err } if err != nil { log.Error(ctx, "error creating nested fs", rlog.Error(err)) return nil, fmt.Errorf("error creating nested fs: %w", err) } return nestedfs, nil } else { return e, nil } }) if err != nil { log.Error(ctx, "error mapping entries", rlog.Error(err)) err = nil } out = slices.DeleteFunc(out, func(e fs.DirEntry) bool { return e == nil }) return out, err } // Stat implements Filesystem. func (r *ResolverFS) Stat(ctx context.Context, filename string) (fs.FileInfo, error) { ctx, span := tracer.Start(ctx, "Stat", r.traceAttrs(attribute.String("filename", filename)), ) defer span.End() if IsRoot(filename) { return r, nil } fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, filename, r.rootFS.Open) if err != nil { return nil, err } span.SetAttributes(attribute.String("fsPath", fsPath), attribute.String("nestedFsPath", nestedFsPath)) if nestedFs != nil { span.AddEvent("calling nested fs") return nestedFs.Stat(ctx, nestedFsPath) } return r.rootFS.Stat(ctx, fsPath) } // Unlink implements Filesystem. func (r *ResolverFS) Unlink(ctx context.Context, filename string) error { fsPath, nestedFs, nestedFsPath, err := r.resolver.ResolvePath(ctx, filename, r.rootFS.Open) if err != nil { return err } if nestedFs != nil { return nestedFs.Unlink(ctx, nestedFsPath) } return r.rootFS.Unlink(ctx, fsPath) } // Rename implements Filesystem. func (r *ResolverFS) Rename(ctx context.Context, oldpath string, newpath string) error { oldFsPath, oldNestedFs, oldNestedFsPath, err := r.resolver.ResolvePath(ctx, oldpath, r.rootFS.Open) if err != nil { return err } newFsPath, newNestedFs, newNestedFsPath, err := r.resolver.ResolvePath(ctx, newpath, r.rootFS.Open) if err != nil { return err } if oldNestedFs == nil && newNestedFs == nil { return r.rootFS.Rename(ctx, oldFsPath, newFsPath) } fmt.Println(oldNestedFs) fmt.Println(newNestedFs) if oldNestedFs == nil || newNestedFs == nil || oldNestedFs == newNestedFs { return oldNestedFs.Rename(ctx, oldNestedFsPath, newNestedFsPath) } return fmt.Errorf("rename between different nested filesystems is not supported") } // Info implements Filesystem. func (r *ResolverFS) Info() (fs.FileInfo, error) { return r, nil } // IsDir implements Filesystem. func (r *ResolverFS) IsDir() bool { return true } // Name implements Filesystem. func (r *ResolverFS) Name() string { return r.rootFS.Name() } // Type implements Filesystem. func (r *ResolverFS) Type() fs.FileMode { return fs.ModeDir } var _ Filesystem = &ResolverFS{} // It factory responsobility to close file type FsFactory func(ctx context.Context, sourcePath string, f File) (Filesystem, error) func NewResolver(factories map[string]FsFactory) *Resolver { return &Resolver{ factories: factories, fsmap: map[string]Filesystem{}, } } type Resolver struct { m sync.Mutex factories map[string]FsFactory fsmap map[string]Filesystem // filesystem cache // TODO: add fsmap clean } type openFile func(ctx context.Context, path string) (File, error) func (r *Resolver) IsNestedFs(f string) bool { for ext := range r.factories { if strings.HasSuffix(f, ext) { return true } } return false } func (r *Resolver) nestedFs(ctx context.Context, fsPath string, file File) (Filesystem, error) { if file.IsDir() { return nil, file.Close(ctx) } r.m.Lock() defer r.m.Unlock() if nestedFs, ok := r.fsmap[fsPath]; ok { return nestedFs, file.Close(ctx) } for ext, nestFactory := range r.factories { if !strings.HasSuffix(fsPath, ext) { continue } nestedFs, err := nestFactory(ctx, fsPath, file) if err != nil { return nil, fmt.Errorf("error calling nest factory: %s with error: %w", fsPath, err) } r.fsmap[fsPath] = nestedFs return nestedFs, nil } return nil, file.Close(ctx) } // open requeue raw open, without resolver call func (r *Resolver) ResolvePath(ctx context.Context, name string, rawOpen openFile) (fsPath string, nestedFs Filesystem, nestedFsPath string, err error) { ctx, span := tracer.Start(ctx, "ResolvePath") defer span.End() name = path.Clean(name) name = strings.TrimPrefix(name, Separator) parts := strings.Split(name, Separator) nestOn := -1 var nestFactory FsFactory PARTS_LOOP: for i, part := range parts { for ext, factory := range r.factories { if strings.HasSuffix(part, ext) { nestOn = i + 1 nestFactory = factory break PARTS_LOOP } } } if nestOn == -1 { return AbsPath(name), nil, "", nil } fsPath = AbsPath(path.Join(parts[:nestOn]...)) nestedFsPath = AbsPath(path.Join(parts[nestOn:]...)) file, err := rawOpen(ctx, fsPath) if err != nil { return "", nil, "", fmt.Errorf("error opening filesystem file: %s with error: %w", fsPath, err) } // fileHash, err := FileHash(ctx, file) // if err != nil { // return "", nil, "", fmt.Errorf("error calculating file hash: %w", err) // } err = file.Close(ctx) if err != nil { return "", nil, "", fmt.Errorf("error closing file: %w", err) } // we dont need lock until now // it must be before fsmap read to exclude race condition: // read -> write // read -> write r.m.Lock() defer r.m.Unlock() if nestedFs, ok := r.fsmap[fsPath]; ok { span.AddEvent("fs loaded from cache", trace.WithAttributes(attribute.String("nestedFs", reflect.TypeOf(nestedFs).Name()))) return fsPath, nestedFs, nestedFsPath, nil } else { ctx, span := tracer.Start(ctx, "CreateFS") defer span.End() fsFile, err := rawOpen(ctx, fsPath) if err != nil { return "", nil, "", fmt.Errorf("error opening filesystem file: %s with error: %w", fsPath, err) } // it is factory responsibility to close file handler then needed nestedFs, err := nestFactory(ctx, name, fsFile) if err != nil { return "", nil, "", fmt.Errorf("error creating filesystem from file: %s with error: %w", fsPath, err) } r.fsmap[fsPath] = nestedFs span.AddEvent("fs created", trace.WithAttributes(attribute.String("nestedFs", reflect.TypeOf(nestedFs).Name()))) return fsPath, nestedFs, nestedFsPath, nil } } var ErrNotExist = fs.ErrNotExist func GetFile[F File](m map[string]F, name string) (File, error) { if name == Separator { return NewDirFile(name), nil } f, ok := m[name] if ok { return f, nil } for p := range m { if strings.HasPrefix(p, name) { return NewDirFile(name), nil } } return nil, ErrNotExist } func ListDirFromFiles[F File](m map[string]F, name string) ([]fs.DirEntry, error) { out := make([]fs.DirEntry, 0, len(m)) name = AddTrailSlash(path.Clean(name)) for p, f := range m { if strings.HasPrefix(p, name) { parts := strings.Split(trimRelPath(p, name), Separator) if len(parts) == 1 { out = append(out, NewFileInfo(parts[0], f.Size())) } else { out = append(out, NewDirInfo(parts[0])) } } } slices.SortStableFunc(out, func(de1, de2 fs.DirEntry) int { return strings.Compare(de1.Name(), de2.Name()) }) out = slices.CompactFunc(out, func(de1, de2 fs.DirEntry) bool { return de1.Name() == de2.Name() }) return out, nil } func ListDirFromInfo(m map[string]fs.FileInfo, name string) ([]fs.DirEntry, error) { out := make([]fs.DirEntry, 0, len(m)) name = AddTrailSlash(path.Clean(name)) for p, f := range m { if strings.HasPrefix(p, name) { parts := strings.Split(trimRelPath(p, name), Separator) if len(parts) == 1 { out = append(out, NewFileInfo(parts[0], f.Size())) } else { out = append(out, NewDirInfo(parts[0])) } } } slices.SortStableFunc(out, func(de1, de2 fs.DirEntry) int { return strings.Compare(de1.Name(), de2.Name()) }) out = slices.CompactFunc(out, func(de1, de2 fs.DirEntry) bool { return de1.Name() == de2.Name() }) return out, nil }