package kemono import ( "context" "database/sql" "fmt" "io" "path" "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/providers/filesystem" "golang.org/x/sync/errgroup" "git.kmsign.ru/royalcat/tstor/plugins/kemono/kemonoapi" ) const DaemonName = "kemono" type Daemon struct { coomerClient *kemonoapi.Client kemonoClient *kemonoapi.Client db *sql.DB storage objstore.Bucket } type creator struct { Service string CreatorID string } func NewDaemon(dataDir string) (*Daemon, error) { bucket, err := filesystem.NewBucket(dataDir) if err != nil { return nil, fmt.Errorf("failed to create filesystem bucket: %w", err) } return &Daemon{ coomerClient: kemonoapi.NewClient("https://coomer.su/"), kemonoClient: kemonoapi.NewClient("https://kemono.su/"), storage: bucket, }, nil } func (d *Daemon) getClient(service string) *kemonoapi.Client { switch service { case "onlyfans", "fansly", "candfans": return d.coomerClient case "patreon", "fanbox", "fantia", "gumroad", "discord", "boosty", "subscribestar", "dlsite", "afdian": return d.kemonoClient } return nil } func getCreatorPath(creator creator) string { return path.Join(creator.Service, creator.CreatorID) } func (d *Daemon) scrapCreator(ctx context.Context, creator creator) error { client := d.getClient(creator.Service) if client == nil { return fmt.Errorf("no site for service %s", creator.Service) } posts := client.FetchPosts(ctx, creator.Service, creator.CreatorID) for post, err := range posts { if err != nil { return err } for _, att := range append([]kemonoapi.File{post.File}, post.Attachments...) { err := d.downloadFile(ctx, client, att) if err != nil { return fmt.Errorf("failed to download file: %w", err) } } } return nil } func getStorageFilePath(file kemonoapi.File) string { return path.Join("data", file.Path) } func (d *Daemon) downloadFile(ctx context.Context, client *kemonoapi.Client, file kemonoapi.File) error { info, err := client.HeadFile(ctx, path.Join("data", file.Path)) if err != nil { return fmt.Errorf("failed to get file info: %w", err) } storageFilePath := getStorageFilePath(file) attrs, err := d.storage.Attributes(ctx, storageFilePath) if err == nil { return nil } if attrs.Size == info.Length && attrs.LastModified.After(info.LastModified) { return nil } r, w := io.Pipe() var g errgroup.Group g.Go(func() error { defer w.Close() return client.DownloadFile(ctx, w, info.URL) }) g.Go(func() error { return d.storage.Upload(ctx, storageFilePath, r) }) return g.Wait() }