118 lines
2.5 KiB
Go
118 lines
2.5 KiB
Go
package kemono
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"io"
|
|
"path"
|
|
|
|
"github.com/thanos-io/objstore"
|
|
"github.com/thanos-io/objstore/providers/filesystem"
|
|
"golang.org/x/sync/errgroup"
|
|
|
|
"git.kmsign.ru/royalcat/tstor/plugins/kemono/kemonoapi"
|
|
)
|
|
|
|
const DaemonName = "kemono"
|
|
|
|
type Daemon struct {
|
|
coomerClient *kemonoapi.Client
|
|
kemonoClient *kemonoapi.Client
|
|
|
|
db *sql.DB
|
|
storage objstore.Bucket
|
|
}
|
|
|
|
type creator struct {
|
|
Service string
|
|
CreatorID string
|
|
}
|
|
|
|
func NewDaemon(dataDir string) (*Daemon, error) {
|
|
bucket, err := filesystem.NewBucket(dataDir)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create filesystem bucket: %w", err)
|
|
}
|
|
|
|
return &Daemon{
|
|
coomerClient: kemonoapi.NewClient("https://coomer.su/"),
|
|
kemonoClient: kemonoapi.NewClient("https://kemono.su/"),
|
|
|
|
storage: bucket,
|
|
}, nil
|
|
|
|
}
|
|
|
|
func (d *Daemon) getClient(service string) *kemonoapi.Client {
|
|
switch service {
|
|
case "onlyfans", "fansly", "candfans":
|
|
return d.coomerClient
|
|
case "patreon", "fanbox", "fantia", "gumroad", "discord", "boosty", "subscribestar", "dlsite", "afdian":
|
|
return d.kemonoClient
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func getCreatorPath(creator creator) string {
|
|
return path.Join(creator.Service, creator.CreatorID)
|
|
}
|
|
|
|
func (d *Daemon) scrapCreator(ctx context.Context, creator creator) error {
|
|
client := d.getClient(creator.Service)
|
|
if client == nil {
|
|
return fmt.Errorf("no site for service %s", creator.Service)
|
|
}
|
|
|
|
posts := client.FetchPosts(ctx, creator.Service, creator.CreatorID)
|
|
for post, err := range posts {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, att := range append([]kemonoapi.File{post.File}, post.Attachments...) {
|
|
err := d.downloadFile(ctx, client, att)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to download file: %w", err)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func getStorageFilePath(file kemonoapi.File) string {
|
|
return path.Join("data", file.Path)
|
|
}
|
|
|
|
func (d *Daemon) downloadFile(ctx context.Context, client *kemonoapi.Client, file kemonoapi.File) error {
|
|
info, err := client.HeadFile(ctx, path.Join("data", file.Path))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get file info: %w", err)
|
|
}
|
|
|
|
storageFilePath := getStorageFilePath(file)
|
|
|
|
attrs, err := d.storage.Attributes(ctx, storageFilePath)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
if attrs.Size == info.Length && attrs.LastModified.After(info.LastModified) {
|
|
return nil
|
|
}
|
|
|
|
r, w := io.Pipe()
|
|
var g errgroup.Group
|
|
g.Go(func() error {
|
|
defer w.Close()
|
|
return client.DownloadFile(ctx, w, info.URL)
|
|
})
|
|
g.Go(func() error {
|
|
return d.storage.Upload(ctx, storageFilePath, r)
|
|
})
|
|
return g.Wait()
|
|
|
|
}
|