delete unrelated plugin
This commit is contained in:
parent
237af4a63e
commit
4d35c4b7e3
11 changed files with 0 additions and 662 deletions
plugins/kemono
|
|
@ -1,6 +0,0 @@
|
|||
# yaml-language-server: $schema=https://raw.githubusercontent.com/oapi-codegen/oapi-codegen/HEAD/configuration-schema.json
|
||||
package: client
|
||||
output: genclient/client.gen.go
|
||||
generate:
|
||||
models: true
|
||||
client: true
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
package kemono
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
|
||||
"github.com/thanos-io/objstore"
|
||||
"github.com/thanos-io/objstore/providers/filesystem"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/plugins/kemono/kemonoapi"
|
||||
)
|
||||
|
||||
const DaemonName = "kemono"
|
||||
|
||||
type Daemon struct {
|
||||
coomerClient *kemonoapi.Client
|
||||
kemonoClient *kemonoapi.Client
|
||||
|
||||
db *sql.DB
|
||||
storage objstore.Bucket
|
||||
}
|
||||
|
||||
type creator struct {
|
||||
Service string
|
||||
CreatorID string
|
||||
}
|
||||
|
||||
func NewDaemon(dataDir string) (*Daemon, error) {
|
||||
bucket, err := filesystem.NewBucket(dataDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create filesystem bucket: %w", err)
|
||||
}
|
||||
|
||||
return &Daemon{
|
||||
coomerClient: kemonoapi.NewClient("https://coomer.su/"),
|
||||
kemonoClient: kemonoapi.NewClient("https://kemono.su/"),
|
||||
|
||||
storage: bucket,
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
func (d *Daemon) getClient(service string) *kemonoapi.Client {
|
||||
switch service {
|
||||
case "onlyfans", "fansly", "candfans":
|
||||
return d.coomerClient
|
||||
case "patreon", "fanbox", "fantia", "gumroad", "discord", "boosty", "subscribestar", "dlsite", "afdian":
|
||||
return d.kemonoClient
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getCreatorPath(creator creator) string {
|
||||
return path.Join(creator.Service, creator.CreatorID)
|
||||
}
|
||||
|
||||
func (d *Daemon) scrapCreator(ctx context.Context, creator creator) error {
|
||||
client := d.getClient(creator.Service)
|
||||
if client == nil {
|
||||
return fmt.Errorf("no site for service %s", creator.Service)
|
||||
}
|
||||
|
||||
posts := client.FetchPosts(ctx, creator.Service, creator.CreatorID)
|
||||
for post, err := range posts {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, att := range append([]kemonoapi.File{post.File}, post.Attachments...) {
|
||||
err := d.downloadFile(ctx, client, att)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getStorageFilePath(file kemonoapi.File) string {
|
||||
return path.Join("data", file.Path)
|
||||
}
|
||||
|
||||
func (d *Daemon) downloadFile(ctx context.Context, client *kemonoapi.Client, file kemonoapi.File) error {
|
||||
info, err := client.HeadFile(ctx, path.Join("data", file.Path))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get file info: %w", err)
|
||||
}
|
||||
|
||||
storageFilePath := getStorageFilePath(file)
|
||||
|
||||
attrs, err := d.storage.Attributes(ctx, storageFilePath)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if attrs.Size == info.Length && attrs.LastModified.After(info.LastModified) {
|
||||
return nil
|
||||
}
|
||||
|
||||
r, w := io.Pipe()
|
||||
var g errgroup.Group
|
||||
g.Go(func() error {
|
||||
defer w.Close()
|
||||
return client.DownloadFile(ctx, w, info.URL)
|
||||
})
|
||||
g.Go(func() error {
|
||||
return d.storage.Upload(ctx, storageFilePath, r)
|
||||
})
|
||||
return g.Wait()
|
||||
|
||||
}
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
module git.kmsign.ru/royalcat/tstor/plugins/kemono
|
||||
|
||||
go 1.23.5
|
||||
|
||||
require (
|
||||
github.com/go-resty/resty/v2 v2.16.2
|
||||
github.com/spf13/cast v1.7.0
|
||||
github.com/thanos-io/objstore v0.0.0-20241212213936-d69df7208cba
|
||||
golang.org/x/sync v0.12.0
|
||||
golang.org/x/time v0.6.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 // indirect
|
||||
github.com/go-kit/log v0.2.1 // indirect
|
||||
github.com/go-logfmt/logfmt v0.5.1 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_golang v1.20.4 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.60.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.13.1 // indirect
|
||||
github.com/stretchr/testify v1.10.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
golang.org/x/net v0.37.0 // indirect
|
||||
golang.org/x/sys v0.31.0 // indirect
|
||||
google.golang.org/protobuf v1.36.5 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
)
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 h1:rydBwnBoywKQMjWF0z8SriYtQ+uUcaFsxuijMjJr5PI=
|
||||
github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4/go.mod h1:kQa0V74HNYMfuJH6jiPiwNdpWXl4xd/K4tzlrcvYDQI=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
|
||||
github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
|
||||
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
|
||||
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-resty/resty/v2 v2.16.2 h1:CpRqTjIzq/rweXUt9+GxzzQdlkqMdt8Lm/fuK/CAbAg=
|
||||
github.com/go-resty/resty/v2 v2.16.2/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI=
|
||||
github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
|
||||
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||
github.com/prometheus/common v0.60.0 h1:+V9PAREWNvJMAuJ1x1BaWl9dewMW4YrHZQbx0sJNllA=
|
||||
github.com/prometheus/common v0.60.0/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw=
|
||||
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w=
|
||||
github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/thanos-io/objstore v0.0.0-20241212213936-d69df7208cba h1:X5YtKhjFsMAgfaD1MxT+hYrP9QftK9iA+UYS3eQW0E0=
|
||||
github.com/thanos-io/objstore v0.0.0-20241212213936-d69df7208cba/go.mod h1:vyzFrBXgP+fGNG2FopEGWOO/zrIuoy7zt3LpLeezRsw=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
|
||||
golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
|
||||
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
|
||||
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
|
||||
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
package kemonoapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"git.kmsign.ru/royalcat/tstor/plugins/kemono/kemonoapi"
|
||||
)
|
||||
|
||||
func TestScrapCreator(t *testing.T) {
|
||||
k := kemonoapi.NewClient("https://coomer.su/")
|
||||
ctx := context.Background()
|
||||
posts := k.FetchPosts(ctx, "onlyfans", "bigtittygothegg")
|
||||
|
||||
for post, err := range posts {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if post.ID == "" {
|
||||
t.Fatal(fmt.Errorf("post id is empty"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
package kemonoapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fileInfo struct {
|
||||
URL string
|
||||
Length int64
|
||||
ContentType string
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
func (c *Client) HeadFile(ctx context.Context, url string) (*fileInfo, error) {
|
||||
resp, err := c.client.R().SetContext(ctx).Head(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to download url %s: %w", url, err)
|
||||
}
|
||||
|
||||
loc := resp.Header().Get("Location")
|
||||
if loc != "" {
|
||||
return c.HeadFile(ctx, loc)
|
||||
}
|
||||
|
||||
length, err := strconv.ParseInt(resp.Header().Get("Content-Length"), 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse Content-Length header: %w", err)
|
||||
}
|
||||
|
||||
lastModified, err := time.Parse(time.RFC1123, resp.Header().Get("Last-Modified"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse Last-Modified header: %w", err)
|
||||
}
|
||||
|
||||
contentType := resp.Header().Get("Content-Type")
|
||||
|
||||
return &fileInfo{
|
||||
URL: url,
|
||||
Length: length,
|
||||
ContentType: contentType,
|
||||
LastModified: lastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) DownloadFile(ctx context.Context, out io.Writer, url string) error {
|
||||
resp, err := c.client.R().SetContext(ctx).SetDoNotParseResponse(true).Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download url %s: %w", url, err)
|
||||
}
|
||||
|
||||
body := resp.RawBody()
|
||||
defer body.Close()
|
||||
|
||||
_, err = io.Copy(out, body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download url %s: %w", url, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
package kemonoapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"iter"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// FetchCreators fetch Creator list
|
||||
func (k *Client) FetchCreators() (creators []Creator, err error) {
|
||||
|
||||
// k.log.Print("fetching creator list...")
|
||||
// url := fmt.Sprintf("https://%s/api/v1/creators", k.Site)
|
||||
// resp, err := k.Downloader.Get(url)
|
||||
// if err != nil {
|
||||
// return nil, fmt.Errorf("fetch creator list error: %s", err)
|
||||
// }
|
||||
|
||||
// reader, err := handleCompressedHTTPResponse(resp)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
// data, err := ioutil.ReadAll(reader)
|
||||
// if err != nil {
|
||||
// return nil, fmt.Errorf("fetch creator list error: %s", err)
|
||||
// }
|
||||
// err = json.Unmarshal(data, &creators)
|
||||
// if err != nil {
|
||||
// return nil, fmt.Errorf("unmarshal creator list error: %s", err)
|
||||
// }
|
||||
return
|
||||
}
|
||||
|
||||
// FetchPosts fetch post list
|
||||
func (k *Client) FetchPosts(ctx context.Context, service, creator_id string) iter.Seq2[Post, error] {
|
||||
const perUnit = 50
|
||||
|
||||
return func(yield func(Post, error) bool) {
|
||||
|
||||
page := 0
|
||||
|
||||
for {
|
||||
k.log.Info("fetching post list", slog.Int("page", page))
|
||||
|
||||
if err := k.ratelimit.Wait(ctx); err != nil {
|
||||
yield(Post{}, err)
|
||||
return
|
||||
}
|
||||
|
||||
posts, err := k.fetchPostsPage(ctx, service, creator_id, page*perUnit)
|
||||
if err != nil {
|
||||
yield(Post{}, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(posts) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, post := range posts {
|
||||
if !yield(post, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
page++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Client) fetchPostsPage(ctx context.Context, service, creator_id string, offset int) ([]Post, error) {
|
||||
resp, err := k.client.R().
|
||||
SetContext(ctx).
|
||||
SetQueryParam("o", strconv.Itoa(offset)).
|
||||
SetPathParam("service", service).
|
||||
SetPathParam("creator_id", creator_id).
|
||||
Get("/{service}/user/{creator_id}")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch post list error: %s", err)
|
||||
}
|
||||
|
||||
var posts []Post
|
||||
err = json.Unmarshal(resp.Body(), &posts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unmarshal post list error: %s", err)
|
||||
}
|
||||
return posts, nil
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
package kemonoapi
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type Downloader interface {
|
||||
Download(<-chan FileWithIndex, Creator, Post) <-chan error
|
||||
Get(url string) (resp *http.Response, err error)
|
||||
WriteContent(Creator, Post, string) error
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
client *resty.Client
|
||||
ratelimit *rate.Limiter
|
||||
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
func NewClient(site string) *Client {
|
||||
k := &Client{
|
||||
ratelimit: rate.NewLimiter(rate.Every(time.Second), 3),
|
||||
client: resty.New().
|
||||
SetBaseURL(site + "/api/v1").
|
||||
SetRetryCount(3).
|
||||
SetRetryWaitTime(5 * time.Second).
|
||||
AddRetryCondition(func(r *resty.Response, err error) bool {
|
||||
return r != nil && r.StatusCode() == http.StatusTooManyRequests
|
||||
}),
|
||||
}
|
||||
if k.log == nil {
|
||||
k.log = slog.Default()
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
|
@ -1,186 +0,0 @@
|
|||
package kemonoapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
type Timestamp struct {
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
func (t *Timestamp) UnmarshalJSON(b []byte) error {
|
||||
var timestamp float64
|
||||
err := json.Unmarshal(b, ×tamp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.Time = time.Unix(int64(timestamp), int64((timestamp-float64(int64(timestamp)))*1e9))
|
||||
return nil
|
||||
}
|
||||
|
||||
type Creator struct {
|
||||
Favorited int `json:"favorited"`
|
||||
Id string `json:"id"`
|
||||
Indexed Timestamp `json:"indexed"`
|
||||
Name string `json:"name"`
|
||||
Service string `json:"service"`
|
||||
Updated Timestamp `json:"updated"`
|
||||
}
|
||||
|
||||
// GetID get creator id
|
||||
func (c Creator) GetID() string {
|
||||
return c.Id
|
||||
}
|
||||
|
||||
// GetService get creator Service
|
||||
func (c Creator) GetService() string {
|
||||
return c.Service
|
||||
}
|
||||
|
||||
func (c Creator) PairString() string {
|
||||
return fmt.Sprintf("%s:%s", c.Service, c.Id)
|
||||
}
|
||||
|
||||
func NewCreator(service, id string) Creator {
|
||||
return Creator{
|
||||
Service: service,
|
||||
Id: id,
|
||||
}
|
||||
}
|
||||
|
||||
// FindCreator Get the Creator by ID and Service
|
||||
func FindCreator(creators []Creator, id, service string) (Creator, bool) {
|
||||
for _, creator := range creators {
|
||||
if creator.Id == id && creator.Service == service {
|
||||
return creator, true
|
||||
}
|
||||
}
|
||||
return Creator{}, false
|
||||
}
|
||||
|
||||
type File struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
// GetURL return the url
|
||||
func (f File) GetURL() string {
|
||||
ext := filepath.Ext(f.Name)
|
||||
name := f.Name[:len(f.Name)-len(ext)]
|
||||
return fmt.Sprintf("%s?f=%s%s", f.Path, url.QueryEscape(name), ext)
|
||||
}
|
||||
|
||||
// GetHash get hash from file path
|
||||
func (f File) GetHash() (string, error) {
|
||||
return SplitHash(f.Path)
|
||||
}
|
||||
|
||||
func (f File) Index(n int) FileWithIndex {
|
||||
return FileWithIndex{
|
||||
Index: n,
|
||||
File: f,
|
||||
}
|
||||
}
|
||||
|
||||
type FileWithIndex struct {
|
||||
Index int
|
||||
File
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
}
|
||||
|
||||
type Author struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Avatar string `json:"avatar,omitempty"`
|
||||
Discriminator string `json:"discriminator,omitempty"`
|
||||
PublicFlags int64 `json:"public_flags,omitempty"`
|
||||
Username string `json:"username,omitempty"`
|
||||
}
|
||||
|
||||
type Post struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
User string `json:"user,omitempty"`
|
||||
Service string `json:"service,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
// Embed []any `json:"embed,omitempty"`
|
||||
SharedFile bool `json:"shared_file,omitempty"`
|
||||
|
||||
Added Time `json:"added,omitempty"`
|
||||
Published Time `json:"published,omitempty"`
|
||||
Edited Time `json:"edited,omitempty"`
|
||||
|
||||
File File `json:"file,omitempty"`
|
||||
Attachments []File `json:"attachments,omitempty"`
|
||||
}
|
||||
|
||||
type Time time.Time
|
||||
|
||||
func (t *Time) UnmarshalJSON(b []byte) error {
|
||||
var timestamp string
|
||||
err := json.Unmarshal(b, ×tamp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if timestamp == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
parsed, err := cast.StringToDate(timestamp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*t = Time(parsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// User a creator according to the service and id
|
||||
type User struct {
|
||||
Service string `json:"service"`
|
||||
Id string `json:"id"`
|
||||
}
|
||||
|
||||
// GetID get user id
|
||||
func (u User) GetID() string {
|
||||
return u.Id
|
||||
}
|
||||
|
||||
// GetService get user Service
|
||||
func (u User) GetService() string {
|
||||
return u.Service
|
||||
}
|
||||
|
||||
type FavoriteCreator struct {
|
||||
FavedSeq int `json:"faved_seq"`
|
||||
Id string `json:"id"`
|
||||
Index string `json:"index"`
|
||||
Name string `json:"name"`
|
||||
Service string `json:"service"`
|
||||
Update string `json:"update"`
|
||||
}
|
||||
|
||||
var SiteMap = map[string]string{
|
||||
"patreon": "kemono",
|
||||
"fanbox": "kemono",
|
||||
"gumroad": "kemono",
|
||||
"subscribestar": "kemono",
|
||||
"dlsite": "kemono",
|
||||
"discord": "kemono",
|
||||
"fantia": "kemono",
|
||||
"boosty": "kemono",
|
||||
"afdian": "kemono",
|
||||
"onlyfans": "coomer",
|
||||
"fansly": "coomer",
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
package kemonoapi
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func isImage(ext string) bool {
|
||||
switch ext {
|
||||
case ".apng", ".avif", ".bmp", ".gif", ".ico", ".cur", ".jpg", ".jpeg", ".jfif", ".pjpeg", ".pjp", ".png", ".svg", ".tif", ".tiff", ".webp", ".jpe":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func SplitHash(str string) (string, error) {
|
||||
parts := strings.Split(str, "/")
|
||||
if len(parts) < 4 {
|
||||
return "", nil
|
||||
}
|
||||
ext := filepath.Ext(parts[3])
|
||||
name := parts[3][:len(parts[3])-len(ext)]
|
||||
return name, nil
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
package main
|
||||
|
||||
import "git.kmsign.ru/royalcat/tstor/plugins/kemono"
|
||||
|
||||
func main() {
|
||||
}
|
||||
|
||||
const DaemonName = kemono.DaemonName
|
||||
|
||||
var NewDaemon = kemono.NewDaemon
|
||||
Loading…
Reference in a new issue