package kemono import ( "log" "net/http" "os" "time" ) type Downloader interface { Download(<-chan FileWithIndex, Creator, Post) <-chan error Get(url string) (resp *http.Response, err error) WriteContent(Creator, Post, string) error } type Log interface { Printf(format string, v ...interface{}) Print(s string) } type DefaultLog struct { log *log.Logger } func (d *DefaultLog) Printf(format string, v ...interface{}) { d.log.Printf(format, v...) } func (d *DefaultLog) Print(s string) { d.log.Print(s) } // Filter return true for continue, false for skip type CreatorFilter func(i int, post Creator) bool type PostFilter func(i int, post Post) bool type AttachmentFilter func(i int, attachment File) bool type Option func(*kemono) type kemono struct { // kemono or coomer ... Site string //download Banner Banner bool // All Creator creators []Creator // Creator filter creatorFilters []CreatorFilter // Post filter map[creator(:)][]PostFilter postFilters map[string][]PostFilter // Attachment filter map[creator(:)][]AttachmentFilter attachmentFilters map[string][]AttachmentFilter // Select a specific creator // If not specified, all creators will be selected users []Creator client *http.Client log Log retry int retryInterval time.Duration } func NewKemono(site string) *kemono { k := &kemono{ Site: site, Banner: true, postFilters: make(map[string][]PostFilter), attachmentFilters: make(map[string][]AttachmentFilter), retry: 3, retryInterval: 5 * time.Second, } if k.log == nil { k.log = &DefaultLog{log: log.New(os.Stdout, "", log.LstdFlags)} } return k } // WithDomain Set Site // func WithDomain(web string) Option { // return func(k *kemono) { // k.Site = web // } // } // func WithBanner(banner bool) Option { // return func(k *kemono) { // k.Banner = banner // } // } // Custom the creator list // func WithCreators(creators []Creator) Option { // return func(k *kemono) { // k.creators = creators // } // } // WithUsers Select a specific creator, if not specified, all creators will be selected // func WithUsers(user ...Creator) Option { // return func(k *kemono) { // for _, u := range user { // exist := false // for _, c := range k.users { // if c.Service == u.Service && c.Id == u.Id { // exist = true // break // } // } // if !exist { // k.users = append(k.users, u) // } // } // } // } // WithUsersPair Select a specific creator, if not specified, all creators will be selected // func WithUsersPair(serviceIdPairs ...string) Option { // return func(k *kemono) { // if len(serviceIdPairs)%2 != 0 { // k.log.Printf("serviceIdPairs length must be even") // return // } // for i := 0; i < len(serviceIdPairs); i += 2 { // exist := false // for _, c := range k.users { // if c.Service == serviceIdPairs[i] && c.Id == serviceIdPairs[i+1] { // exist = true // break // } // } // if !exist { // k.users = append(k.users, NewCreator(serviceIdPairs[i], serviceIdPairs[i+1])) // } // } // } // } // SetLog set log func SetLog(log Log) Option { return func(k *kemono) { k.log = log } } // // SetRetry set retry // func SetRetry(retry int) Option { // return func(k *kemono) { // k.retry = retry // } // } // // SetRetryInterval set retry interval // func SetRetryInterval(retryInterval time.Duration) Option { // return func(k *kemono) { // k.retryInterval = retryInterval // } // } // // WithCreatorFilter Creator filter // func WithCreatorFilter(filter ...CreatorFilter) Option { // return func(k *kemono) { // k.addCreatorFilter(filter...) // } // } // // WithPostFilter Post filter // func WithPostFilter(filter ...PostFilter) Option { // return func(k *kemono) { // k.addPostFilter(filter...) // } // } // func WithUserPostFilter(creator Creator, filter ...PostFilter) Option { // return func(k *kemono) { // k.addUserPostFilter(creator.PairString(), filter...) // } // } // // WithAttachmentFilter Attachment filter // func WithAttachmentFilter(filter ...AttachmentFilter) Option { // return func(k *kemono) { // k.addAttachmentFilter(filter...) // } // } // func WithUserAttachmentFilter(creator Creator, filter ...AttachmentFilter) Option { // return func(k *kemono) { // k.addUserAttachmentFilter(creator.PairString(), filter...) // } // } // // Start fetch and download // func (k *kemono) Start() error { // // initialize the creators // if len(k.creators) == 0 { // // fetch creators from kemono // cs, err := k.FetchCreators() // if err != nil { // return err // } // k.creators = cs // } // //find creators // if len(k.users) != 0 { // var creators []Creator // for _, user := range k.users { // c, ok := FindCreator(k.creators, user.Id, user.Service) // if !ok { // k.log.Printf("Creator %s:%s not found", user.Service, user.Id) // continue // } // creators = append(creators, c) // } // k.users = creators // } else { // k.users = k.creators // } // // Filter selected creators // k.users = k.FilterCreators(k.users) // // start download // k.log.Printf("Start download %d creators", len(k.users)) // for _, creator := range k.users { // // fetch posts // posts, err := k.FetchPosts(creator.Service, creator.Id) // if err != nil { // return err // } // // filter posts // posts = k.FilterPosts(posts) // // filter attachments // for i, post := range posts { // // download banner if banner is true or file is not image // if (k.Banner || !isImage(filepath.Ext(post.File.Name))) && post.File.Path != "" { // res := make([]File, len(post.Attachments)+1) // copy(res[1:], post.Attachments) // res[0] = post.File // post.Attachments = res // } // posts[i].Attachments = k.FilterAttachments(fmt.Sprintf("%s:%s", post.Service, post.User), post.Attachments) // } // // download posts // err = k.DownloadPosts(creator, posts) // if err != nil { // return err // } // } // return nil // } // func (k *kemono) addCreatorFilter(filter ...CreatorFilter) { // k.creatorFilters = append(k.creatorFilters, filter...) // } // func (k *kemono) addPostFilter(filter ...PostFilter) { // k.postFilters["*"] = append(k.postFilters["*"], filter...) // } // func (k *kemono) addUserPostFilter(user string, filter ...PostFilter) { // k.postFilters[user] = append(k.postFilters[user], filter...) // } // func (k *kemono) addAttachmentFilter(filter ...AttachmentFilter) { // k.attachmentFilters["*"] = append(k.attachmentFilters["*"], filter...) // } // func (k *kemono) addUserAttachmentFilter(user string, filter ...AttachmentFilter) { // k.attachmentFilters[user] = append(k.attachmentFilters[user], filter...) // } // func (k *kemono) filterCreator(i int, creator Creator) bool { // for _, filter := range k.creatorFilters { // if !filter(i, creator) { // return false // } // } // return true // } // func (k *kemono) filterPost(i int, post Post) bool { // for _, filter := range k.postFilters["*"] { // if !filter(i, post) { // return false // } // } // for _, filter := range k.postFilters[fmt.Sprintf("%s:%s", post.Service, post.User)] { // if !filter(i, post) { // return false // } // } // return true // } // func (k *kemono) filterAttachment(user string, i int, attachment File) bool { // for _, filter := range k.attachmentFilters["*"] { // if !filter(i, attachment) { // return false // } // } // for _, filter := range k.attachmentFilters[user] { // if !filter(i, attachment) { // return false // } // } // return true // } // func (k *kemono) FilterCreators(creators []Creator) []Creator { // var filteredCreators []Creator // for i, creator := range creators { // if k.filterCreator(i, creator) { // filteredCreators = append(filteredCreators, creator) // } // } // return filteredCreators // } // func (k *kemono) FilterPosts(posts []Post) []Post { // var filteredPosts []Post // for i, post := range posts { // if k.filterPost(i, post) { // filteredPosts = append(filteredPosts, post) // } // } // return filteredPosts // } // func (k *kemono) FilterAttachments(user string, attachments []File) []File { // var filteredAttachments []File // for i, attachment := range attachments { // if k.filterAttachment(user, i, attachment) { // filteredAttachments = append(filteredAttachments, attachment) // } // } // return filteredAttachments // } // // ReleaseDateFilter A Post filter that filters posts with release date // func ReleaseDateFilter(from, to time.Time) PostFilter { // return func(i int, post Post) bool { // return post.Published.After(from) && post.Published.Before(to) // } // } // // ReleaseDateAfterFilter A Post filter that filters posts with release date after // func ReleaseDateAfterFilter(from time.Time) PostFilter { // return func(i int, post Post) bool { // return post.Published.After(from) // } // } // // ReleaseDateBeforeFilter A Post filter that filters posts with release date before // func ReleaseDateBeforeFilter(to time.Time) PostFilter { // return func(i int, post Post) bool { // return post.Published.Before(to) // } // } // // EditDateFilter A Post filter that filters posts with edit date // func EditDateFilter(from, to time.Time) PostFilter { // return func(i int, post Post) bool { // return post.Edited.After(from) && post.Edited.Before(to) // } // } // // EditDateAfterFilter A Post filter that filters posts with edit date after // func EditDateAfterFilter(from time.Time) PostFilter { // return func(i int, post Post) bool { // return post.Edited.After(from) // } // } // // EditDateBeforeFilter A Post filter that filters posts with edit date before // func EditDateBeforeFilter(to time.Time) PostFilter { // return func(i int, post Post) bool { // return post.Edited.Before(to) // } // } // func IdFilter(ids ...string) PostFilter { // return func(i int, post Post) bool { // for _, id := range ids { // if id == post.Id { // return true // } // } // return false // } // } // // NumbFilter A Post filter that filters posts with a specific number // func NumbFilter(f func(i int) bool) PostFilter { // return func(i int, post Post) bool { // return f(i) // } // } // // ExtensionFilter A attachmentFilter filter that filters attachments with a specific extension // func ExtensionFilter(extension ...string) AttachmentFilter { // return func(i int, attachment File) bool { // ext := filepath.Ext(attachment.Name) // for _, e := range extension { // if ext == e { // return true // } // } // return false // } // } // // ExtensionExcludeFilter A attachmentFilter filter that filters attachments without a specific extension // func ExtensionExcludeFilter(extension ...string) AttachmentFilter { // return func(i int, attachment File) bool { // ext := filepath.Ext(attachment.Name) // for _, e := range extension { // if ext == e { // return false // } // } // return true // } // }