royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

dht store and stats query
All checks were successful
docker / build-docker (linux/arm64) (push) Successful in 2m8s
Details
docker / build-docker (linux/amd64) (push) Successful in 2m10s
Details

Browse source
This commit is contained in:
royalcat 2024-07-10 12:26:17 +03:00
parent 93892a6f1d
commit d5aa78cb39
13 changed files with 1114 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
graphql/sources/torrent_query.graphql
View file

@ -1,5 +1,6 @@
type TorrentDaemonQuery {
torrents(filter: TorrentsFilter): [Torrent!]! @resolver
stats: TorrentStats! @resolver
}
input TorrentsFilter {

19
graphql/sources/torrent_types.graphql
View file

@ -32,3 +32,22 @@ enum TorrentPriority {
READAHEAD
NOW
}
type TorrentStats {
bytesWritten: Int!
bytesWrittenData: Int!
bytesRead: Int!
bytesReadData: Int!
bytesReadUsefulData: Int!
bytesReadUsefulIntendedData: Int!
chunksWritten: Int!
chunksRead: Int!
chunksReadUseful: Int!
chunksReadWasted: Int!
metadataChunksRead: Int!
piecesDirtiedGood: Int!
piecesDirtiedBad: Int!
}

947
src/delivery/graphql/generated.go
View file

File diff suppressed because it is too large Load diff

17
src/delivery/graphql/model/models_gen.go
View file

@ -192,6 +192,7 @@ type TorrentDaemonMutation struct {
type TorrentDaemonQuery struct {
Torrents []*Torrent `json:"torrents"`
Stats *TorrentStats `json:"stats"`
}
type TorrentFs struct {
@ -269,6 +270,22 @@ func (TorrentProgress) IsProgress() {}
func (this TorrentProgress) GetCurrent() int64 { return this.Current }
func (this TorrentProgress) GetTotal() int64 { return this.Total }
type TorrentStats struct {
BytesWritten int64 `json:"bytesWritten"`
BytesWrittenData int64 `json:"bytesWrittenData"`
BytesRead int64 `json:"bytesRead"`
BytesReadData int64 `json:"bytesReadData"`
BytesReadUsefulData int64 `json:"bytesReadUsefulData"`
BytesReadUsefulIntendedData int64 `json:"bytesReadUsefulIntendedData"`
ChunksWritten int64 `json:"chunksWritten"`
ChunksRead int64 `json:"chunksRead"`
ChunksReadUseful int64 `json:"chunksReadUseful"`
ChunksReadWasted int64 `json:"chunksReadWasted"`
MetadataChunksRead int64 `json:"metadataChunksRead"`
PiecesDirtiedGood int64 `json:"piecesDirtiedGood"`
PiecesDirtiedBad int64 `json:"piecesDirtiedBad"`
}
type TorrentsFilter struct {
Infohash *StringFilter `json:"infohash,omitempty"`
Name *StringFilter `json:"name,omitempty"`

20
src/delivery/graphql/resolver/torrent_query.resolvers.go
View file

@ -86,6 +86,26 @@ func (r *torrentDaemonQueryResolver) Torrents(ctx context.Context, obj *model.To
return tr, nil
}
// Stats is the resolver for the stats field.
func (r *torrentDaemonQueryResolver) Stats(ctx context.Context, obj *model.TorrentDaemonQuery) (*model.TorrentStats, error) {
stats := r.Service.Stats()
return &model.TorrentStats{
BytesWritten: stats.BytesWritten.Int64(),
BytesRead: stats.BytesRead.Int64(),
BytesWrittenData: stats.BytesWrittenData.Int64(),
BytesReadData: stats.BytesReadData.Int64(),
BytesReadUsefulData: stats.BytesReadUsefulData.Int64(),
BytesReadUsefulIntendedData: stats.BytesReadUsefulIntendedData.Int64(),
ChunksWritten: stats.ChunksWritten.Int64(),
ChunksRead: stats.ChunksRead.Int64(),
ChunksReadUseful: stats.ChunksReadUseful.Int64(),
ChunksReadWasted: stats.ChunksReadWasted.Int64(),
MetadataChunksRead: stats.MetadataChunksRead.Int64(),
PiecesDirtiedGood: stats.PiecesDirtiedGood.Int64(),
PiecesDirtiedBad: stats.PiecesDirtiedBad.Int64(),
}, nil
}
// TorrentDaemonQuery returns graph.TorrentDaemonQueryResolver implementation.
func (r *Resolver) TorrentDaemonQuery() graph.TorrentDaemonQueryResolver {
return &torrentDaemonQueryResolver{r}

2
src/delivery/http.go
View file

@ -23,7 +23,7 @@ func New(fc *filecache.Cache, s *torrent.Daemon, vfs vfs.Filesystem, logPath str
// middleware.Recover(),
middleware.Gzip(),
middleware.Decompress(),
Logger(),
// Logger(),
)
echopprof.Register(r)

10
src/sources/torrent/client.go
View file

@ -22,15 +22,16 @@ func newClient(st storage.ClientImpl, fis bep44.Store, cfg *config.TorrentClient
torrentCfg.PeerID = string(id[:])
torrentCfg.DefaultStorage = st
torrentCfg.AlwaysWantConns = true
torrentCfg.DropMutuallyCompletePeers = true
torrentCfg.TorrentPeersLowWater = 100
torrentCfg.TorrentPeersHighWater = 1000
torrentCfg.AcceptPeerConnections = true
torrentCfg.DisableAggressiveUpload = false
torrentCfg.Seed = true
// torrentCfg.DownloadRateLimiter = rate.NewLimiter(rate.Inf, 0)
// torrentCfg
torrentCfg.DisableAggressiveUpload = false
tl := tlog.NewLogger()
tl.SetHandlers(&dlog.Torrent{L: l})
torrentCfg.Logger = tl
torrentCfg.Callbacks.NewPeer = append(torrentCfg.Callbacks.NewPeer, func(p *torrent.Peer) {
l := l.With("ip", p.RemoteAddr.String())
@ -55,6 +56,7 @@ func newClient(st storage.ClientImpl, fis bep44.Store, cfg *config.TorrentClient
// l.Debug("peer closed", "ip", c.RemoteAddr.String())
// })
torrentCfg.PeriodicallyAnnounceTorrentsToDht = true
torrentCfg.ConfigureAnacrolixDhtServer = func(cfg *dht.ServerConfig) {
cfg.Store = fis
cfg.Exp = 2 * time.Hour

74
src/sources/torrent/controller.go
View file

@ -29,12 +29,12 @@ type Controller struct {
log *rlog.Logger
}
func newController(t *torrent.Torrent, torrentFileProperties kv.Store[string, FileProperties], storage TorrentFileDeleter) *Controller {
func newController(t *torrent.Torrent, torrentFileProperties kv.Store[string, FileProperties], storage TorrentFileDeleter, log *rlog.Logger) *Controller {
return &Controller{
t: t,
storage: storage,
fileProperties: torrentFileProperties,
log: rlog.Component("torrent-client", "controller").With(slog.String("infohash", t.InfoHash().HexString())),
log: log.WithComponent("controller").With(slog.String("infohash", t.InfoHash().HexString())),
}
}
@ -101,7 +101,7 @@ func (s *Controller) Files(ctx context.Context) ([]*FileController, error) {
}
props := kvsingle.New(s.fileProperties, v.Path())
ctl := NewFileController(v, props)
ctl := NewFileController(v, props, s.log)
files = append(files, ctl)
}
@ -179,15 +179,25 @@ func (s *Controller) ValidateTorrent(ctx context.Context) error {
}
func (c *Controller) SetPriority(ctx context.Context, priority types.PiecePriority) error {
// log := c.log.With(slog.Int("priority", int(priority)))
for _, f := range c.t.Files() {
err := c.setFilePriority(ctx, f, priority)
log := c.log.With(slog.Int("priority", int(priority)))
files, err := c.Files(ctx)
if err != nil {
return err
}
for _, f := range files {
excluded, err := f.Excluded(ctx)
if err != nil {
log.Error(ctx, "failed to get file exclusion status", rlog.Error(err))
}
if excluded {
continue
}
err = f.SetPriority(ctx, priority)
if err != nil {
log.Error(ctx, "failed to set file priority", rlog.Error(err))
}
}
return nil
}
@ -200,37 +210,35 @@ func (c *Controller) Priority(ctx context.Context) (types.PiecePriority, error)
return 0, err
}
for _, v := range files {
props, err := v.Properties(ctx)
if err != nil {
return 0, err
}
if props.Priority > prio {
prio = props.Priority
filePriority := v.Priority()
if filePriority > prio {
prio = filePriority
}
}
return prio, nil
}
func (c *Controller) setFilePriority(ctx context.Context, file *torrent.File, priority types.PiecePriority) error {
err := c.fileProperties.Edit(ctx, file.Path(), func(ctx context.Context, v FileProperties) (FileProperties, error) {
v.Priority = priority
return v, nil
})
if err == kv.ErrKeyNotFound {
seterr := c.fileProperties.Set(ctx, file.Path(), FileProperties{Priority: priority})
if seterr != nil {
return seterr
}
err = nil
}
// func (c *Controller) setFilePriority(ctx context.Context, file *torrent.File, priority types.PiecePriority) error {
// err := c.fileProperties.Edit(ctx, file.Path(), func(ctx context.Context, v FileProperties) (FileProperties, error) {
// v.Priority = priority
// return v, nil
// })
if err != nil {
return err
}
file.SetPriority(priority)
return nil
}
// if err == kv.ErrKeyNotFound {
// seterr := c.fileProperties.Set(ctx, file.Path(), FileProperties{Priority: priority})
// if seterr != nil {
// return seterr
// }
// err = nil
// }
// if err != nil {
// return err
// }
// file.SetPriority(priority)
// return nil
// }
func (c *Controller) initializeTorrentPriories(ctx context.Context) error {
ctx, span := tracer.Start(ctx, "initializeTorrentPriories")
@ -248,12 +256,10 @@ func (c *Controller) initializeTorrentPriories(ctx context.Context) error {
log.Error(ctx, "failed to get file properties", rlog.Error(err))
continue
}
log = log.With(slog.Int("priority", int(props.Priority)))
file.file.SetPriority(props.Priority)
}
log.Info(ctx, "torrent initialization complete", slog.String("infohash", c.InfoHash()), slog.String("torrent_name", c.Name()))
log.Debug(ctx, "torrent initialization complete", slog.String("infohash", c.InfoHash()), slog.String("torrent_name", c.Name()))
return nil
}

33
src/sources/torrent/daemon.go
View file

@ -45,7 +45,7 @@ type Daemon struct {
client *torrent.Client
infoBytes *infoBytesStore
Storage *fileStorage
fis *fileItemStore
fis *dhtFileItemStore
dirsAquire kv.Store[string, DirAquire]
fileProperties kv.Store[string, FileProperties]
@ -70,7 +70,7 @@ func NewService(sourceFs billy.Filesystem, conf config.TorrentClient) (*Daemon,
return nil, fmt.Errorf("error creating metadata folder: %w", err)
}
s.fis, err = newFileItemStore(filepath.Join(conf.MetadataFolder, "items"), 2*time.Hour)
s.fis, err = newDHTStore(filepath.Join(conf.MetadataFolder, "dht-item-store"), 3*time.Hour)
if err != nil {
return nil, fmt.Errorf("error starting item store: %w", err)
}
@ -149,9 +149,11 @@ func (s *Daemon) loadTorrent(ctx context.Context, f vfs.File) (*Controller, erro
return nil, fmt.Errorf("loading torrent metadata from file %s, error: %w", stat.Name(), err)
}
var ctl *Controller
t, ok := s.client.Torrent(mi.HashInfoBytes())
if !ok {
if ok {
ctl = s.newController(t)
} else {
span.AddEvent("torrent not found, loading from file")
log.Info(ctx, "torrent not found, loading from file")
@ -176,10 +178,12 @@ func (s *Daemon) loadTorrent(ctx context.Context, f vfs.File) (*Controller, erro
t, _ = s.client.AddTorrentOpt(torrent.AddTorrentOpts{
InfoHash: spec.InfoHash,
InfoHashV2: spec.InfoHashV2,
Storage: s.Storage,
InfoBytes: infoBytes,
ChunkSize: spec.ChunkSize,
})
t.AllowDataDownload()
t.AllowDataUpload()
@ -199,6 +203,13 @@ func (s *Daemon) loadTorrent(ctx context.Context, f vfs.File) (*Controller, erro
}
span.AddEvent("got info")
ctl = s.newController(t)
err = ctl.initializeTorrentPriories(ctx)
if err != nil {
return nil, fmt.Errorf("initialize torrent priorities: %w", err)
}
// info := t.Info()
// if info == nil {
// return nil, fmt.Errorf("info is nil")
@ -216,13 +227,6 @@ func (s *Daemon) loadTorrent(ctx context.Context, f vfs.File) (*Controller, erro
// }
}
ctl := s.newController(t)
err = ctl.initializeTorrentPriories(ctx)
if err != nil {
log.Error(ctx, "error initializing torrent priorities", rlog.Error(err))
}
return ctl, nil
}
@ -232,11 +236,7 @@ func isValidInfoHashBytes(d []byte) bool {
return err == nil
}
func (s *Daemon) Stats() (*Stats, error) {
return &Stats{}, nil
}
func (s *Daemon) GetStats() torrent.ConnStats {
func (s *Daemon) Stats() torrent.ConnStats {
return s.client.ConnStats()
}
@ -316,6 +316,7 @@ func (s *Daemon) newController(t *torrent.Torrent) *Controller {
return newController(t,
storeByTorrent(s.fileProperties, t.InfoHash()),
s.Storage,
s.log,
)
}

18
src/sources/torrent/fileitem.go → src/sources/torrent/dht_fileitem_store.go
View file

@ -10,16 +10,16 @@ import (
"github.com/dgraph-io/badger/v4"
)
var _ bep44.Store = &fileItemStore{}
var _ bep44.Store = &dhtFileItemStore{}
type fileItemStore struct {
type dhtFileItemStore struct {
ttl time.Duration
db *badger.DB
}
func newFileItemStore(path string, itemsTTL time.Duration) (*fileItemStore, error) {
func newDHTStore(path string, itemsTTL time.Duration) (*dhtFileItemStore, error) {
opts := badger.DefaultOptions(path).
WithLogger(dlog.BadgerLogger("torrent-client", "item-store")).
WithLogger(dlog.BadgerLogger("torrent-client", "dht-item-store")).
WithValueLogFileSize(1<<26 - 1)
db, err := badger.Open(opts)
@ -32,13 +32,13 @@ func newFileItemStore(path string, itemsTTL time.Duration) (*fileItemStore, erro
return nil, err
}
return &fileItemStore{
return &dhtFileItemStore{
db: db,
ttl: itemsTTL,
}, nil
}
func (fis *fileItemStore) Put(i *bep44.Item) error {
func (fis *dhtFileItemStore) Put(i *bep44.Item) error {
tx := fis.db.NewTransaction(true)
defer tx.Discard()
@ -58,7 +58,7 @@ func (fis *fileItemStore) Put(i *bep44.Item) error {
return tx.Commit()
}
func (fis *fileItemStore) Get(t bep44.Target) (*bep44.Item, error) {
func (fis *dhtFileItemStore) Get(t bep44.Target) (*bep44.Item, error) {
tx := fis.db.NewTransaction(false)
defer tx.Discard()
@ -84,11 +84,11 @@ func (fis *fileItemStore) Get(t bep44.Target) (*bep44.Item, error) {
return i, nil
}
func (fis *fileItemStore) Del(t bep44.Target) error {
func (fis *dhtFileItemStore) Del(t bep44.Target) error {
// ignore this
return nil
}
func (fis *fileItemStore) Close() error {
func (fis *dhtFileItemStore) Close() error {
return fis.db.Close()
}

13
src/sources/torrent/file_controller.go
View file

@ -2,8 +2,10 @@ package torrent
import (
"context"
"log/slog"
"git.kmsign.ru/royalcat/tstor/pkg/kvsingle"
"git.kmsign.ru/royalcat/tstor/pkg/rlog"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/types"
@ -13,12 +15,14 @@ import (
type FileController struct {
file *torrent.File
properties *kvsingle.Value[string, FileProperties]
log *rlog.Logger
}
func NewFileController(f *torrent.File, properties *kvsingle.Value[string, FileProperties]) *FileController {
func NewFileController(f *torrent.File, properties *kvsingle.Value[string, FileProperties], log *rlog.Logger) *FileController {
return &FileController{
file: f,
properties: properties,
log: log.WithComponent("file-controller").With(slog.String("file", f.Path())),
}
}
@ -38,6 +42,8 @@ func (s *FileController) Properties(ctx context.Context) (FileProperties, error)
}
func (s *FileController) SetPriority(ctx context.Context, priority types.PiecePriority) error {
log := s.log.With(slog.Int("priority", int(priority)))
err := s.properties.Edit(ctx, func(ctx context.Context, v FileProperties) (FileProperties, error) {
v.Priority = priority
return v, nil
@ -55,6 +61,7 @@ func (s *FileController) SetPriority(ctx context.Context, priority types.PiecePr
return err
}
log.Debug(ctx, "file priority set")
s.file.SetPriority(priority)
return nil
@ -83,6 +90,10 @@ func (s *FileController) Size() int64 {
return s.file.Length()
}
func (s *FileController) Priority() types.PiecePriority {
return s.file.Priority()
}
func (s *FileController) BytesCompleted() int64 {
return s.file.BytesCompleted()
}

4
src/vfs/log.go
View file

@ -124,7 +124,7 @@ func (fs *LogFS) Open(ctx context.Context, filename string) (file File, err erro
if isLoggableError(err) {
fs.log.Error(ctx, "Failed to open file")
}
file = wrapLogFile(file, filename, fs.log, fs.readTimeout, fs.tel)
file = WrapLogFile(file, filename, fs.log, fs.readTimeout, fs.tel)
if file != nil {
fs.tel.openedFiles.Add(ctx, 1)
@ -221,7 +221,7 @@ func (f *LogFile) Type() fs.FileMode {
var _ File = (*LogFile)(nil)
func wrapLogFile(f File, filename string, log *rlog.Logger, timeout time.Duration, tel *fsTelemetry) *LogFile {
func WrapLogFile(f File, filename string, log *rlog.Logger, timeout time.Duration, tel *fsTelemetry) *LogFile {
return &LogFile{
filename: filename,
f: f,

16
ui/lib/api/schema.graphql
View file

@ -106,6 +106,7 @@ type TorrentDaemonMutation {
}
type TorrentDaemonQuery {
torrents(filter: TorrentsFilter): [Torrent!]! @resolver
stats: TorrentStats! @resolver
}
type TorrentFS implements Dir & FsEntry {
name: String!
@ -154,6 +155,21 @@ type TorrentProgress implements Progress {
current: Int!
total: Int!
}
type TorrentStats {
bytesWritten: Int!
bytesWrittenData: Int!
bytesRead: Int!
bytesReadData: Int!
bytesReadUsefulData: Int!
bytesReadUsefulIntendedData: Int!
chunksWritten: Int!
chunksRead: Int!
chunksReadUseful: Int!
chunksReadWasted: Int!
metadataChunksRead: Int!
piecesDirtiedGood: Int!
piecesDirtiedBad: Int!
}
input TorrentsFilter {
infohash: StringFilter
name: StringFilter