dht store and stats query

graphql/sources/torrent_query.graphql

@@ -1,5 +1,6 @@
 type TorrentDaemonQuery {
   torrents(filter: TorrentsFilter): [Torrent!]! @resolver
+  stats: TorrentStats! @resolver
 }
 
 input TorrentsFilter {

graphql/sources/torrent_types.graphql

@@ -32,3 +32,22 @@ enum TorrentPriority {
   READAHEAD
   NOW
 }
+
+type TorrentStats {
+  bytesWritten: Int!
+  bytesWrittenData: Int!
+  bytesRead: Int!
+  bytesReadData: Int!
+  bytesReadUsefulData: Int!
+  bytesReadUsefulIntendedData: Int!
+
+  chunksWritten: Int!
+  chunksRead: Int!
+  chunksReadUseful: Int!
+  chunksReadWasted: Int!
+
+  metadataChunksRead: Int!
+
+  piecesDirtiedGood: Int!
+  piecesDirtiedBad: Int!
+}

src/delivery/graphql/model/models_gen.go

@@ -191,7 +191,8 @@ type TorrentDaemonMutation struct {
 }
 
 type TorrentDaemonQuery struct {
-	Torrents []*Torrent `json:"torrents"`
+	Torrents []*Torrent    `json:"torrents"`
+	Stats    *TorrentStats `json:"stats"`
 }
 
 type TorrentFs struct {
@@ -269,6 +270,22 @@ func (TorrentProgress) IsProgress()            {}
 func (this TorrentProgress) GetCurrent() int64 { return this.Current }
 func (this TorrentProgress) GetTotal() int64   { return this.Total }
 
+type TorrentStats struct {
+	BytesWritten                int64 `json:"bytesWritten"`
+	BytesWrittenData            int64 `json:"bytesWrittenData"`
+	BytesRead                   int64 `json:"bytesRead"`
+	BytesReadData               int64 `json:"bytesReadData"`
+	BytesReadUsefulData         int64 `json:"bytesReadUsefulData"`
+	BytesReadUsefulIntendedData int64 `json:"bytesReadUsefulIntendedData"`
+	ChunksWritten               int64 `json:"chunksWritten"`
+	ChunksRead                  int64 `json:"chunksRead"`
+	ChunksReadUseful            int64 `json:"chunksReadUseful"`
+	ChunksReadWasted            int64 `json:"chunksReadWasted"`
+	MetadataChunksRead          int64 `json:"metadataChunksRead"`
+	PiecesDirtiedGood           int64 `json:"piecesDirtiedGood"`
+	PiecesDirtiedBad            int64 `json:"piecesDirtiedBad"`
+}
+
 type TorrentsFilter struct {
 	Infohash       *StringFilter          `json:"infohash,omitempty"`
 	Name           *StringFilter          `json:"name,omitempty"`

src/delivery/graphql/resolver/torrent_query.resolvers.go

@@ -86,6 +86,26 @@ func (r *torrentDaemonQueryResolver) Torrents(ctx context.Context, obj *model.To
 	return tr, nil
 }
 
+// Stats is the resolver for the stats field.
+func (r *torrentDaemonQueryResolver) Stats(ctx context.Context, obj *model.TorrentDaemonQuery) (*model.TorrentStats, error) {
+	stats := r.Service.Stats()
+	return &model.TorrentStats{
+		BytesWritten:                stats.BytesWritten.Int64(),
+		BytesRead:                   stats.BytesRead.Int64(),
+		BytesWrittenData:            stats.BytesWrittenData.Int64(),
+		BytesReadData:               stats.BytesReadData.Int64(),
+		BytesReadUsefulData:         stats.BytesReadUsefulData.Int64(),
+		BytesReadUsefulIntendedData: stats.BytesReadUsefulIntendedData.Int64(),
+		ChunksWritten:               stats.ChunksWritten.Int64(),
+		ChunksRead:                  stats.ChunksRead.Int64(),
+		ChunksReadUseful:            stats.ChunksReadUseful.Int64(),
+		ChunksReadWasted:            stats.ChunksReadWasted.Int64(),
+		MetadataChunksRead:          stats.MetadataChunksRead.Int64(),
+		PiecesDirtiedGood:           stats.PiecesDirtiedGood.Int64(),
+		PiecesDirtiedBad:            stats.PiecesDirtiedBad.Int64(),
+	}, nil
+}
+
 // TorrentDaemonQuery returns graph.TorrentDaemonQueryResolver implementation.
 func (r *Resolver) TorrentDaemonQuery() graph.TorrentDaemonQueryResolver {
 	return &torrentDaemonQueryResolver{r}

src/delivery/http.go

@@ -23,7 +23,7 @@ func New(fc *filecache.Cache, s *torrent.Daemon, vfs vfs.Filesystem, logPath str
 		// middleware.Recover(),
 		middleware.Gzip(),
 		middleware.Decompress(),
-		Logger(),
+		// Logger(),
 	)
 
 	echopprof.Register(r)

src/sources/torrent/client.go

@@ -22,15 +22,16 @@ func newClient(st storage.ClientImpl, fis bep44.Store, cfg *config.TorrentClient
 	torrentCfg.PeerID = string(id[:])
 	torrentCfg.DefaultStorage = st
 	torrentCfg.AlwaysWantConns = true
+	torrentCfg.DropMutuallyCompletePeers = true
+	torrentCfg.TorrentPeersLowWater = 100
+	torrentCfg.TorrentPeersHighWater = 1000
 	torrentCfg.AcceptPeerConnections = true
-	torrentCfg.DisableAggressiveUpload = false
-
 	torrentCfg.Seed = true
-	// torrentCfg.DownloadRateLimiter = rate.NewLimiter(rate.Inf, 0)
+	torrentCfg.DisableAggressiveUpload = false
-	// torrentCfg
 
 	tl := tlog.NewLogger()
 	tl.SetHandlers(&dlog.Torrent{L: l})
+
 	torrentCfg.Logger = tl
 	torrentCfg.Callbacks.NewPeer = append(torrentCfg.Callbacks.NewPeer, func(p *torrent.Peer) {
 		l := l.With("ip", p.RemoteAddr.String())
@@ -55,6 +56,7 @@ func newClient(st storage.ClientImpl, fis bep44.Store, cfg *config.TorrentClient
 	// 	l.Debug("peer closed", "ip", c.RemoteAddr.String())
 	// })
 
+	torrentCfg.PeriodicallyAnnounceTorrentsToDht = true
 	torrentCfg.ConfigureAnacrolixDhtServer = func(cfg *dht.ServerConfig) {
 		cfg.Store = fis
 		cfg.Exp = 2 * time.Hour

src/sources/torrent/controller.go

@@ -29,12 +29,12 @@ type Controller struct {
 	log             *rlog.Logger
 }
 
-func newController(t *torrent.Torrent, torrentFileProperties kv.Store[string, FileProperties], storage TorrentFileDeleter) *Controller {
+func newController(t *torrent.Torrent, torrentFileProperties kv.Store[string, FileProperties], storage TorrentFileDeleter, log *rlog.Logger) *Controller {
 	return &Controller{
 		t:              t,
 		storage:        storage,
 		fileProperties: torrentFileProperties,
-		log:            rlog.Component("torrent-client", "controller").With(slog.String("infohash", t.InfoHash().HexString())),
+		log:            log.WithComponent("controller").With(slog.String("infohash", t.InfoHash().HexString())),
 	}
 }
 
@@ -101,7 +101,7 @@ func (s *Controller) Files(ctx context.Context) ([]*FileController, error) {
 		}
 
 		props := kvsingle.New(s.fileProperties, v.Path())
-		ctl := NewFileController(v, props)
+		ctl := NewFileController(v, props, s.log)
 		files = append(files, ctl)
 	}
 
@@ -179,15 +179,25 @@ func (s *Controller) ValidateTorrent(ctx context.Context) error {
 }
 
 func (c *Controller) SetPriority(ctx context.Context, priority types.PiecePriority) error {
-	// log := c.log.With(slog.Int("priority", int(priority)))
+	log := c.log.With(slog.Int("priority", int(priority)))
-
+	files, err := c.Files(ctx)
-	for _, f := range c.t.Files() {
+	if err != nil {
-		err := c.setFilePriority(ctx, f, priority)
+		return err
+	}
+	for _, f := range files {
+		excluded, err := f.Excluded(ctx)
 		if err != nil {
-			return err
+			log.Error(ctx, "failed to get file exclusion status", rlog.Error(err))
+		}
+		if excluded {
+			continue
+		}
+
+		err = f.SetPriority(ctx, priority)
+		if err != nil {
+			log.Error(ctx, "failed to set file priority", rlog.Error(err))
 		}
 	}
-
 	return nil
 }
 
@@ -200,37 +210,35 @@ func (c *Controller) Priority(ctx context.Context) (types.PiecePriority, error)
 		return 0, err
 	}
 	for _, v := range files {
-		props, err := v.Properties(ctx)
+		filePriority := v.Priority()
-		if err != nil {
+		if filePriority > prio {
-			return 0, err
+			prio = filePriority
-		}
-		if props.Priority > prio {
-			prio = props.Priority
 		}
 	}
 
 	return prio, nil
 }
-func (c *Controller) setFilePriority(ctx context.Context, file *torrent.File, priority types.PiecePriority) error {
-	err := c.fileProperties.Edit(ctx, file.Path(), func(ctx context.Context, v FileProperties) (FileProperties, error) {
-		v.Priority = priority
-		return v, nil
-	})
 
-	if err == kv.ErrKeyNotFound {
+// func (c *Controller) setFilePriority(ctx context.Context, file *torrent.File, priority types.PiecePriority) error {
-		seterr := c.fileProperties.Set(ctx, file.Path(), FileProperties{Priority: priority})
+// 	err := c.fileProperties.Edit(ctx, file.Path(), func(ctx context.Context, v FileProperties) (FileProperties, error) {
-		if seterr != nil {
+// 		v.Priority = priority
-			return seterr
+// 		return v, nil
-		}
+// 	})
-		err = nil
-	}
 
-	if err != nil {
+// 	if err == kv.ErrKeyNotFound {
-		return err
+// 		seterr := c.fileProperties.Set(ctx, file.Path(), FileProperties{Priority: priority})
-	}
+// 		if seterr != nil {
-	file.SetPriority(priority)
+// 			return seterr
-	return nil
+// 		}
-}
+// 		err = nil
+// 	}
+
+// 	if err != nil {
+// 		return err
+// 	}
+// 	file.SetPriority(priority)
+// 	return nil
+// }
 
 func (c *Controller) initializeTorrentPriories(ctx context.Context) error {
 	ctx, span := tracer.Start(ctx, "initializeTorrentPriories")
@@ -248,12 +256,10 @@ func (c *Controller) initializeTorrentPriories(ctx context.Context) error {
 			log.Error(ctx, "failed to get file properties", rlog.Error(err))
 			continue
 		}
-		log = log.With(slog.Int("priority", int(props.Priority)))
-
 		file.file.SetPriority(props.Priority)
 	}
 
-	log.Info(ctx, "torrent initialization complete", slog.String("infohash", c.InfoHash()), slog.String("torrent_name", c.Name()))
+	log.Debug(ctx, "torrent initialization complete", slog.String("infohash", c.InfoHash()), slog.String("torrent_name", c.Name()))
 
 	return nil
 }

src/sources/torrent/daemon.go

@@ -45,7 +45,7 @@ type Daemon struct {
 	client         *torrent.Client
 	infoBytes      *infoBytesStore
 	Storage        *fileStorage
-	fis            *fileItemStore
+	fis            *dhtFileItemStore
 	dirsAquire     kv.Store[string, DirAquire]
 	fileProperties kv.Store[string, FileProperties]
 
@@ -70,7 +70,7 @@ func NewService(sourceFs billy.Filesystem, conf config.TorrentClient) (*Daemon,
 		return nil, fmt.Errorf("error creating metadata folder: %w", err)
 	}
 
-	s.fis, err = newFileItemStore(filepath.Join(conf.MetadataFolder, "items"), 2*time.Hour)
+	s.fis, err = newDHTStore(filepath.Join(conf.MetadataFolder, "dht-item-store"), 3*time.Hour)
 	if err != nil {
 		return nil, fmt.Errorf("error starting item store: %w", err)
 	}
@@ -149,9 +149,11 @@ func (s *Daemon) loadTorrent(ctx context.Context, f vfs.File) (*Controller, erro
 		return nil, fmt.Errorf("loading torrent metadata from file %s, error: %w", stat.Name(), err)
 	}
 
+	var ctl *Controller
 	t, ok := s.client.Torrent(mi.HashInfoBytes())
-	if !ok {
+	if ok {
-
+		ctl = s.newController(t)
+	} else {
 		span.AddEvent("torrent not found, loading from file")
 		log.Info(ctx, "torrent not found, loading from file")
 
@@ -175,11 +177,13 @@ func (s *Daemon) loadTorrent(ctx context.Context, f vfs.File) (*Controller, erro
 		}
 
 		t, _ = s.client.AddTorrentOpt(torrent.AddTorrentOpts{
-			InfoHash:  spec.InfoHash,
+			InfoHash:   spec.InfoHash,
-			Storage:   s.Storage,
+			InfoHashV2: spec.InfoHashV2,
-			InfoBytes: infoBytes,
+			Storage:    s.Storage,
-			ChunkSize: spec.ChunkSize,
+			InfoBytes:  infoBytes,
+			ChunkSize:  spec.ChunkSize,
 		})
+
 		t.AllowDataDownload()
 		t.AllowDataUpload()
 
@@ -199,6 +203,13 @@ func (s *Daemon) loadTorrent(ctx context.Context, f vfs.File) (*Controller, erro
 		}
 		span.AddEvent("got info")
 
+		ctl = s.newController(t)
+
+		err = ctl.initializeTorrentPriories(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("initialize torrent priorities: %w", err)
+		}
+
 		// info := t.Info()
 		// if info == nil {
 		// 	return nil, fmt.Errorf("info is nil")
@@ -216,13 +227,6 @@ func (s *Daemon) loadTorrent(ctx context.Context, f vfs.File) (*Controller, erro
 		// }
 	}
 
-	ctl := s.newController(t)
-
-	err = ctl.initializeTorrentPriories(ctx)
-	if err != nil {
-		log.Error(ctx, "error initializing torrent priorities", rlog.Error(err))
-	}
-
 	return ctl, nil
 }
 
@@ -232,11 +236,7 @@ func isValidInfoHashBytes(d []byte) bool {
 	return err == nil
 }
 
-func (s *Daemon) Stats() (*Stats, error) {
+func (s *Daemon) Stats() torrent.ConnStats {
-	return &Stats{}, nil
-}
-
-func (s *Daemon) GetStats() torrent.ConnStats {
 	return s.client.ConnStats()
 }
 
@@ -316,6 +316,7 @@ func (s *Daemon) newController(t *torrent.Torrent) *Controller {
 	return newController(t,
 		storeByTorrent(s.fileProperties, t.InfoHash()),
 		s.Storage,
+		s.log,
 	)
 }
 

src/sources/torrent/dht_fileitem_store.go

@@ -10,16 +10,16 @@ import (
 	"github.com/dgraph-io/badger/v4"
 )
 
-var _ bep44.Store = &fileItemStore{}
+var _ bep44.Store = &dhtFileItemStore{}
 
-type fileItemStore struct {
+type dhtFileItemStore struct {
 	ttl time.Duration
 	db  *badger.DB
 }
 
-func newFileItemStore(path string, itemsTTL time.Duration) (*fileItemStore, error) {
+func newDHTStore(path string, itemsTTL time.Duration) (*dhtFileItemStore, error) {
 	opts := badger.DefaultOptions(path).
-		WithLogger(dlog.BadgerLogger("torrent-client", "item-store")).
+		WithLogger(dlog.BadgerLogger("torrent-client", "dht-item-store")).
 		WithValueLogFileSize(1<<26 - 1)
 
 	db, err := badger.Open(opts)
@@ -32,13 +32,13 @@ func newFileItemStore(path string, itemsTTL time.Duration) (*fileItemStore, erro
 		return nil, err
 	}
 
-	return &fileItemStore{
+	return &dhtFileItemStore{
 		db:  db,
 		ttl: itemsTTL,
 	}, nil
 }
 
-func (fis *fileItemStore) Put(i *bep44.Item) error {
+func (fis *dhtFileItemStore) Put(i *bep44.Item) error {
 	tx := fis.db.NewTransaction(true)
 	defer tx.Discard()
 
@@ -58,7 +58,7 @@ func (fis *fileItemStore) Put(i *bep44.Item) error {
 	return tx.Commit()
 }
 
-func (fis *fileItemStore) Get(t bep44.Target) (*bep44.Item, error) {
+func (fis *dhtFileItemStore) Get(t bep44.Target) (*bep44.Item, error) {
 	tx := fis.db.NewTransaction(false)
 	defer tx.Discard()
 
@@ -84,11 +84,11 @@ func (fis *fileItemStore) Get(t bep44.Target) (*bep44.Item, error) {
 	return i, nil
 }
 
-func (fis *fileItemStore) Del(t bep44.Target) error {
+func (fis *dhtFileItemStore) Del(t bep44.Target) error {
 	// ignore this
 	return nil
 }
 
-func (fis *fileItemStore) Close() error {
+func (fis *dhtFileItemStore) Close() error {
 	return fis.db.Close()
 }

src/sources/torrent/file_controller.go

@@ -2,8 +2,10 @@ package torrent
 
 import (
 	"context"
+	"log/slog"
 
 	"git.kmsign.ru/royalcat/tstor/pkg/kvsingle"
+	"git.kmsign.ru/royalcat/tstor/pkg/rlog"
 	"github.com/anacrolix/torrent"
 	"github.com/anacrolix/torrent/metainfo"
 	"github.com/anacrolix/torrent/types"
@@ -13,12 +15,14 @@ import (
 type FileController struct {
 	file       *torrent.File
 	properties *kvsingle.Value[string, FileProperties]
+	log        *rlog.Logger
 }
 
-func NewFileController(f *torrent.File, properties *kvsingle.Value[string, FileProperties]) *FileController {
+func NewFileController(f *torrent.File, properties *kvsingle.Value[string, FileProperties], log *rlog.Logger) *FileController {
 	return &FileController{
 		file:       f,
 		properties: properties,
+		log:        log.WithComponent("file-controller").With(slog.String("file", f.Path())),
 	}
 }
 
@@ -38,6 +42,8 @@ func (s *FileController) Properties(ctx context.Context) (FileProperties, error)
 }
 
 func (s *FileController) SetPriority(ctx context.Context, priority types.PiecePriority) error {
+	log := s.log.With(slog.Int("priority", int(priority)))
+
 	err := s.properties.Edit(ctx, func(ctx context.Context, v FileProperties) (FileProperties, error) {
 		v.Priority = priority
 		return v, nil
@@ -55,6 +61,7 @@ func (s *FileController) SetPriority(ctx context.Context, priority types.PiecePr
 		return err
 	}
 
+	log.Debug(ctx, "file priority set")
 	s.file.SetPriority(priority)
 
 	return nil
@@ -83,6 +90,10 @@ func (s *FileController) Size() int64 {
 	return s.file.Length()
 }
 
+func (s *FileController) Priority() types.PiecePriority {
+	return s.file.Priority()
+}
+
 func (s *FileController) BytesCompleted() int64 {
 	return s.file.BytesCompleted()
 }

src/vfs/log.go

@@ -124,7 +124,7 @@ func (fs *LogFS) Open(ctx context.Context, filename string) (file File, err erro
 	if isLoggableError(err) {
 		fs.log.Error(ctx, "Failed to open file")
 	}
-	file = wrapLogFile(file, filename, fs.log, fs.readTimeout, fs.tel)
+	file = WrapLogFile(file, filename, fs.log, fs.readTimeout, fs.tel)
 
 	if file != nil {
 		fs.tel.openedFiles.Add(ctx, 1)
@@ -221,7 +221,7 @@ func (f *LogFile) Type() fs.FileMode {
 
 var _ File = (*LogFile)(nil)
 
-func wrapLogFile(f File, filename string, log *rlog.Logger, timeout time.Duration, tel *fsTelemetry) *LogFile {
+func WrapLogFile(f File, filename string, log *rlog.Logger, timeout time.Duration, tel *fsTelemetry) *LogFile {
 	return &LogFile{
 		filename: filename,
 		f:        f,

ui/lib/api/schema.graphql

@@ -106,6 +106,7 @@ type TorrentDaemonMutation {
 }
 type TorrentDaemonQuery {
 	torrents(filter: TorrentsFilter): [Torrent!]! @resolver
+	stats: TorrentStats! @resolver
 }
 type TorrentFS implements Dir & FsEntry {
 	name: String!
@@ -154,6 +155,21 @@ type TorrentProgress implements Progress {
 	current: Int!
 	total: Int!
 }
+type TorrentStats {
+	bytesWritten: Int!
+	bytesWrittenData: Int!
+	bytesRead: Int!
+	bytesReadData: Int!
+	bytesReadUsefulData: Int!
+	bytesReadUsefulIntendedData: Int!
+	chunksWritten: Int!
+	chunksRead: Int!
+	chunksReadUseful: Int!
+	chunksReadWasted: Int!
+	metadataChunksRead: Int!
+	piecesDirtiedGood: Int!
+	piecesDirtiedBad: Int!
+}
 input TorrentsFilter {
 	infohash: StringFilter
 	name: StringFilter