Improve logs (#89)

assets/js/logs.js

@@ -0,0 +1,69 @@
+Distribyted.logs = {
+    loadView: function () {
+        fetch("/api/log")
+            .then(response => {
+                if (response.ok) {
+                    return response.body.getReader();
+                } else {
+                    response.json().then(json => {
+                        Distribyted.message.error('Error getting logs from server. Error: ' + json.error);
+                    }).catch(error => {
+                        Distribyted.message.error('Error getting logs from server. Error: ' + error);
+                    })
+                }
+            })
+            .then(reader => {
+                var decoder = new TextDecoder()
+                var lastString = ''
+                reader.read().then(function processText({ done, value }) {
+                    if (done) {
+                        return;
+                    }
+
+                    const string = `${lastString}${decoder.decode(value)}`
+                    const lines = string.split(/\r\n|[\r\n]/g)
+                    this.lastString = lines.pop() || ''
+
+                    lines.forEach(element => {
+                        try {
+                            var json = JSON.parse(element)
+                            var properties = ""
+                            for (let [key, value] of Object.entries(json)) {
+                                if (key == "level" || key == "component" || key == "message" || key == "time") {
+                                    continue
+                                }
+
+                                properties += `<b>${key}</b>=${value} `
+                            }
+
+                            var tableClass = "table-primary"
+                            switch (json.level) {
+                                case "info":
+                                    tableClass = ""
+                                    break;
+                                case "error":
+                                    tableClass = "table-danger"
+                                    break;
+                                case "warn":
+                                    tableClass = "table-warning"
+                                    break;
+                                case "debug":
+                                    tableClass = "table-info"
+                                    break;
+                                default:
+                                    break;
+                            }
+                            template = `<tr class="${tableClass}"><td>${new Date(json.time*1000).toLocaleString()}</td><td>${json.level}</td><td>${json.component}</td><td>${json.message}</td><td>${properties}</td></tr>`;
+                            document.getElementById("log_table").innerHTML += template;
+                        } catch (err) {
+                            // server can send some corrupted json line
+                            console.log(err);
+                        }
+                    });
+
+
+                    return reader.read().then(processText);
+                }).catch(err => console.log(err));
+            }).catch(err => console.log(err));
+    }
+}

assets/js/routes.js

@@ -129,6 +129,7 @@ Distribyted.routes = {
             .then(function (response) {
                 if (response.ok) {
                     Distribyted.message.info('Torrent deleted.')
+                    Distribyted.routes.loadView();
                 } else {
                     response.json().then(json => {
                         Distribyted.message.error('Error deletting torrent. Response: ' + json.error)
@@ -159,7 +160,8 @@ $("#new-magnet").submit(function (event) {
     let url = '/api/routes/' + route + '/torrent'
     let body = JSON.stringify({ magnet: magnet })
 
-    console.log("LOG", url, body)
+    document.getElementById("submit_magnet_loading").style = "display:block"
+
     fetch(url, {
         method: 'POST',
         body: body
@@ -167,6 +169,7 @@ $("#new-magnet").submit(function (event) {
         .then(function (response) {
             if (response.ok) {
                 Distribyted.message.info('New magnet added.')
+                Distribyted.routes.loadView();
             } else {
                 response.json().then(json => {
                     Distribyted.message.error('Error adding new magnet. Response: ' + json.error)
@@ -176,6 +179,8 @@ $("#new-magnet").submit(function (event) {
             }
         })
         .catch(function (error) {
-            Distribyted.message.error('Error deletting torrent: ' + error.message)
+            Distribyted.message.error('Error adding torrent: ' + error.message)
+        }).then(function () {
+            document.getElementById("submit_magnet_loading").style = "display:none"
         });
 });

cmd/distribyted/main.go

@@ -25,10 +25,6 @@ import (
 	"github.com/distribyted/distribyted/webdav"
 )
 
-func init() {
-	dlog.Load()
-}
-
 const (
 	configFlag     = "config"
 	fuseAllowOther = "fuse-allow-other"
@@ -96,6 +92,8 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
 		return fmt.Errorf("error loading configuration: %w", err)
 	}
 
+	dlog.Load(conf.Log)
+
 	if err := os.MkdirAll(conf.Torrent.MetadataFolder, 0744); err != nil {
 		return fmt.Errorf("error creating metadata folder: %w", err)
 	}
@@ -131,7 +129,7 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
 		return fmt.Errorf("error starting magnet database: %w", err)
 	}
 
-	ts := torrent.NewService(cl, dbl, ss, c)
+	ts := torrent.NewService(cl, dbl, ss, c, conf.Torrent.AddTimeout)
 
 	mh := fuse.NewHandler(fuseAllowOther || conf.Fuse.AllowOther, conf.Fuse.Path)
 
@@ -189,7 +187,9 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
 		log.Warn().Msg("webDAV configuration not found!")
 	}()
 
-	err = http.New(fc, ss, ts, ch, port)
+	logFilename := filepath.Join(conf.Log.Path, dlog.FileName)
+
+	err = http.New(fc, ss, ts, ch, port, logFilename)
 	log.Error().Err(err).Msg("error initializing HTTP server")
 	return err
 }

config/config.go

@@ -11,6 +11,7 @@ const (
 const (
 	metadataFolder = "./distribyted-data/metadata"
 	mountFolder    = "./distribyted-data/mount"
+	logsFolder     = "./distribyted-data/logs"
 )
 
 func DefaultConfig() *Root {
@@ -24,13 +25,19 @@ func DefaultConfig() *Root {
 			Pass: "admin",
 		},
 		Torrent: &TorrentGlobal{
-			GlobalCacheSize: 1024,
+			GlobalCacheSize: 2048,
 			MetadataFolder:  metadataFolder,
+			AddTimeout:      60,
 		},
 		Fuse: &FuseGlobal{
 			AllowOther: false,
 			Path:       mountFolder,
 		},
+		Log: &Log{
+			Path:       logsFolder,
+			MaxBackups: 2,
+			MaxSize:    50,
+		},
 		Routes: []*Route{
 			{
 				Name: "multimedia",

config/handler.go

@@ -7,7 +7,6 @@ import (
 	"path/filepath"
 
 	"github.com/distribyted/distribyted"
-	"github.com/rs/zerolog/log"
 	"gopkg.in/yaml.v3"
 )
 
@@ -43,7 +42,7 @@ func (c *Handler) createFromTemplateFile() ([]byte, error) {
 func (c *Handler) GetRaw() ([]byte, error) {
 	f, err := ioutil.ReadFile(c.p)
 	if os.IsNotExist(err) {
-		log.Info().Str("file", c.p).Msg("configuration file does not exist, creating from template file")
+		fmt.Println("configuration file does not exist, creating from template file:", c.p)
 		return c.createFromTemplateFile()
 	}
 
@@ -69,11 +68,3 @@ func (c *Handler) Get() (*Root, error) {
 
 	return conf, nil
 }
-
-func (c *Handler) Set(b []byte) error {
-	if err := yaml.Unmarshal(b, &Root{}); err != nil {
-		return fmt.Errorf("error parsing configuration file: %w", err)
-	}
-
-	return ioutil.WriteFile(c.p, b, 0644)
-}

config/model.go

@@ -6,11 +6,21 @@ type Root struct {
 	WebDAV     *WebDAVGlobal  `yaml:"webdav"`
 	Torrent    *TorrentGlobal `yaml:"torrent"`
 	Fuse       *FuseGlobal    `yaml:"fuse"`
+	Log        *Log           `yaml:"log"`
 
 	Routes []*Route `yaml:"routes"`
 }
 
+type Log struct {
+	Debug      bool   `yaml:"debug"`
+	MaxBackups int    `yaml:"max_backups"`
+	MaxSize    int    `yaml:"max_size"`
+	MaxAge     int    `yaml:"max_age"`
+	Path       string `yaml:"path"`
+}
+
 type TorrentGlobal struct {
+	AddTimeout      int    `yaml:"add_timeout,omitempty"`
 	GlobalCacheSize int64  `yaml:"global_cache_size,omitempty"`
 	MetadataFolder  string `yaml:"metadata_folder,omitempty"`
 	DisableIPv6     bool   `yaml:"disable_ipv6,omitempty"`

go.mod

@@ -105,4 +105,5 @@ require (
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 )

go.sum

@@ -598,6 +598,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

http/api.go

@@ -1,7 +1,11 @@
 package http
 
 import (
+	"bytes"
+	"io"
+	"math"
 	"net/http"
+	"os"
 	"sort"
 
 	"github.com/anacrolix/missinggo/v2/filecache"
@@ -61,3 +65,48 @@ var apiDelTorrentHandler = func(s *torrent.Service) gin.HandlerFunc {
 		ctx.JSON(http.StatusOK, nil)
 	}
 }
+
+var apiLogHandler = func(path string) gin.HandlerFunc {
+	return func(ctx *gin.Context) {
+		f, err := os.Open(path)
+		if err != nil {
+			ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+
+		fi, err := f.Stat()
+		if err != nil {
+			ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+
+		max := math.Max(float64(-fi.Size()), -1024*8*8)
+		_, err = f.Seek(int64(max), os.SEEK_END)
+		if err != nil {
+			ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+
+		var b bytes.Buffer
+		ctx.Stream(func(w io.Writer) bool {
+			_, err := b.ReadFrom(f)
+			if err != nil {
+				ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+				return false
+			}
+
+			_, err = b.WriteTo(w)
+			if err != nil {
+				ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+				return false
+			}
+
+			return true
+		})
+
+		if err := f.Close(); err != nil {
+			ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+	}
+}

http/http.go

@@ -13,7 +13,7 @@ import (
 	"github.com/shurcooL/httpfs/html/vfstemplate"
 )
 
-func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.Handler, port int) error {
+func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.Handler, port int, logPath string) error {
 	gin.SetMode(gin.ReleaseMode)
 	r := gin.New()
 	r.Use(gin.Recovery())
@@ -30,9 +30,11 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.
 
 	r.GET("/", indexHandler)
 	r.GET("/routes", routesHandler(ss))
+	r.GET("/logs", logsHandler)
 
 	api := r.Group("/api")
 	{
+		api.GET("/log", apiLogHandler(logPath))
 		api.GET("/status", apiStatusHandler(fc, ss))
 		api.GET("/routes", apiRoutesHandler(ss))
 		api.POST("/routes/:route/torrent", apiAddTorrentHandler(s))

http/web.go

@@ -16,3 +16,7 @@ var routesHandler = func(ss *torrent.Stats) gin.HandlerFunc {
 		c.HTML(http.StatusOK, "routes.html", ss.RoutesStats())
 	}
 }
+
+var logsHandler = func(c *gin.Context) {
+	c.HTML(http.StatusOK, "logs.html", nil)
+}

log/log.go

@@ -1,16 +1,50 @@
 package log
 
 import (
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/distribyted/distribyted/config"
 	"github.com/mattn/go-colorable"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
-func Load() {
+const FileName = "distribyted.log"
+
+func Load(config *config.Log) {
+	var writers []io.Writer
+
 	// fix console colors on windows
 	cso := colorable.NewColorableStdout()
 
-	log.Logger = log.Output(zerolog.ConsoleWriter{Out: cso})
+	writers = append(writers, zerolog.ConsoleWriter{Out: cso})
+	writers = append(writers, newRollingFile(config))
+	mw := io.MultiWriter(writers...)
+
+	log.Logger = log.Output(mw)
 	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
-	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+
+	l := zerolog.InfoLevel
+	if config.Debug {
+		l = zerolog.DebugLevel
+	}
+
+	zerolog.SetGlobalLevel(l)
+}
+
+func newRollingFile(config *config.Log) io.Writer {
+	if err := os.MkdirAll(config.Path, 0744); err != nil {
+		log.Error().Err(err).Str("path", config.Path).Msg("can't create log directory")
+		return nil
+	}
+
+	return &lumberjack.Logger{
+		Filename:   filepath.Join(config.Path, FileName),
+		MaxBackups: config.MaxBackups, // files
+		MaxSize:    config.MaxSize,    // megabytes
+		MaxAge:     config.MaxAge,     // days
+	}
 }

templates/config_template.yaml

@@ -1,4 +1,5 @@
-# This is a configuration file example. You can edit it and add and remove torrents and magnet URIs. Read the following comments for more info.
+# This is a configuration file example. You can edit it and add and remove torrents
+# and magnet URIs. Read the following comments for more info.
 
 # HTTP specific configuration.
 http:
@@ -12,21 +13,48 @@ webdav:
 
 # Specific configuration for torrent backend.
 torrent:
-  # Size in MB for the cache. This is the maximum space used by distribyted to store torrent data. Less used torrent data will be discarded if this value is reached.
+  # Size in MB for the cache. This is the maximum space used by distribyted to store
+  # torrent data. Less used torrent data will be discarded if this value is reached.
   # global_cache_size: -1 #No limit
-  global_cache_size: 1024
+  global_cache_size: 2048
+
   # Folder where distribyted metadata will be stored.
   metadata_folder: ./distribyted-data/metadata
-  # Disable IPv6
+
+  # Disable IPv6.
   #disable_ipv6: true
 
+  # Timeout in seconds when adding a magnet or a torrent.
+  add_timeout: 60
 fuse:
   # Folder where fuse will mount torrent filesystem
   # For windows users: You can set here also a disk letter like X: or Z:
   path: ./distribyted-data/mount
-  # Add this flag if you want to allow other users to access this fuse mountpoint. You need to add user_allow_other flag to /etc/fuse.conf file.
+  # Add this flag if you want to allow other users to access this fuse mountpoint.
+  # You need to add user_allow_other flag to /etc/fuse.conf file.
   # allow_other: true 
 
+log:
+  path: ./distribyted-data/logs
+
+  # MaxBackups is the maximum number of old log files to retain.  The default
+	# is to retain all old log files (though MaxAge may still cause them to get
+	# deleted.)
+  max_backups: 2
+
+  # MaxAge is the maximum number of days to retain old log files based on the
+	# timestamp encoded in their filename.  Note that a day is defined as 24
+	# hours and may not exactly correspond to calendar days due to daylight
+	# savings, leap seconds, etc. The default is not to remove old log files
+	# based on age.
+  # max_age: 30
+
+  # MaxSize is the maximum size in megabytes of the log file before it gets
+	# rotated. It defaults to 100 megabytes.
+  max_size: 50
+
+  # debug: true
+
 # List of folders where torrents will be mounted as a filesystem.
 routes:
   - name: multimedia

templates/logs.html

@@ -0,0 +1,77 @@
+<!DOCTYPE html>
+
+<head>
+    {{template "header.html" "Logs"}}
+</head>
+
+
+<body class="header-fixed sidebar-fixed sidebar-dark header-light" id="body">
+    <div class="wrapper">
+        {{template "navbar.html" "logs"}}
+
+        <div class="page-wrapper">
+            <!-- Header -->
+            <header class="main-header " id="header">
+                <nav class="navbar navbar-static-top navbar-expand-lg">
+                    <!-- Sidebar toggle button -->
+                    <button id="sidebar-toggler" class="sidebar-toggle">
+                        <span class="sr-only">Toggle navigation</span>
+                    </button>
+                </nav>
+            </header>
+
+            <div class="content-wrapper">
+                <div class="content">
+                    <div class="row">
+                        <div class="col-lg-12">
+                            <div class="card card-default" data-scroll-height="1000"
+                                style="height: 1000px; overflow: hidden;">
+                                <div
+                                    class="card-header justify-content-between align-items-center card-header-border-bottom">
+                                    <h2>Logs</h2>
+                                </div>
+                                <div style="position: relative; overflow: hidden; width: auto; height: 100%;">
+                                    <div class="card-body"
+                                        style="overflow: hidden; width: auto; height: 100%; overflow-y: scroll; display: flex; flex-direction: column-reverse;">
+                                        <table class="table">
+                                            <thead>
+                                                <tr>
+                                                    <th scope="col" style="width: 5%">Time</th>
+                                                    <th scope="col" style="width: 5%">Type</th>
+                                                    <th scope="col" style="width: 15%">Component</th>
+                                                    <th scope="col" style="width: 40%">Message</th>
+                                                    <th scope="col" style="width: 35%">Properties</th>
+                                                </tr>
+                                            </thead>
+                                            <tbody id="log_table">
+                                            </tbody>
+                                        </table>
+                                    </div>
+                                    <div class="slimScrollBar"
+                                        style="background: rgb(153, 153, 153) none repeat scroll 0% 0%; width: 5px; position: absolute; top: 0px; opacity: 0.4; display: none; border-radius: 7px; z-index: 99; right: 1px; height: 242.986px;">
+                                    </div>
+                                    <div class="slimScrollRail"
+                                        style="width: 5px; height: 100%; position: absolute; top: 0px; display: none; border-radius: 7px; background: rgb(51, 51, 51) none repeat scroll 0% 0%; opacity: 0.2; z-index: 90; right: 1px;">
+                                    </div>
+                                </div>
+                                <div class="mt-3"></div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+
+                <footer class="footer mt-auto">
+                    <div class="copyright bg-white">
+                    </div>
+                </footer>
+            </div>
+
+            {{template "footer.html"}}
+
+            <script src="assets/js/logs.js"></script>
+            <script>
+                Distribyted.logs.loadView();
+            </script>
+</body>
+
+</html>

templates/navbar.html

@@ -37,6 +37,16 @@
             <span class="nav-text">Routes</span>
           </a>
         </li>
+        {{if eq . "logs"}}
+        <li class="active">
+          {{else}}
+        <li>
+          {{end}}
+          <a class="sidenav-item-link" href="/logs">
+            <i class="mdi mdi-information-outline"></i>
+            <span class="nav-text">Logs</span>
+          </a>
+        </li>
         <!-- {{if eq . "watched-folders"}}
         <li class="active">
           {{else}}

templates/routes.html

@@ -27,6 +27,7 @@
                     <div class="card card-default">
                         <div class="card-header card-header-border-bottom">
                             <h2>Add New Magnet to a Route</h2>
+
                         </div>
                         <div class="card-body">
                             <form id="new-magnet">
@@ -43,9 +44,16 @@
                                         {{end}}
                                     </select>
                                 </div>
-                                <div class="form-footer pt-4 pt-5 mt-4 border-top">
+                                <div class="form-group">
                                     <button type="submit" class="btn btn-primary btn-default">Add</button>
                                 </div>
+                                <div class="form-footer pt-4 pt-5 mt-4 border-top">
+                                    <div class="sk-double-bounce" id="submit_magnet_loading" style="display: none;">
+                                        <div class="double-bounce1"></div>
+                                        <div class="double-bounce2"></div>
+                                    </div>
+                                </div>
+                             
                             </form>
                         </div>
                     </div>

torrent/service.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"path"
 	"sync"
+	"time"
 
 	"github.com/anacrolix/torrent"
 	"github.com/anacrolix/torrent/metainfo"
@@ -26,10 +27,15 @@ type Service struct {
 	db        loader.LoaderAdder
 
 	log     zerolog.Logger
+	timeout int
 }
 
-func NewService(cfg loader.Loader, db loader.LoaderAdder, stats *Stats, c *torrent.Client) *Service {
+func NewService(cfg loader.Loader, db loader.LoaderAdder, stats *Stats, c *torrent.Client, timeout int) *Service {
 	l := log.Logger.With().Str("component", "torrent-service").Logger()
+
+	if timeout == 0 {
+		timeout = 60
+	}
 	return &Service{
 		log:       l,
 		s:         stats,
@@ -37,6 +43,7 @@ func NewService(cfg loader.Loader, db loader.LoaderAdder, stats *Stats, c *torre
 		fss:       make(map[string]fs.Filesystem),
 		cfgLoader: cfg,
 		db:        db,
+		timeout:   timeout,
 	}
 }
 
@@ -114,7 +121,14 @@ func (s *Service) addTorrent(r string, t *torrent.Torrent) error {
 	// only get info if name is not available
 	if t.Info() == nil {
 		s.log.Info().Str("hash", t.InfoHash().String()).Msg("getting torrent info")
-		<-t.GotInfo()
+		select {
+		case <-time.After(time.Duration(s.timeout) * time.Second):
+			s.log.Error().Str("hash", t.InfoHash().String()).Msg("timeout getting torrent info")
+			return errors.New("timeout getting torrent info")
+		case <-t.GotInfo():
+			s.log.Info().Str("hash", t.InfoHash().String()).Msg("obtained torrent info")
+		}
+
 	}
 
 	// Add to stats