royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Improve logs (#89)

Browse source
This commit is contained in:
Antonio Navarro Perez 2021-11-20 11:57:25 -08:00 committed by GitHub
parent 66eadf35dc
commit 5d4e48f0f9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 343 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
assets/js/logs.js Normal file
View file

@ -0,0 +1,69 @@
Distribyted.logs = {
loadView: function () {
fetch("/api/log")
.then(response => {
if (response.ok) {
return response.body.getReader();
} else {
response.json().then(json => {
Distribyted.message.error('Error getting logs from server. Error: ' + json.error);
}).catch(error => {
Distribyted.message.error('Error getting logs from server. Error: ' + error);
})
}
})
.then(reader => {
var decoder = new TextDecoder()
var lastString = ''
reader.read().then(function processText({ done, value }) {
if (done) {
return;
}
const string = `${lastString}${decoder.decode(value)}`
const lines = string.split(/\r\n|[\r\n]/g)
this.lastString = lines.pop() || ''
lines.forEach(element => {
try {
var json = JSON.parse(element)
var properties = ""
for (let [key, value] of Object.entries(json)) {
if (key == "level" || key == "component" || key == "message" || key == "time") {
continue
}
properties += `<b>${key}</b>=${value} `
}
var tableClass = "table-primary"
switch (json.level) {
case "info":
tableClass = ""
break;
case "error":
tableClass = "table-danger"
break;
case "warn":
tableClass = "table-warning"
break;
case "debug":
tableClass = "table-info"
break;
default:
break;
}
template = `<tr class="${tableClass}"><td>${new Date(json.time*1000).toLocaleString()}</td><td>${json.level}</td><td>${json.component}</td><td>${json.message}</td><td>${properties}</td></tr>`;
document.getElementById("log_table").innerHTML += template;
} catch (err) {
// server can send some corrupted json line
console.log(err);
}
});
return reader.read().then(processText);
}).catch(err => console.log(err));
}).catch(err => console.log(err));
}
}

9
assets/js/routes.js
View file

@ -129,6 +129,7 @@ Distribyted.routes = {
.then(function (response) {
if (response.ok) {
Distribyted.message.info('Torrent deleted.')
Distribyted.routes.loadView();
} else {
response.json().then(json => {
Distribyted.message.error('Error deletting torrent. Response: ' + json.error)
@ -159,7 +160,8 @@ $("#new-magnet").submit(function (event) {
let url = '/api/routes/' + route + '/torrent'
let body = JSON.stringify({ magnet: magnet })
console.log("LOG", url, body)
document.getElementById("submit_magnet_loading").style = "display:block"
fetch(url, {
method: 'POST',
body: body
@ -167,6 +169,7 @@ $("#new-magnet").submit(function (event) {
.then(function (response) {
if (response.ok) {
Distribyted.message.info('New magnet added.')
Distribyted.routes.loadView();
} else {
response.json().then(json => {
Distribyted.message.error('Error adding new magnet. Response: ' + json.error)
@ -176,6 +179,8 @@ $("#new-magnet").submit(function (event) {
}
})
.catch(function (error) {
Distribyted.message.error('Error deletting torrent: ' + error.message)
Distribyted.message.error('Error adding torrent: ' + error.message)
}).then(function () {
document.getElementById("submit_magnet_loading").style = "display:none"
});
});

1
assets/plugins/js-yaml/js-yaml.min.js vendored
View file

File diff suppressed because one or more lines are too long

12
cmd/distribyted/main.go
View file

@ -25,10 +25,6 @@ import (
"github.com/distribyted/distribyted/webdav"
)
func init() {
dlog.Load()
}
const (
configFlag = "config"
fuseAllowOther = "fuse-allow-other"
@ -96,6 +92,8 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
return fmt.Errorf("error loading configuration: %w", err)
}
dlog.Load(conf.Log)
if err := os.MkdirAll(conf.Torrent.MetadataFolder, 0744); err != nil {
return fmt.Errorf("error creating metadata folder: %w", err)
}
@ -131,7 +129,7 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
return fmt.Errorf("error starting magnet database: %w", err)
}
ts := torrent.NewService(cl, dbl, ss, c)
ts := torrent.NewService(cl, dbl, ss, c, conf.Torrent.AddTimeout)
mh := fuse.NewHandler(fuseAllowOther || conf.Fuse.AllowOther, conf.Fuse.Path)
@ -189,7 +187,9 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
log.Warn().Msg("webDAV configuration not found!")
}()
err = http.New(fc, ss, ts, ch, port)
logFilename := filepath.Join(conf.Log.Path, dlog.FileName)
err = http.New(fc, ss, ts, ch, port, logFilename)
log.Error().Err(err).Msg("error initializing HTTP server")
return err
}

9
config/config.go
View file

@ -11,6 +11,7 @@ const (
const (
metadataFolder = "./distribyted-data/metadata"
mountFolder = "./distribyted-data/mount"
logsFolder = "./distribyted-data/logs"
)
func DefaultConfig() *Root {
@ -24,13 +25,19 @@ func DefaultConfig() *Root {
Pass: "admin",
},
Torrent: &TorrentGlobal{
GlobalCacheSize: 1024,
GlobalCacheSize: 2048,
MetadataFolder: metadataFolder,
AddTimeout: 60,
},
Fuse: &FuseGlobal{
AllowOther: false,
Path: mountFolder,
},
Log: &Log{
Path: logsFolder,
MaxBackups: 2,
MaxSize: 50,
},
Routes: []*Route{
{
Name: "multimedia",

11
config/handler.go
View file

@ -7,7 +7,6 @@ import (
"path/filepath"
"github.com/distribyted/distribyted"
"github.com/rs/zerolog/log"
"gopkg.in/yaml.v3"
)
@ -43,7 +42,7 @@ func (c *Handler) createFromTemplateFile() ([]byte, error) {
func (c *Handler) GetRaw() ([]byte, error) {
f, err := ioutil.ReadFile(c.p)
if os.IsNotExist(err) {
log.Info().Str("file", c.p).Msg("configuration file does not exist, creating from template file")
fmt.Println("configuration file does not exist, creating from template file:", c.p)
return c.createFromTemplateFile()
}
@ -69,11 +68,3 @@ func (c *Handler) Get() (*Root, error) {
return conf, nil
}
func (c *Handler) Set(b []byte) error {
if err := yaml.Unmarshal(b, &Root{}); err != nil {
return fmt.Errorf("error parsing configuration file: %w", err)
}
return ioutil.WriteFile(c.p, b, 0644)
}

10
config/model.go
View file

@ -6,11 +6,21 @@ type Root struct {
WebDAV *WebDAVGlobal `yaml:"webdav"`
Torrent *TorrentGlobal `yaml:"torrent"`
Fuse *FuseGlobal `yaml:"fuse"`
Log *Log `yaml:"log"`
Routes []*Route `yaml:"routes"`
}
type Log struct {
Debug bool `yaml:"debug"`
MaxBackups int `yaml:"max_backups"`
MaxSize int `yaml:"max_size"`
MaxAge int `yaml:"max_age"`
Path string `yaml:"path"`
}
type TorrentGlobal struct {
AddTimeout int `yaml:"add_timeout,omitempty"`
GlobalCacheSize int64 `yaml:"global_cache_size,omitempty"`
MetadataFolder string `yaml:"metadata_folder,omitempty"`
DisableIPv6 bool `yaml:"disable_ipv6,omitempty"`

1
go.mod
View file

@ -105,4 +105,5 @@ require (
golang.org/x/text v0.3.6 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
)

2
go.sum
View file

@ -598,6 +598,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

49
http/api.go
View file

@ -1,7 +1,11 @@
package http
import (
"bytes"
"io"
"math"
"net/http"
"os"
"sort"
"github.com/anacrolix/missinggo/v2/filecache"
@ -61,3 +65,48 @@ var apiDelTorrentHandler = func(s *torrent.Service) gin.HandlerFunc {
ctx.JSON(http.StatusOK, nil)
}
}
var apiLogHandler = func(path string) gin.HandlerFunc {
return func(ctx *gin.Context) {
f, err := os.Open(path)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
fi, err := f.Stat()
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
max := math.Max(float64(-fi.Size()), -1024*8*8)
_, err = f.Seek(int64(max), os.SEEK_END)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
var b bytes.Buffer
ctx.Stream(func(w io.Writer) bool {
_, err := b.ReadFrom(f)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return false
}
_, err = b.WriteTo(w)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return false
}
return true
})
if err := f.Close(); err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
}
}

4
http/http.go
View file

@ -13,7 +13,7 @@ import (
"github.com/shurcooL/httpfs/html/vfstemplate"
)
func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.Handler, port int) error {
func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.Handler, port int, logPath string) error {
gin.SetMode(gin.ReleaseMode)
r := gin.New()
r.Use(gin.Recovery())
@ -30,9 +30,11 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.
r.GET("/", indexHandler)
r.GET("/routes", routesHandler(ss))
r.GET("/logs", logsHandler)
api := r.Group("/api")
{
api.GET("/log", apiLogHandler(logPath))
api.GET("/status", apiStatusHandler(fc, ss))
api.GET("/routes", apiRoutesHandler(ss))
api.POST("/routes/:route/torrent", apiAddTorrentHandler(s))

4
http/web.go
View file

@ -16,3 +16,7 @@ var routesHandler = func(ss *torrent.Stats) gin.HandlerFunc {
c.HTML(http.StatusOK, "routes.html", ss.RoutesStats())
}
}
var logsHandler = func(c *gin.Context) {
c.HTML(http.StatusOK, "logs.html", nil)
}

40
log/log.go
View file

@ -1,16 +1,50 @@
package log
import (
"io"
"os"
"path/filepath"
"github.com/distribyted/distribyted/config"
"github.com/mattn/go-colorable"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"gopkg.in/natefinch/lumberjack.v2"
)
func Load() {
const FileName = "distribyted.log"
func Load(config *config.Log) {
var writers []io.Writer
// fix console colors on windows
cso := colorable.NewColorableStdout()
log.Logger = log.Output(zerolog.ConsoleWriter{Out: cso})
writers = append(writers, zerolog.ConsoleWriter{Out: cso})
writers = append(writers, newRollingFile(config))
mw := io.MultiWriter(writers...)
log.Logger = log.Output(mw)
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
zerolog.SetGlobalLevel(zerolog.InfoLevel)
l := zerolog.InfoLevel
if config.Debug {
l = zerolog.DebugLevel
}
zerolog.SetGlobalLevel(l)
}
func newRollingFile(config *config.Log) io.Writer {
if err := os.MkdirAll(config.Path, 0744); err != nil {
log.Error().Err(err).Str("path", config.Path).Msg("can't create log directory")
return nil
}
return &lumberjack.Logger{
Filename: filepath.Join(config.Path, FileName),
MaxBackups: config.MaxBackups, // files
MaxSize: config.MaxSize, // megabytes
MaxAge: config.MaxAge, // days
}
}

38
templates/config_template.yaml
View file

@ -1,4 +1,5 @@
# This is a configuration file example. You can edit it and add and remove torrents and magnet URIs. Read the following comments for more info.
# This is a configuration file example. You can edit it and add and remove torrents
# and magnet URIs. Read the following comments for more info.
# HTTP specific configuration.
http:
@ -12,21 +13,48 @@ webdav:
# Specific configuration for torrent backend.
torrent:
# Size in MB for the cache. This is the maximum space used by distribyted to store torrent data. Less used torrent data will be discarded if this value is reached.
# Size in MB for the cache. This is the maximum space used by distribyted to store
# torrent data. Less used torrent data will be discarded if this value is reached.
# global_cache_size: -1 #No limit
global_cache_size: 1024
global_cache_size: 2048
# Folder where distribyted metadata will be stored.
metadata_folder: ./distribyted-data/metadata
# Disable IPv6
# Disable IPv6.
#disable_ipv6: true
# Timeout in seconds when adding a magnet or a torrent.
add_timeout: 60
fuse:
# Folder where fuse will mount torrent filesystem
# For windows users: You can set here also a disk letter like X: or Z:
path: ./distribyted-data/mount
# Add this flag if you want to allow other users to access this fuse mountpoint. You need to add user_allow_other flag to /etc/fuse.conf file.
# Add this flag if you want to allow other users to access this fuse mountpoint.
# You need to add user_allow_other flag to /etc/fuse.conf file.
# allow_other: true
log:
path: ./distribyted-data/logs
# MaxBackups is the maximum number of old log files to retain. The default
# is to retain all old log files (though MaxAge may still cause them to get
# deleted.)
max_backups: 2
# MaxAge is the maximum number of days to retain old log files based on the
# timestamp encoded in their filename. Note that a day is defined as 24
# hours and may not exactly correspond to calendar days due to daylight
# savings, leap seconds, etc. The default is not to remove old log files
# based on age.
# max_age: 30
# MaxSize is the maximum size in megabytes of the log file before it gets
# rotated. It defaults to 100 megabytes.
max_size: 50
# debug: true
# List of folders where torrents will be mounted as a filesystem.
routes:
- name: multimedia

77
templates/logs.html Normal file
View file

@ -0,0 +1,77 @@
<!DOCTYPE html>
<head>
{{template "header.html" "Logs"}}
</head>
<body class="header-fixed sidebar-fixed sidebar-dark header-light" id="body">
<div class="wrapper">
{{template "navbar.html" "logs"}}
<div class="page-wrapper">
<!-- Header -->
<header class="main-header " id="header">
<nav class="navbar navbar-static-top navbar-expand-lg">
<!-- Sidebar toggle button -->
<button id="sidebar-toggler" class="sidebar-toggle">
<span class="sr-only">Toggle navigation</span>
</button>
</nav>
</header>
<div class="content-wrapper">
<div class="content">
<div class="row">
<div class="col-lg-12">
<div class="card card-default" data-scroll-height="1000"
style="height: 1000px; overflow: hidden;">
<div
class="card-header justify-content-between align-items-center card-header-border-bottom">
<h2>Logs</h2>
</div>
<div style="position: relative; overflow: hidden; width: auto; height: 100%;">
<div class="card-body"
style="overflow: hidden; width: auto; height: 100%; overflow-y: scroll; display: flex; flex-direction: column-reverse;">
<table class="table">
<thead>
<tr>
<th scope="col" style="width: 5%">Time</th>
<th scope="col" style="width: 5%">Type</th>
<th scope="col" style="width: 15%">Component</th>
<th scope="col" style="width: 40%">Message</th>
<th scope="col" style="width: 35%">Properties</th>
</tr>
</thead>
<tbody id="log_table">
</tbody>
</table>
</div>
<div class="slimScrollBar"
style="background: rgb(153, 153, 153) none repeat scroll 0% 0%; width: 5px; position: absolute; top: 0px; opacity: 0.4; display: none; border-radius: 7px; z-index: 99; right: 1px; height: 242.986px;">
</div>
<div class="slimScrollRail"
style="width: 5px; height: 100%; position: absolute; top: 0px; display: none; border-radius: 7px; background: rgb(51, 51, 51) none repeat scroll 0% 0%; opacity: 0.2; z-index: 90; right: 1px;">
</div>
</div>
<div class="mt-3"></div>
</div>
</div>
</div>
</div>
<footer class="footer mt-auto">
<div class="copyright bg-white">
</div>
</footer>
</div>
{{template "footer.html"}}
<script src="assets/js/logs.js"></script>
<script>
Distribyted.logs.loadView();
</script>
</body>
</html>

10
templates/navbar.html
View file

@ -37,6 +37,16 @@
<span class="nav-text">Routes</span>
</a>
</li>
{{if eq . "logs"}}
<li class="active">
{{else}}
<li>
{{end}}
<a class="sidenav-item-link" href="/logs">
<i class="mdi mdi-information-outline"></i>
<span class="nav-text">Logs</span>
</a>
</li>
<!-- {{if eq . "watched-folders"}}
<li class="active">
{{else}}

10
templates/routes.html
View file

@ -27,6 +27,7 @@
<div class="card card-default">
<div class="card-header card-header-border-bottom">
<h2>Add New Magnet to a Route</h2>
</div>
<div class="card-body">
<form id="new-magnet">
@ -43,9 +44,16 @@
{{end}}
</select>
</div>
<div class="form-footer pt-4 pt-5 mt-4 border-top">
<div class="form-group">
<button type="submit" class="btn btn-primary btn-default">Add</button>
</div>
<div class="form-footer pt-4 pt-5 mt-4 border-top">
<div class="sk-double-bounce" id="submit_magnet_loading" style="display: none;">
<div class="double-bounce1"></div>
<div class="double-bounce2"></div>
</div>
</div>
</form>
</div>
</div>

18
torrent/service.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"path"
"sync"
"time"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
@ -26,10 +27,15 @@ type Service struct {
db loader.LoaderAdder
log zerolog.Logger
timeout int
}
func NewService(cfg loader.Loader, db loader.LoaderAdder, stats *Stats, c *torrent.Client) *Service {
func NewService(cfg loader.Loader, db loader.LoaderAdder, stats *Stats, c *torrent.Client, timeout int) *Service {
l := log.Logger.With().Str("component", "torrent-service").Logger()
if timeout == 0 {
timeout = 60
}
return &Service{
log: l,
s: stats,
@ -37,6 +43,7 @@ func NewService(cfg loader.Loader, db loader.LoaderAdder, stats *Stats, c *torre
fss: make(map[string]fs.Filesystem),
cfgLoader: cfg,
db: db,
timeout: timeout,
}
}
@ -114,7 +121,14 @@ func (s *Service) addTorrent(r string, t *torrent.Torrent) error {
// only get info if name is not available
if t.Info() == nil {
s.log.Info().Str("hash", t.InfoHash().String()).Msg("getting torrent info")
<-t.GotInfo()
select {
case <-time.After(time.Duration(s.timeout) * time.Second):
s.log.Error().Str("hash", t.InfoHash().String()).Msg("timeout getting torrent info")
return errors.New("timeout getting torrent info")
case <-t.GotInfo():
s.log.Info().Str("hash", t.InfoHash().String()).Msg("obtained torrent info")
}
}
// Add to stats