royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Modify configuration structure. (#49)

Browse source
This commit is contained in:
Antonio Navarro Perez 2021-04-04 19:24:58 +02:00 committed by GitHub
parent cfede1d8f1
commit 5bb067be1a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 352 additions and 170 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
README.md
View file

@ -1,10 +1,11 @@
[![Releases][releases-shield]][releases-url]
[![Contributors][contributors-shield]][contributors-url] [![Contributors][contributors-shield]][contributors-url]
[![Forks][forks-shield]][forks-url] [![Forks][forks-shield]][forks-url]
[![Stargazers][stars-shield]][stars-url] [![Stargazers][stars-shield]][stars-url]
[![Issues][issues-shield]][issues-url] [![Issues][issues-shield]][issues-url]
[![GPL3 License][license-shield]][license-url] [![GPL3 License][license-shield]][license-url]
[![Coveralls][coveralls-shield]][coveralls-url] [![Coveralls][coveralls-shield]][coveralls-url]
[![Docker Image][docker-pulls-shield]][docker-pulls-url]
<!-- PROJECT LOGO --> <!-- PROJECT LOGO -->
<br /> <br />
<p align="center"> <p align="center">
@ -30,7 +31,7 @@
- [Table of Contents](#table-of-contents) - [Table of Contents](#table-of-contents)
- [About The Project](#about-the-project) - [About The Project](#about-the-project)
- [Use Cases](#use-cases) - [Use Cases](#use-cases)
- [Supported _Expandable_ File Formats](#supported-_expandable_-file-formats) - [Supported _Expandable_ File Formats](#supported-expandable-file-formats)
- [Supported](#supported) - [Supported](#supported)
- [To Be Supported](#to-be-supported) - [To Be Supported](#to-be-supported)
- [Not Supported](#not-supported) - [Not Supported](#not-supported)
@ -39,8 +40,6 @@
- [Usage](#usage) - [Usage](#usage)
- [Docker](#docker) - [Docker](#docker)
- [Configuration File](#configuration-file) - [Configuration File](#configuration-file)
- [root](#root)
- [mountpoints](#mountpoints)
- [Contributing](#contributing) - [Contributing](#contributing)
- [License](#license) - [License](#license)
@ -116,11 +115,11 @@ Docker run example:
```shell ```shell
docker run \ docker run \
--rm -p 4444:4444 \ --rm -p 4444:4444 -p 36911:36911 \
--cap-add SYS_ADMIN \ --cap-add SYS_ADMIN \
--device /dev/fuse \ --device /dev/fuse \
--security-opt apparmor:unconfined \ --security-opt apparmor:unconfined \
-v /tmp/mountpoints:/distribyted-data/mountpoints:shared \ -v /tmp/mount:/distribyted-data/mount:shared \
-v /tmp/metadata:/distribyted-data/metadata \ -v /tmp/metadata:/distribyted-data/metadata \
-v /tmp/config:/distribyted-data/config \ -v /tmp/config:/distribyted-data/config \
distribyted/distribyted:latest distribyted/distribyted:latest
@ -135,8 +134,9 @@ distribyted:
restart: always restart: always
ports: ports:
- "4444:4444/tcp" - "4444:4444/tcp"
- "36911:36911/tcp"
volumes: volumes:
- /home/user/mountpoints:/distribyted-data/mountpoints:shared - /home/user/mount:/distribyted-data/mount:shared
- /home/user/metadata:/distribyted-data/metadata - /home/user/metadata:/distribyted-data/metadata
- /home/user/config:/distribyted-data/config - /home/user/config:/distribyted-data/config
security_opt: security_opt:
@ -149,20 +149,7 @@ distribyted:
### Configuration File ### Configuration File
#### root You can see the default configuration file with some explanation comments [here](templates/config_template.yaml).
|Config key|Description|
|-|-|
|max-cache-size| Size in MB for the cache. This is the maximum space used by distribyted to store torrent data. Less used torrent data will be discarded if this value is reached.|
|metadata-folder-name| Folder where distribyted metadata will be stored.|
|mountPoints|List of folders where torrents will be mounted as a filesystem. Possible configuration keys described [here](#mountpoints).|
#### mountpoints
|Config key|Description|
|-|-|
|path|Path where a new fuse mount will be initialized. On Windows you can use a drive letter (`X:` per example) or a folder path that **does not exist**.|
|torrents|List of `magnetUri`s or/and `torrentPath`s to be loaded on this fuse mount.|
## Contributing ## Contributing
@ -178,6 +165,10 @@ Contributions are what make the open-source community such an amazing place to l
Distributed under the GPL3 license. See `LICENSE` for more information. Distributed under the GPL3 license. See `LICENSE` for more information.
[releases-shield]: https://img.shields.io/github/v/release/distribyted/distribyted.svg?style=flat-square
[releases-url]: https://github.com/distribyted/distribyted/releases
[docker-pulls-shield]:https://img.shields.io/docker/pulls/distribyted/distribyted.svg?style=flat-square
[docker-pulls-url]:https://hub.docker.com/r/distribyted/distribyted
[contributors-shield]: https://img.shields.io/github/contributors/distribyted/distribyted.svg?style=flat-square [contributors-shield]: https://img.shields.io/github/contributors/distribyted/distribyted.svg?style=flat-square
[contributors-url]: https://github.com/distribyted/distribyted/graphs/contributors [contributors-url]: https://github.com/distribyted/distribyted/graphs/contributors
[forks-shield]: https://img.shields.io/github/forks/distribyted/distribyted.svg?style=flat-square [forks-shield]: https://img.shields.io/github/forks/distribyted/distribyted.svg?style=flat-square

43
cmd/distribyted/main.go
View file

@ -10,6 +10,7 @@ import (
t "github.com/anacrolix/torrent" t "github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/storage"
"github.com/distribyted/distribyted/config" "github.com/distribyted/distribyted/config"
"github.com/distribyted/distribyted/fs"
"github.com/distribyted/distribyted/fuse" "github.com/distribyted/distribyted/fuse"
"github.com/distribyted/distribyted/http" "github.com/distribyted/distribyted/http"
"github.com/distribyted/distribyted/stats" "github.com/distribyted/distribyted/stats"
@ -93,14 +94,14 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
return fmt.Errorf("error loading configuration: %w", err) return fmt.Errorf("error loading configuration: %w", err)
} }
fc, err := newCache(conf.MetadataFolder) fc, err := newCache(conf.Torrent.MetadataFolder)
if err != nil { if err != nil {
return fmt.Errorf("error creating cache: %w", err) return fmt.Errorf("error creating cache: %w", err)
} }
st := storage.NewResourcePieces(fc.AsResourceProvider()) st := storage.NewResourcePieces(fc.AsResourceProvider())
c, err := torrent.NewClient(st) c, err := torrent.NewClient(st, conf.Torrent)
if err != nil { if err != nil {
return fmt.Errorf("error starting torrent client: %w", err) return fmt.Errorf("error starting torrent client: %w", err)
} }
@ -109,7 +110,7 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
th := torrent.NewHandler(c, ss) th := torrent.NewHandler(c, ss)
mh := fuse.NewHandler(fuseAllowOther || conf.AllowOther) mh := fuse.NewHandler(fuseAllowOther || conf.Fuse.AllowOther, conf.Fuse.Path)
sigChan := make(chan os.Signal) sigChan := make(chan os.Signal)
signal.Notify(sigChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) signal.Notify(sigChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
@ -121,21 +122,21 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
ch.OnReload(func(c *config.Root, ef config.EventFunc) error { ch.OnReload(func(c *config.Root, ef config.EventFunc) error {
ef("unmounting filesystems") ef("unmounting filesystems")
mh.UnmountAll() mh.Unmount()
th.RemoveAll() th.RemoveAll()
ef(fmt.Sprintf("setting cache size to %d MB", c.MaxCacheSize)) ef(fmt.Sprintf("setting cache size to %d MB", c.Torrent.GlobalCacheSize))
fc.SetCapacity(c.MaxCacheSize * 1024 * 1024) fc.SetCapacity(c.Torrent.GlobalCacheSize * 1024 * 1024)
for _, mp := range c.MountPoints { for _, mp := range c.Routes {
ef(fmt.Sprintf("loading %v with %d torrents...", mp.Path, len(mp.Torrents))) ef(fmt.Sprintf("loading %v with %d torrents...", mp.Name, len(mp.Torrents)))
if err := th.Load(mp.Path, mp.Torrents); err != nil { if err := th.Load(mp.Name, mp.Torrents); err != nil {
return fmt.Errorf("error loading folder %v: %w", mp.Path, err) return fmt.Errorf("error loading route %v: %w", mp.Name, err)
} }
ef(fmt.Sprintf("%v loaded", mp.Path)) ef(fmt.Sprintf("%v loaded", mp.Name))
} }
return mh.MountAll(th.Fileststems(), ef) return mh.Mount(th.Fileststems(), ef)
}) })
@ -149,15 +150,23 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
go func() { go func() {
if conf.WebDAV != nil { if conf.WebDAV != nil {
wdth := torrent.NewHandler(c, ss) port = webDAVPort
if err := wdth.Load("::/webDAV", conf.WebDAV.Torrents); err != nil { if port == 0 {
log.Error().Err(err).Msg("error loading torrents for webDAV") port = conf.WebDAV.Port
} }
if err := webdav.NewWebDAVServer(wdth.Fileststems(), webDAVPort); err != nil { cfs, err := fs.NewContainerFs(th.Fileststems())
if err != nil {
log.Error().Err(err).Msg("error adding files to webDAV")
return
}
if err := webdav.NewWebDAVServer(cfs, port); err != nil {
log.Error().Err(err).Msg("error starting webDAV") log.Error().Err(err).Msg("error starting webDAV")
} }
} }
log.Warn().Msg("webDAV configuration not found!")
}() }()
err = http.New(fc, ss, ch, port) err = http.New(fc, ss, ch, port)
@ -171,7 +180,7 @@ func tryClose(c *t.Client, mountService *fuse.Handler) {
log.Info().Msg("closing torrent client...") log.Info().Msg("closing torrent client...")
c.Close() c.Close()
log.Info().Msg("unmounting fuse filesystem...") log.Info().Msg("unmounting fuse filesystem...")
mountService.UnmountAll() mountService.Unmount()
log.Info().Msg("exiting") log.Info().Msg("exiting")
os.Exit(1) os.Exit(1)

55
config/config.go Normal file
View file

@ -0,0 +1,55 @@
package config
const (
m1 = "magnet:?xt=urn:btih:c9e15763f722f23e98a29decdfae341b98d53056&dn=Cosmos+Laundromat&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fcosmos-laundromat.torrent"
m2 = "magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c&dn=Big+Buck+Bunny&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fbig-buck-bunny.torrent"
m3 = "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent"
m4 = "magnet:?xt=urn:btih:209c8226b299b308beaf2b9cd3fb49212dbd13ec&dn=Tears+of+Steel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Ftears-of-steel.torrent"
m5 = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"
)
const (
metadataFolder = "./distribyted-data/metadata"
mountFolder = "./distribyted-data/mount"
)
func DefaultConfig() *Root {
return &Root{
HTTPGlobal: &HTTPGlobal{
Port: 4444,
},
WebDAV: &WebDAVGlobal{
Port: 36911,
},
Torrent: &TorrentGlobal{
GlobalCacheSize: 1024,
MetadataFolder: metadataFolder,
},
Fuse: &FuseGlobal{
AllowOther: false,
Path: mountFolder,
},
Routes: []*Route{
{
Name: "multimedia",
Torrents: []*Torrent{
{
MagnetURI: m1,
},
{
MagnetURI: m2,
},
{
MagnetURI: m3,
},
{
MagnetURI: m4,
},
{
MagnetURI: m5,
},
},
},
},
}
}

27
config/config_test.go Normal file
View file

@ -0,0 +1,27 @@
package config
import (
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
)
func TestTemplateConfig(t *testing.T) {
t.Parallel()
require := require.New(t)
f, err := os.Open("../templates/config_template.yaml")
require.NoError(err)
b, err := ioutil.ReadAll(f)
require.NoError(err)
conf := &Root{}
err = yaml.Unmarshal(b, conf)
require.NoError(err)
require.Equal(DefaultConfig(), conf)
}

55
config/model.go
View file

@ -2,35 +2,60 @@ package config
// Root is the main yaml config object // Root is the main yaml config object
type Root struct { type Root struct {
MaxCacheSize int64 `yaml:"max-cache-size,omitempty"` HTTPGlobal *HTTPGlobal `yaml:"http"`
MetadataFolder string `yaml:"metadata-folder-name,omitempty"` WebDAV *WebDAVGlobal `yaml:"webdav"`
AllowOther bool `yaml:"fuse-allow-other,omitempty"` Torrent *TorrentGlobal `yaml:"torrent"`
Fuse *FuseGlobal `yaml:"fuse"`
MountPoints []*MountPoint `yaml:"mountPoints"` Routes []*Route `yaml:"routes"`
WebDAV *WebDAV `yaml:"webDav"`
} }
type WebDAV struct { type TorrentGlobal struct {
Torrents []*Torrent `yaml:"torrents"` GlobalCacheSize int64 `yaml:"global_cache_size,omitempty"`
MetadataFolder string `yaml:"metadata_folder,omitempty"`
DisableIPv6 bool `yaml:"disable_ipv6,omitempty"`
} }
type MountPoint struct { type WebDAVGlobal struct {
Port int `yaml:"port"`
}
type HTTPGlobal struct {
Port int `yaml:"port"`
}
type FuseGlobal struct {
AllowOther bool `yaml:"allow_other,omitempty"`
Path string `yaml:"path"` Path string `yaml:"path"`
}
type Route struct {
Name string `yaml:"name"`
Torrents []*Torrent `yaml:"torrents"` Torrents []*Torrent `yaml:"torrents"`
} }
type Torrent struct { type Torrent struct {
MagnetURI string `yaml:"magnetUri,omitempty"` MagnetURI string `yaml:"magnet_uri,omitempty"`
TorrentPath string `yaml:"torrentPath,omitempty"` TorrentPath string `yaml:"torrent_path,omitempty"`
FolderName string `yaml:"folderName,omitempty"`
} }
func AddDefaults(r *Root) *Root { func AddDefaults(r *Root) *Root {
if r.MaxCacheSize == 0 { if r.Torrent == nil {
r.MaxCacheSize = 1024 // 1GB r.Torrent = &TorrentGlobal{}
} }
if r.MetadataFolder == "" { if r.Torrent.GlobalCacheSize == 0 {
r.MetadataFolder = "./distribyted-data/metadata" r.Torrent.GlobalCacheSize = 1024 // 1GB
}
if r.Torrent.MetadataFolder == "" {
r.Torrent.MetadataFolder = metadataFolder
}
if r.Fuse == nil {
r.Fuse = &FuseGlobal{}
}
if r.Fuse.Path == "" {
r.Fuse.Path = mountFolder
} }
return r return r

24
fs/container.go Normal file
View file

@ -0,0 +1,24 @@
package fs
type ContainerFs struct {
s *storage
}
func NewContainerFs(fss map[string]Filesystem) (*ContainerFs, error) {
s := newStorage(SupportedFactories)
for p, fs := range fss {
if err := s.AddFS(fs, p); err != nil {
return nil, err
}
}
return &ContainerFs{s: s}, nil
}
func (fs *ContainerFs) Open(filename string) (File, error) {
return fs.s.Get(filename)
}
func (fs *ContainerFs) ReadDir(path string) (map[string]File, error) {
return fs.s.Children(path), nil
}

28
fs/container_test.go Normal file
View file

@ -0,0 +1,28 @@
package fs
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestContainer(t *testing.T) {
t.Parallel()
require := require.New(t)
fss := map[string]Filesystem{
"/test": &DummyFs{},
}
c, err := NewContainerFs(fss)
require.NoError(err)
f, err := c.Open("/test/dir/here")
require.NoError(err)
require.NotNil(f)
files, err := c.ReadDir("/")
require.NoError(err)
require.Len(files, 1)
}

20
fs/storage.go
View file

@ -48,6 +48,22 @@ func (s *storage) Has(path string) bool {
return false return false
} }
func (s *storage) AddFS(fs Filesystem, p string) error {
p = clean(p)
if s.Has(p) {
if dir, err := s.Get(p); err == nil {
if !dir.IsDir() {
return os.ErrExist
}
}
return nil
}
s.filesystems[p] = fs
return s.createParent(p, &Dir{})
}
func (s *storage) Add(f File, p string) error { func (s *storage) Add(f File, p string) error {
p = clean(p) p = clean(p)
if s.Has(p) { if s.Has(p) {
@ -72,9 +88,7 @@ func (s *storage) Add(f File, p string) error {
s.files[p] = f s.files[p] = f
} }
s.createParent(p, f) return s.createParent(p, f)
return nil
} }
func (s *storage) createParent(p string, f File) error { func (s *storage) createParent(p string, f File) error {

18
fs/storage_test.go
View file

@ -102,6 +102,24 @@ func TestStorageWindowsPath(t *testing.T) {
require.Equal(&Dummy{}, file) require.Equal(&Dummy{}, file)
} }
func TestStorageAddFs(t *testing.T) {
t.Parallel()
require := require.New(t)
s := newStorage(dummyFactories)
err := s.AddFS(&DummyFs{}, "/test")
require.NoError(err)
f, err := s.Get("/test/dir/here/file1.txt")
require.NoError(err)
require.NotNil(f)
err = s.AddFS(&DummyFs{}, "/test")
require.Error(err)
}
var _ Filesystem = &DummyFs{} var _ Filesystem = &DummyFs{}
type DummyFs struct { type DummyFs struct {

45
fuse/handler.go
View file

@ -13,29 +13,34 @@ import (
type Handler struct { type Handler struct {
fuseAllowOther bool fuseAllowOther bool
path string
hosts map[string]*fuse.FileSystemHost host *fuse.FileSystemHost
} }
func NewHandler(fuseAllowOther bool) *Handler { func NewHandler(fuseAllowOther bool, path string) *Handler {
return &Handler{ return &Handler{
fuseAllowOther: fuseAllowOther, fuseAllowOther: fuseAllowOther,
hosts: make(map[string]*fuse.FileSystemHost), path: path,
} }
} }
func (s *Handler) MountAll(fss map[string]fs.Filesystem, ef config.EventFunc) error { func (s *Handler) Mount(fss map[string]fs.Filesystem, ef config.EventFunc) error {
for p, fss := range fss { folder := s.path
folder := p
// On windows, the folder must don't exist // On windows, the folder must don't exist
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
folder = path.Dir(folder) folder = path.Dir(s.path)
} }
if err := os.MkdirAll(folder, 0744); err != nil && !os.IsExist(err) { if err := os.MkdirAll(folder, 0744); err != nil && !os.IsExist(err) {
return err return err
} }
host := fuse.NewFileSystemHost(NewFS(fss)) cfs, err := fs.NewContainerFs(fss)
if err != nil {
return err
}
host := fuse.NewFileSystemHost(NewFS(cfs))
// TODO improve error handling here // TODO improve error handling here
go func() { go func() {
@ -45,27 +50,25 @@ func (s *Handler) MountAll(fss map[string]fs.Filesystem, ef config.EventFunc) er
config = append(config, "-o", "allow_other") config = append(config, "-o", "allow_other")
} }
ok := host.Mount(p, config) ok := host.Mount(s.path, config)
if !ok { if !ok {
log.Error().Str("path", p).Msg("error trying to mount filesystem") log.Error().Str("path", s.path).Msg("error trying to mount filesystem")
} }
}() }()
s.hosts[p] = host s.host = host
}
return nil return nil
} }
func (s *Handler) UnmountAll() { func (s *Handler) Unmount() {
for path, server := range s.hosts { if s.host == nil {
log.Info().Str("path", path).Msg("unmounting") return
ok := server.Unmount()
if !ok {
//TODO try to force unmount if possible
log.Error().Str("path", path).Msg("unmount failed")
}
} }
s.hosts = make(map[string]*fuse.FileSystemHost) ok := s.host.Unmount()
if !ok {
//TODO try to force unmount if possible
log.Error().Str("path", s.path).Msg("unmount failed")
}
} }

65
templates/config_template.yaml
View file

@ -1,35 +1,44 @@
# This is a configuration file example. You can edit it and add and remove torrents and magnet URIs. Read the following comments for more info. # This is a configuration file example. You can edit it and add and remove torrents and magnet URIs. Read the following comments for more info.
# Size in MB for the cache. This is the maximum space used by distribyted to store torrent data. Less used torrent data will be discarded if this value is reached. # HTTP specific configuration.
# max-cache-size: -1 #No limit http:
max-cache-size: 1024 port: 4444
# Add this flag if you want to allow other users to access this fuse mountpoint. You need to add user_allow_other flag to /etc/fuse.conf file. # WebDAV specific configuration. Remove this to disable WebDAV.
# fuse-allow-other: true webdav:
port: 36911
# Folder where distribyted metadata will be stored. # Specific configuration for torrent backend.
metadata-folder-name: ./distribyted-data/metadata torrent:
# List of folders where torrents will be mounted as a filesystem. # Size in MB for the cache. This is the maximum space used by distribyted to store torrent data. Less used torrent data will be discarded if this value is reached.
mountPoints: # global_cache_size: -1 #No limit
# Example mountpoint containing some multimedia files global_cache_size: 1024
# Folder where distribyted metadata will be stored.
metadata_folder: ./distribyted-data/metadata
# Disable IPv6
#disable_ipv6: true
fuse:
# Folder where fuse will mount torrent filesystem
# For windows users: You can set here also a disk letter like X: or Z: # For windows users: You can set here also a disk letter like X: or Z:
- path: ./distribyted-data/mountpoints/multimedia path: ./distribyted-data/mount
torrents: # Add this flag if you want to allow other users to access this fuse mountpoint. You need to add user_allow_other flag to /etc/fuse.conf file.
# - torrentPath: /path/to/torrent/file.torrent # allow_other: true
- magnetUri: "magnet:?xt=urn:btih:c9e15763f722f23e98a29decdfae341b98d53056&dn=Cosmos+Laundromat&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fcosmos-laundromat.torrent"
- magnetUri: "magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c&dn=Big+Buck+Bunny&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fbig-buck-bunny.torrent"
- magnetUri: "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent"
# Example mountpoint containing some datasets, some of them compressed in zip format
# - path: ./distribyted-data/mountpoints/datasets
# torrents:
# - magnetUri: "magnet:?xt=urn:btih:9dea07ba660a722ae1008c4c8afdd303b6f6e53b&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# - magnetUri: "magnet:?xt=urn:btih:d8b3a315172c8d804528762f37fa67db14577cdb&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# - magnetUri: "magnet:?xt=urn:btih:1e0a00b9c606cf87c03e676f75929463c7756fb5&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# TODO temporal configuration for new WebDAV feature # List of folders where torrents will be mounted as a filesystem.
webDav: routes:
- name: multimedia
torrents: torrents:
# - torrentPath: /path/to/torrent/file.torrent # You can also add torrents from a specific path
- magnetUri: "magnet:?xt=urn:btih:209c8226b299b308beaf2b9cd3fb49212dbd13ec&dn=Tears+of+Steel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Ftears-of-steel.torrent" # - torrent_path: /path/to/torrent/file.torrent
- magnetUri: "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent" - magnet_uri: "magnet:?xt=urn:btih:c9e15763f722f23e98a29decdfae341b98d53056&dn=Cosmos+Laundromat&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fcosmos-laundromat.torrent"
- magnet_uri: "magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c&dn=Big+Buck+Bunny&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fbig-buck-bunny.torrent"
- magnet_uri: "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent"
- magnet_uri: "magnet:?xt=urn:btih:209c8226b299b308beaf2b9cd3fb49212dbd13ec&dn=Tears+of+Steel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Ftears-of-steel.torrent"
- magnet_uri: "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"
# Example mountpoint containing some datasets, some of them compressed in zip format
# - name: datasets
# torrents:
# - magnet_uri: "magnet:?xt=urn:btih:9dea07ba660a722ae1008c4c8afdd303b6f6e53b&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# - magnet_uri: "magnet:?xt=urn:btih:d8b3a315172c8d804528762f37fa67db14577cdb&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# - magnet_uri: "magnet:?xt=urn:btih:1e0a00b9c606cf87c03e676f75929463c7756fb5&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"

6
torrent/client.go
View file

@ -4,15 +4,17 @@ import (
"github.com/anacrolix/log" "github.com/anacrolix/log"
"github.com/anacrolix/torrent" "github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/storage"
"github.com/distribyted/distribyted/config"
) )
func NewClient(st storage.ClientImpl) (*torrent.Client, error) { func NewClient(st storage.ClientImpl, cfg *config.TorrentGlobal) (*torrent.Client, error) {
// TODO download and upload limits // TODO download and upload limits
torrentCfg := torrent.NewDefaultClientConfig() torrentCfg := torrent.NewDefaultClientConfig()
torrentCfg.Logger = log.Discard torrentCfg.Logger = log.Discard
torrentCfg.Seed = true torrentCfg.Seed = true
torrentCfg.DefaultStorage = st torrentCfg.DefaultStorage = st
return torrent.NewClient(torrentCfg) torrentCfg.DisableIPv6 = cfg.DisableIPv6
return torrent.NewClient(torrentCfg)
} }

10
torrent/handler.go
View file

@ -27,7 +27,7 @@ func NewHandler(c *torrent.Client, s *stats.Torrent) *Handler {
} }
} }
func (s *Handler) Load(path string, ts []*config.Torrent) error { func (s *Handler) Load(route string, ts []*config.Torrent) error {
var torrents []*torrent.Torrent var torrents []*torrent.Torrent
for _, mpcTorrent := range ts { for _, mpcTorrent := range ts {
var t *torrent.Torrent var t *torrent.Torrent
@ -36,10 +36,8 @@ func (s *Handler) Load(path string, ts []*config.Torrent) error {
switch { switch {
case mpcTorrent.MagnetURI != "": case mpcTorrent.MagnetURI != "":
t, err = s.c.AddMagnet(mpcTorrent.MagnetURI) t, err = s.c.AddMagnet(mpcTorrent.MagnetURI)
break
case mpcTorrent.TorrentPath != "": case mpcTorrent.TorrentPath != "":
t, err = s.c.AddTorrentFromFile(mpcTorrent.TorrentPath) t, err = s.c.AddTorrentFromFile(mpcTorrent.TorrentPath)
break
default: default:
err = fmt.Errorf("no magnet URI or torrent path provided") err = fmt.Errorf("no magnet URI or torrent path provided")
} }
@ -53,13 +51,13 @@ func (s *Handler) Load(path string, ts []*config.Torrent) error {
<-t.GotInfo() <-t.GotInfo()
} }
s.s.Add(path, t) s.s.Add(route, t)
torrents = append(torrents, t) torrents = append(torrents, t)
log.Info().Str("name", t.Name()).Str("path", path).Msg("torrent added to mountpoint") log.Info().Str("name", t.Name()).Str("route", route).Msg("torrent added to mountpoint")
} }
folder := path folder := "/" + route
s.fssMu.Lock() s.fssMu.Lock()
defer s.fssMu.Unlock() defer s.fssMu.Unlock()

33
webdav/fs.go
View file

@ -16,16 +16,11 @@ import (
var _ webdav.FileSystem = &WebDAV{} var _ webdav.FileSystem = &WebDAV{}
type WebDAV struct { type WebDAV struct {
fss []fs.Filesystem fs fs.Filesystem
} }
func newFS(mFss map[string]fs.Filesystem) *WebDAV { func newFS(fs fs.Filesystem) *WebDAV {
var fss []fs.Filesystem return &WebDAV{fs: fs}
for _, fs := range mFss {
fss = append(fss, fs)
}
return &WebDAV{fss: fss}
} }
func (wd *WebDAV) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) { func (wd *WebDAV) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
@ -73,35 +68,19 @@ func (wd *WebDAV) Rename(ctx context.Context, oldName, newName string) error {
} }
func (wd *WebDAV) lookupFile(path string) (fs.File, error) { func (wd *WebDAV) lookupFile(path string) (fs.File, error) {
for _, f := range wd.fss { return wd.fs.Open(path)
file, err := f.Open(path)
if err == os.ErrNotExist {
continue
}
if err != nil {
return nil, err
}
if file != nil {
return file, nil
}
}
return nil, os.ErrNotExist
} }
func (wd *WebDAV) listDir(path string) ([]os.FileInfo, error) { func (wd *WebDAV) listDir(path string) ([]os.FileInfo, error) {
var out []os.FileInfo files, err := wd.fs.ReadDir(path)
for _, ifs := range wd.fss {
files, err := ifs.ReadDir(path)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var out []os.FileInfo
for n, f := range files { for n, f := range files {
out = append(out, newFileInfo(n, f.Size(), f.IsDir())) out = append(out, newFileInfo(n, f.Size(), f.IsDir()))
} }
}
return out, nil return out, nil
} }

4
webdav/handler.go
View file

@ -5,10 +5,10 @@ import (
"golang.org/x/net/webdav" "golang.org/x/net/webdav"
) )
func newHandler(fss map[string]fs.Filesystem) *webdav.Handler { func newHandler(fs fs.Filesystem) *webdav.Handler {
return &webdav.Handler{ return &webdav.Handler{
Prefix: "/", Prefix: "/",
FileSystem: newFS(fss), FileSystem: newFS(fs),
LockSystem: webdav.NewMemLS(), LockSystem: webdav.NewMemLS(),
} }
} }

4
webdav/http.go
View file

@ -8,7 +8,7 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
func NewWebDAVServer(fss map[string]fs.Filesystem, port int) error { func NewWebDAVServer(fs fs.Filesystem, port int) error {
log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", port)).Msg("starting webDAV server") log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", port)).Msg("starting webDAV server")
return http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", port), newHandler(fss)) return http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", port), newHandler(fs))
} }