royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Modify configuration structure. (#49)

Browse source
This commit is contained in:
Antonio Navarro Perez 2021-04-04 19:24:58 +02:00 committed by GitHub
parent cfede1d8f1
commit 5bb067be1a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 352 additions and 170 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
README.md
View file

@ -1,10 +1,11 @@
[![Releases][releases-shield]][releases-url]
[![Contributors][contributors-shield]][contributors-url]
[![Forks][forks-shield]][forks-url]
[![Stargazers][stars-shield]][stars-url]
[![Issues][issues-shield]][issues-url]
[![GPL3 License][license-shield]][license-url]
[![Coveralls][coveralls-shield]][coveralls-url]
[![Docker Image][docker-pulls-shield]][docker-pulls-url]
<!-- PROJECT LOGO -->
<br />
<p align="center">
@ -30,7 +31,7 @@
- [Table of Contents](#table-of-contents)
- [About The Project](#about-the-project)
- [Use Cases](#use-cases)
- [Supported _Expandable_ File Formats](#supported-_expandable_-file-formats)
- [Supported _Expandable_ File Formats](#supported-expandable-file-formats)
- [Supported](#supported)
- [To Be Supported](#to-be-supported)
- [Not Supported](#not-supported)
@ -39,8 +40,6 @@
- [Usage](#usage)
- [Docker](#docker)
- [Configuration File](#configuration-file)
- [root](#root)
- [mountpoints](#mountpoints)
- [Contributing](#contributing)
- [License](#license)
@ -116,11 +115,11 @@ Docker run example:
```shell
docker run \
--rm -p 4444:4444 \
--rm -p 4444:4444 -p 36911:36911 \
--cap-add SYS_ADMIN \
--device /dev/fuse \
--security-opt apparmor:unconfined \
-v /tmp/mountpoints:/distribyted-data/mountpoints:shared \
-v /tmp/mount:/distribyted-data/mount:shared \
-v /tmp/metadata:/distribyted-data/metadata \
-v /tmp/config:/distribyted-data/config \
distribyted/distribyted:latest
@ -135,8 +134,9 @@ distribyted:
restart: always
ports:
- "4444:4444/tcp"
- "36911:36911/tcp"
volumes:
- /home/user/mountpoints:/distribyted-data/mountpoints:shared
- /home/user/mount:/distribyted-data/mount:shared
- /home/user/metadata:/distribyted-data/metadata
- /home/user/config:/distribyted-data/config
security_opt:
@ -149,20 +149,7 @@ distribyted:
### Configuration File
#### root
|Config key|Description|
|-|-|
|max-cache-size| Size in MB for the cache. This is the maximum space used by distribyted to store torrent data. Less used torrent data will be discarded if this value is reached.|
|metadata-folder-name| Folder where distribyted metadata will be stored.|
|mountPoints|List of folders where torrents will be mounted as a filesystem. Possible configuration keys described [here](#mountpoints).|
#### mountpoints
|Config key|Description|
|-|-|
|path|Path where a new fuse mount will be initialized. On Windows you can use a drive letter (`X:` per example) or a folder path that **does not exist**.|
|torrents|List of `magnetUri`s or/and `torrentPath`s to be loaded on this fuse mount.|
You can see the default configuration file with some explanation comments [here](templates/config_template.yaml).
## Contributing
@ -178,6 +165,10 @@ Contributions are what make the open-source community such an amazing place to l
Distributed under the GPL3 license. See `LICENSE` for more information.
[releases-shield]: https://img.shields.io/github/v/release/distribyted/distribyted.svg?style=flat-square
[releases-url]: https://github.com/distribyted/distribyted/releases
[docker-pulls-shield]:https://img.shields.io/docker/pulls/distribyted/distribyted.svg?style=flat-square
[docker-pulls-url]:https://hub.docker.com/r/distribyted/distribyted
[contributors-shield]: https://img.shields.io/github/contributors/distribyted/distribyted.svg?style=flat-square
[contributors-url]: https://github.com/distribyted/distribyted/graphs/contributors
[forks-shield]: https://img.shields.io/github/forks/distribyted/distribyted.svg?style=flat-square

43
cmd/distribyted/main.go
View file

@ -10,6 +10,7 @@ import (
t "github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/storage"
"github.com/distribyted/distribyted/config"
"github.com/distribyted/distribyted/fs"
"github.com/distribyted/distribyted/fuse"
"github.com/distribyted/distribyted/http"
"github.com/distribyted/distribyted/stats"
@ -93,14 +94,14 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
return fmt.Errorf("error loading configuration: %w", err)
}
fc, err := newCache(conf.MetadataFolder)
fc, err := newCache(conf.Torrent.MetadataFolder)
if err != nil {
return fmt.Errorf("error creating cache: %w", err)
}
st := storage.NewResourcePieces(fc.AsResourceProvider())
c, err := torrent.NewClient(st)
c, err := torrent.NewClient(st, conf.Torrent)
if err != nil {
return fmt.Errorf("error starting torrent client: %w", err)
}
@ -109,7 +110,7 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
th := torrent.NewHandler(c, ss)
mh := fuse.NewHandler(fuseAllowOther || conf.AllowOther)
mh := fuse.NewHandler(fuseAllowOther || conf.Fuse.AllowOther, conf.Fuse.Path)
sigChan := make(chan os.Signal)
signal.Notify(sigChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
@ -121,21 +122,21 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
ch.OnReload(func(c *config.Root, ef config.EventFunc) error {
ef("unmounting filesystems")
mh.UnmountAll()
mh.Unmount()
th.RemoveAll()
ef(fmt.Sprintf("setting cache size to %d MB", c.MaxCacheSize))
fc.SetCapacity(c.MaxCacheSize * 1024 * 1024)
ef(fmt.Sprintf("setting cache size to %d MB", c.Torrent.GlobalCacheSize))
fc.SetCapacity(c.Torrent.GlobalCacheSize * 1024 * 1024)
for _, mp := range c.MountPoints {
ef(fmt.Sprintf("loading %v with %d torrents...", mp.Path, len(mp.Torrents)))
if err := th.Load(mp.Path, mp.Torrents); err != nil {
return fmt.Errorf("error loading folder %v: %w", mp.Path, err)
for _, mp := range c.Routes {
ef(fmt.Sprintf("loading %v with %d torrents...", mp.Name, len(mp.Torrents)))
if err := th.Load(mp.Name, mp.Torrents); err != nil {
return fmt.Errorf("error loading route %v: %w", mp.Name, err)
}
ef(fmt.Sprintf("%v loaded", mp.Path))
ef(fmt.Sprintf("%v loaded", mp.Name))
}
return mh.MountAll(th.Fileststems(), ef)
return mh.Mount(th.Fileststems(), ef)
})
@ -149,15 +150,23 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
go func() {
if conf.WebDAV != nil {
wdth := torrent.NewHandler(c, ss)
if err := wdth.Load("::/webDAV", conf.WebDAV.Torrents); err != nil {
log.Error().Err(err).Msg("error loading torrents for webDAV")
port = webDAVPort
if port == 0 {
port = conf.WebDAV.Port
}
if err := webdav.NewWebDAVServer(wdth.Fileststems(), webDAVPort); err != nil {
cfs, err := fs.NewContainerFs(th.Fileststems())
if err != nil {
log.Error().Err(err).Msg("error adding files to webDAV")
return
}
if err := webdav.NewWebDAVServer(cfs, port); err != nil {
log.Error().Err(err).Msg("error starting webDAV")
}
}
log.Warn().Msg("webDAV configuration not found!")
}()
err = http.New(fc, ss, ch, port)
@ -171,7 +180,7 @@ func tryClose(c *t.Client, mountService *fuse.Handler) {
log.Info().Msg("closing torrent client...")
c.Close()
log.Info().Msg("unmounting fuse filesystem...")
mountService.UnmountAll()
mountService.Unmount()
log.Info().Msg("exiting")
os.Exit(1)

55
config/config.go Normal file
View file

@ -0,0 +1,55 @@
package config
const (
m1 = "magnet:?xt=urn:btih:c9e15763f722f23e98a29decdfae341b98d53056&dn=Cosmos+Laundromat&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fcosmos-laundromat.torrent"
m2 = "magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c&dn=Big+Buck+Bunny&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fbig-buck-bunny.torrent"
m3 = "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent"
m4 = "magnet:?xt=urn:btih:209c8226b299b308beaf2b9cd3fb49212dbd13ec&dn=Tears+of+Steel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Ftears-of-steel.torrent"
m5 = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"
)
const (
metadataFolder = "./distribyted-data/metadata"
mountFolder = "./distribyted-data/mount"
)
func DefaultConfig() *Root {
return &Root{
HTTPGlobal: &HTTPGlobal{
Port: 4444,
},
WebDAV: &WebDAVGlobal{
Port: 36911,
},
Torrent: &TorrentGlobal{
GlobalCacheSize: 1024,
MetadataFolder: metadataFolder,
},
Fuse: &FuseGlobal{
AllowOther: false,
Path: mountFolder,
},
Routes: []*Route{
{
Name: "multimedia",
Torrents: []*Torrent{
{
MagnetURI: m1,
},
{
MagnetURI: m2,
},
{
MagnetURI: m3,
},
{
MagnetURI: m4,
},
{
MagnetURI: m5,
},
},
},
},
}
}

27
config/config_test.go Normal file
View file

@ -0,0 +1,27 @@
package config
import (
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
)
func TestTemplateConfig(t *testing.T) {
t.Parallel()
require := require.New(t)
f, err := os.Open("../templates/config_template.yaml")
require.NoError(err)
b, err := ioutil.ReadAll(f)
require.NoError(err)
conf := &Root{}
err = yaml.Unmarshal(b, conf)
require.NoError(err)
require.Equal(DefaultConfig(), conf)
}

55
config/model.go
View file

@ -2,35 +2,60 @@ package config
// Root is the main yaml config object
type Root struct {
MaxCacheSize int64 `yaml:"max-cache-size,omitempty"`
MetadataFolder string `yaml:"metadata-folder-name,omitempty"`
AllowOther bool `yaml:"fuse-allow-other,omitempty"`
HTTPGlobal *HTTPGlobal `yaml:"http"`
WebDAV *WebDAVGlobal `yaml:"webdav"`
Torrent *TorrentGlobal `yaml:"torrent"`
Fuse *FuseGlobal `yaml:"fuse"`
MountPoints []*MountPoint `yaml:"mountPoints"`
WebDAV *WebDAV `yaml:"webDav"`
Routes []*Route `yaml:"routes"`
}
type WebDAV struct {
Torrents []*Torrent `yaml:"torrents"`
type TorrentGlobal struct {
GlobalCacheSize int64 `yaml:"global_cache_size,omitempty"`
MetadataFolder string `yaml:"metadata_folder,omitempty"`
DisableIPv6 bool `yaml:"disable_ipv6,omitempty"`
}
type MountPoint struct {
type WebDAVGlobal struct {
Port int `yaml:"port"`
}
type HTTPGlobal struct {
Port int `yaml:"port"`
}
type FuseGlobal struct {
AllowOther bool `yaml:"allow_other,omitempty"`
Path string `yaml:"path"`
}
type Route struct {
Name string `yaml:"name"`
Torrents []*Torrent `yaml:"torrents"`
}
type Torrent struct {
MagnetURI string `yaml:"magnetUri,omitempty"`
TorrentPath string `yaml:"torrentPath,omitempty"`
FolderName string `yaml:"folderName,omitempty"`
MagnetURI string `yaml:"magnet_uri,omitempty"`
TorrentPath string `yaml:"torrent_path,omitempty"`
}
func AddDefaults(r *Root) *Root {
if r.MaxCacheSize == 0 {
r.MaxCacheSize = 1024 // 1GB
if r.Torrent == nil {
r.Torrent = &TorrentGlobal{}
}
if r.MetadataFolder == "" {
r.MetadataFolder = "./distribyted-data/metadata"
if r.Torrent.GlobalCacheSize == 0 {
r.Torrent.GlobalCacheSize = 1024 // 1GB
}
if r.Torrent.MetadataFolder == "" {
r.Torrent.MetadataFolder = metadataFolder
}
if r.Fuse == nil {
r.Fuse = &FuseGlobal{}
}
if r.Fuse.Path == "" {
r.Fuse.Path = mountFolder
}
return r

24
fs/container.go Normal file
View file

@ -0,0 +1,24 @@
package fs
type ContainerFs struct {
s *storage
}
func NewContainerFs(fss map[string]Filesystem) (*ContainerFs, error) {
s := newStorage(SupportedFactories)
for p, fs := range fss {
if err := s.AddFS(fs, p); err != nil {
return nil, err
}
}
return &ContainerFs{s: s}, nil
}
func (fs *ContainerFs) Open(filename string) (File, error) {
return fs.s.Get(filename)
}
func (fs *ContainerFs) ReadDir(path string) (map[string]File, error) {
return fs.s.Children(path), nil
}

28
fs/container_test.go Normal file
View file

@ -0,0 +1,28 @@
package fs
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestContainer(t *testing.T) {
t.Parallel()
require := require.New(t)
fss := map[string]Filesystem{
"/test": &DummyFs{},
}
c, err := NewContainerFs(fss)
require.NoError(err)
f, err := c.Open("/test/dir/here")
require.NoError(err)
require.NotNil(f)
files, err := c.ReadDir("/")
require.NoError(err)
require.Len(files, 1)
}

20
fs/storage.go
View file

@ -48,6 +48,22 @@ func (s *storage) Has(path string) bool {
return false
}
func (s *storage) AddFS(fs Filesystem, p string) error {
p = clean(p)
if s.Has(p) {
if dir, err := s.Get(p); err == nil {
if !dir.IsDir() {
return os.ErrExist
}
}
return nil
}
s.filesystems[p] = fs
return s.createParent(p, &Dir{})
}
func (s *storage) Add(f File, p string) error {
p = clean(p)
if s.Has(p) {
@ -72,9 +88,7 @@ func (s *storage) Add(f File, p string) error {
s.files[p] = f
}
s.createParent(p, f)
return nil
return s.createParent(p, f)
}
func (s *storage) createParent(p string, f File) error {

18
fs/storage_test.go
View file

@ -102,6 +102,24 @@ func TestStorageWindowsPath(t *testing.T) {
require.Equal(&Dummy{}, file)
}
func TestStorageAddFs(t *testing.T) {
t.Parallel()
require := require.New(t)
s := newStorage(dummyFactories)
err := s.AddFS(&DummyFs{}, "/test")
require.NoError(err)
f, err := s.Get("/test/dir/here/file1.txt")
require.NoError(err)
require.NotNil(f)
err = s.AddFS(&DummyFs{}, "/test")
require.Error(err)
}
var _ Filesystem = &DummyFs{}
type DummyFs struct {

45
fuse/handler.go
View file

@ -13,29 +13,34 @@ import (
type Handler struct {
fuseAllowOther bool
path string
hosts map[string]*fuse.FileSystemHost
host *fuse.FileSystemHost
}
func NewHandler(fuseAllowOther bool) *Handler {
func NewHandler(fuseAllowOther bool, path string) *Handler {
return &Handler{
fuseAllowOther: fuseAllowOther,
hosts: make(map[string]*fuse.FileSystemHost),
path: path,
}
}
func (s *Handler) MountAll(fss map[string]fs.Filesystem, ef config.EventFunc) error {
for p, fss := range fss {
folder := p
func (s *Handler) Mount(fss map[string]fs.Filesystem, ef config.EventFunc) error {
folder := s.path
// On windows, the folder must don't exist
if runtime.GOOS == "windows" {
folder = path.Dir(folder)
folder = path.Dir(s.path)
}
if err := os.MkdirAll(folder, 0744); err != nil && !os.IsExist(err) {
return err
}
host := fuse.NewFileSystemHost(NewFS(fss))
cfs, err := fs.NewContainerFs(fss)
if err != nil {
return err
}
host := fuse.NewFileSystemHost(NewFS(cfs))
// TODO improve error handling here
go func() {
@ -45,27 +50,25 @@ func (s *Handler) MountAll(fss map[string]fs.Filesystem, ef config.EventFunc) er
config = append(config, "-o", "allow_other")
}
ok := host.Mount(p, config)
ok := host.Mount(s.path, config)
if !ok {
log.Error().Str("path", p).Msg("error trying to mount filesystem")
log.Error().Str("path", s.path).Msg("error trying to mount filesystem")
}
}()
s.hosts[p] = host
}
s.host = host
return nil
}
func (s *Handler) UnmountAll() {
for path, server := range s.hosts {
log.Info().Str("path", path).Msg("unmounting")
ok := server.Unmount()
if !ok {
//TODO try to force unmount if possible
log.Error().Str("path", path).Msg("unmount failed")
}
func (s *Handler) Unmount() {
if s.host == nil {
return
}
s.hosts = make(map[string]*fuse.FileSystemHost)
ok := s.host.Unmount()
if !ok {
//TODO try to force unmount if possible
log.Error().Str("path", s.path).Msg("unmount failed")
}
}

63
templates/config_template.yaml
View file

@ -1,35 +1,44 @@
# This is a configuration file example. You can edit it and add and remove torrents and magnet URIs. Read the following comments for more info.
# HTTP specific configuration.
http:
port: 4444
# WebDAV specific configuration. Remove this to disable WebDAV.
webdav:
port: 36911
# Specific configuration for torrent backend.
torrent:
# Size in MB for the cache. This is the maximum space used by distribyted to store torrent data. Less used torrent data will be discarded if this value is reached.
# max-cache-size: -1 #No limit
max-cache-size: 1024
# Add this flag if you want to allow other users to access this fuse mountpoint. You need to add user_allow_other flag to /etc/fuse.conf file.
# fuse-allow-other: true
# global_cache_size: -1 #No limit
global_cache_size: 1024
# Folder where distribyted metadata will be stored.
metadata-folder-name: ./distribyted-data/metadata
# List of folders where torrents will be mounted as a filesystem.
mountPoints:
# Example mountpoint containing some multimedia files
metadata_folder: ./distribyted-data/metadata
# Disable IPv6
#disable_ipv6: true
fuse:
# Folder where fuse will mount torrent filesystem
# For windows users: You can set here also a disk letter like X: or Z:
- path: ./distribyted-data/mountpoints/multimedia
path: ./distribyted-data/mount
# Add this flag if you want to allow other users to access this fuse mountpoint. You need to add user_allow_other flag to /etc/fuse.conf file.
# allow_other: true
# List of folders where torrents will be mounted as a filesystem.
routes:
- name: multimedia
torrents:
# - torrentPath: /path/to/torrent/file.torrent
- magnetUri: "magnet:?xt=urn:btih:c9e15763f722f23e98a29decdfae341b98d53056&dn=Cosmos+Laundromat&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fcosmos-laundromat.torrent"
- magnetUri: "magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c&dn=Big+Buck+Bunny&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fbig-buck-bunny.torrent"
- magnetUri: "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent"
# You can also add torrents from a specific path
# - torrent_path: /path/to/torrent/file.torrent
- magnet_uri: "magnet:?xt=urn:btih:c9e15763f722f23e98a29decdfae341b98d53056&dn=Cosmos+Laundromat&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fcosmos-laundromat.torrent"
- magnet_uri: "magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c&dn=Big+Buck+Bunny&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fbig-buck-bunny.torrent"
- magnet_uri: "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent"
- magnet_uri: "magnet:?xt=urn:btih:209c8226b299b308beaf2b9cd3fb49212dbd13ec&dn=Tears+of+Steel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Ftears-of-steel.torrent"
- magnet_uri: "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"
# Example mountpoint containing some datasets, some of them compressed in zip format
# - path: ./distribyted-data/mountpoints/datasets
# - name: datasets
# torrents:
# - magnetUri: "magnet:?xt=urn:btih:9dea07ba660a722ae1008c4c8afdd303b6f6e53b&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# - magnetUri: "magnet:?xt=urn:btih:d8b3a315172c8d804528762f37fa67db14577cdb&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# - magnetUri: "magnet:?xt=urn:btih:1e0a00b9c606cf87c03e676f75929463c7756fb5&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# TODO temporal configuration for new WebDAV feature
webDav:
torrents:
# - torrentPath: /path/to/torrent/file.torrent
- magnetUri: "magnet:?xt=urn:btih:209c8226b299b308beaf2b9cd3fb49212dbd13ec&dn=Tears+of+Steel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Ftears-of-steel.torrent"
- magnetUri: "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"
# - magnet_uri: "magnet:?xt=urn:btih:9dea07ba660a722ae1008c4c8afdd303b6f6e53b&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# - magnet_uri: "magnet:?xt=urn:btih:d8b3a315172c8d804528762f37fa67db14577cdb&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"
# - magnet_uri: "magnet:?xt=urn:btih:1e0a00b9c606cf87c03e676f75929463c7756fb5&tr=http%3A%2F%2Facademictorrents.com%2Fannounce.php&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969"

6
torrent/client.go
View file

@ -4,15 +4,17 @@ import (
"github.com/anacrolix/log"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/storage"
"github.com/distribyted/distribyted/config"
)
func NewClient(st storage.ClientImpl) (*torrent.Client, error) {
func NewClient(st storage.ClientImpl, cfg *config.TorrentGlobal) (*torrent.Client, error) {
// TODO download and upload limits
torrentCfg := torrent.NewDefaultClientConfig()
torrentCfg.Logger = log.Discard
torrentCfg.Seed = true
torrentCfg.DefaultStorage = st
return torrent.NewClient(torrentCfg)
torrentCfg.DisableIPv6 = cfg.DisableIPv6
return torrent.NewClient(torrentCfg)
}

10
torrent/handler.go
View file

@ -27,7 +27,7 @@ func NewHandler(c *torrent.Client, s *stats.Torrent) *Handler {
}
}
func (s *Handler) Load(path string, ts []*config.Torrent) error {
func (s *Handler) Load(route string, ts []*config.Torrent) error {
var torrents []*torrent.Torrent
for _, mpcTorrent := range ts {
var t *torrent.Torrent
@ -36,10 +36,8 @@ func (s *Handler) Load(path string, ts []*config.Torrent) error {
switch {
case mpcTorrent.MagnetURI != "":
t, err = s.c.AddMagnet(mpcTorrent.MagnetURI)
break
case mpcTorrent.TorrentPath != "":
t, err = s.c.AddTorrentFromFile(mpcTorrent.TorrentPath)
break
default:
err = fmt.Errorf("no magnet URI or torrent path provided")
}
@ -53,13 +51,13 @@ func (s *Handler) Load(path string, ts []*config.Torrent) error {
<-t.GotInfo()
}
s.s.Add(path, t)
s.s.Add(route, t)
torrents = append(torrents, t)
log.Info().Str("name", t.Name()).Str("path", path).Msg("torrent added to mountpoint")
log.Info().Str("name", t.Name()).Str("route", route).Msg("torrent added to mountpoint")
}
folder := path
folder := "/" + route
s.fssMu.Lock()
defer s.fssMu.Unlock()

33
webdav/fs.go
View file

@ -16,16 +16,11 @@ import (
var _ webdav.FileSystem = &WebDAV{}
type WebDAV struct {
fss []fs.Filesystem
fs fs.Filesystem
}
func newFS(mFss map[string]fs.Filesystem) *WebDAV {
var fss []fs.Filesystem
for _, fs := range mFss {
fss = append(fss, fs)
}
return &WebDAV{fss: fss}
func newFS(fs fs.Filesystem) *WebDAV {
return &WebDAV{fs: fs}
}
func (wd *WebDAV) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
@ -73,35 +68,19 @@ func (wd *WebDAV) Rename(ctx context.Context, oldName, newName string) error {
}
func (wd *WebDAV) lookupFile(path string) (fs.File, error) {
for _, f := range wd.fss {
file, err := f.Open(path)
if err == os.ErrNotExist {
continue
}
if err != nil {
return nil, err
}
if file != nil {
return file, nil
}
}
return nil, os.ErrNotExist
return wd.fs.Open(path)
}
func (wd *WebDAV) listDir(path string) ([]os.FileInfo, error) {
var out []os.FileInfo
for _, ifs := range wd.fss {
files, err := ifs.ReadDir(path)
files, err := wd.fs.ReadDir(path)
if err != nil {
return nil, err
}
var out []os.FileInfo
for n, f := range files {
out = append(out, newFileInfo(n, f.Size(), f.IsDir()))
}
}
return out, nil
}

4
webdav/handler.go
View file

@ -5,10 +5,10 @@ import (
"golang.org/x/net/webdav"
)
func newHandler(fss map[string]fs.Filesystem) *webdav.Handler {
func newHandler(fs fs.Filesystem) *webdav.Handler {
return &webdav.Handler{
Prefix: "/",
FileSystem: newFS(fss),
FileSystem: newFS(fs),
LockSystem: webdav.NewMemLS(),
}
}

4
webdav/http.go
View file

@ -8,7 +8,7 @@ import (
"github.com/rs/zerolog/log"
)
func NewWebDAVServer(fss map[string]fs.Filesystem, port int) error {
func NewWebDAVServer(fs fs.Filesystem, port int) error {
log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", port)).Msg("starting webDAV server")
return http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", port), newHandler(fss))
return http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", port), newHandler(fs))
}