From 47198b3bc6433b18284e666050980e952e1c24ee Mon Sep 17 00:00:00 2001 From: Antonio Navarro Perez Date: Tue, 23 Nov 2021 13:05:49 +0100 Subject: [PATCH] Add HTTPFS implementation (#93) --- cmd/distribyted/main.go | 8 ++- config/config.go | 4 +- config/config_test.go | 16 ++++- config/model.go | 23 ++++++- fs/fs.go | 46 ++++++++++++++ fs/fs_test.go | 24 ++++++++ fs/memory.go | 49 +++++++++++++++ fs/memory_test.go | 39 ++++++++++++ go.mod | 27 ++++---- go.sum | 69 +++++++++++++++------ http/http.go | 14 ++++- iio/reader.go | 5 ++ iio/wrapper.go | 39 ++++++++++++ iio/wrapper_test.go | 56 +++++++++++++++++ templates/config_template.yaml | 5 ++ torrent/http.go | 109 +++++++++++++++++++++++++++++++++ torrent/service.go | 4 -- webdav/fs_test.go | 56 +++++++++++++++++ 18 files changed, 549 insertions(+), 44 deletions(-) create mode 100644 fs/fs_test.go create mode 100644 fs/memory.go create mode 100644 fs/memory_test.go create mode 100644 iio/wrapper_test.go create mode 100644 torrent/http.go create mode 100644 webdav/fs_test.go diff --git a/cmd/distribyted/main.go b/cmd/distribyted/main.go index c282b52..a23325b 100644 --- a/cmd/distribyted/main.go +++ b/cmd/distribyted/main.go @@ -212,9 +212,15 @@ func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error { log.Warn().Msg("webDAV configuration not found!") }() + cfs, err := fs.NewContainerFs(fss) + if err != nil { + return fmt.Errorf("error when loading torrents: %w", err) + } + + httpfs := torrent.NewHTTPFS(cfs) logFilename := filepath.Join(conf.Log.Path, dlog.FileName) - err = http.New(fc, ss, ts, ch, servers, port, logFilename) + err = http.New(fc, ss, ts, ch, servers, httpfs, logFilename, conf.HTTPGlobal) log.Error().Err(err).Msg("error initializing HTTP server") return err } diff --git a/config/config.go b/config/config.go index 534dd5e..f807424 100644 --- a/config/config.go +++ b/config/config.go @@ -18,7 +18,9 @@ const ( func DefaultConfig() *Root { return &Root{ HTTPGlobal: &HTTPGlobal{ - Port: 4444, + Port: 4444, + IP: "0.0.0.0", + HTTPFS: true, }, WebDAV: &WebDAVGlobal{ Port: 36911, diff --git a/config/config_test.go b/config/config_test.go index 5d4517e..7a4a7fe 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -5,8 +5,8 @@ import ( "os" "testing" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" ) func TestTemplateConfig(t *testing.T) { @@ -25,3 +25,17 @@ func TestTemplateConfig(t *testing.T) { require.Equal(DefaultConfig(), conf) } + +func TestDefaults(t *testing.T) { + t.Parallel() + + require := require.New(t) + + r := &Root{} + dr := AddDefaults(r) + require.NotNil(dr) + require.NotNil(dr.Fuse) + require.NotNil(dr.HTTPGlobal) + require.NotNil(dr.Log) + require.NotNil(dr.Torrent) +} diff --git a/config/model.go b/config/model.go index 355cc61..f36e8b1 100644 --- a/config/model.go +++ b/config/model.go @@ -34,7 +34,9 @@ type WebDAVGlobal struct { } type HTTPGlobal struct { - Port int `yaml:"port"` + Port int `yaml:"port"` + IP string `yaml:"ip"` + HTTPFS bool `yaml:"httpfs"` } type FuseGlobal struct { @@ -63,8 +65,13 @@ func AddDefaults(r *Root) *Root { if r.Torrent == nil { r.Torrent = &TorrentGlobal{} } + + if r.Torrent.AddTimeout == 0 { + r.Torrent.AddTimeout = 60 + } + if r.Torrent.GlobalCacheSize == 0 { - r.Torrent.GlobalCacheSize = 1024 // 1GB + r.Torrent.GlobalCacheSize = 2048 // 2GB } if r.Torrent.MetadataFolder == "" { @@ -78,5 +85,17 @@ func AddDefaults(r *Root) *Root { r.Fuse.Path = mountFolder } + if r.HTTPGlobal == nil { + r.HTTPGlobal = &HTTPGlobal{} + } + + if r.HTTPGlobal.IP == "" { + r.HTTPGlobal.IP = "0.0.0.0" + } + + if r.Log == nil { + r.Log = &Log{} + } + return r } diff --git a/fs/fs.go b/fs/fs.go index 845ae9b..1c1514a 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -1,6 +1,9 @@ package fs import ( + "os" + "time" + "github.com/distribyted/distribyted/iio" ) @@ -21,3 +24,46 @@ type Filesystem interface { // directory entries. ReadDir(path string) (map[string]File, error) } + +type fileInfo struct { + name string + size int64 + isDir bool +} + +func NewFileInfo(name string, size int64, isDir bool) *fileInfo { + return &fileInfo{ + name: name, + size: size, + isDir: isDir, + } +} + +func (fi *fileInfo) Name() string { + return fi.name +} + +func (fi *fileInfo) Size() int64 { + return fi.size +} + +func (fi *fileInfo) Mode() os.FileMode { + if fi.isDir { + return 0555 | os.ModeDir + } + + return 0555 +} + +func (fi *fileInfo) ModTime() time.Time { + // TODO fix it + return time.Now() +} + +func (fi *fileInfo) IsDir() bool { + return fi.isDir +} + +func (fi *fileInfo) Sys() interface{} { + return nil +} diff --git a/fs/fs_test.go b/fs/fs_test.go new file mode 100644 index 0000000..d6fbb2f --- /dev/null +++ b/fs/fs_test.go @@ -0,0 +1,24 @@ +package fs + +import ( + "io/fs" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFileinfo(t *testing.T) { + t.Parallel() + + require := require.New(t) + + fi := NewFileInfo("name", 42, false) + + require.Equal(fi.IsDir(), false) + require.Equal(fi.Name(), "name") + require.Equal(fi.Size(), int64(42)) + require.NotNil(fi.ModTime()) + require.Equal(fi.Mode(), fs.FileMode(0555)) + require.Equal(fi.Sys(), nil) + +} diff --git a/fs/memory.go b/fs/memory.go new file mode 100644 index 0000000..c9fc3e9 --- /dev/null +++ b/fs/memory.go @@ -0,0 +1,49 @@ +package fs + +import ( + "bytes" +) + +var _ Filesystem = &Memory{} + +type Memory struct { + Storage *storage +} + +func NewMemory() *Memory { + return &Memory{ + Storage: newStorage(nil), + } +} + +func (fs *Memory) Open(filename string) (File, error) { + return fs.Storage.Get(filename) +} + +func (fs *Memory) ReadDir(path string) (map[string]File, error) { + return fs.Storage.Children(path), nil +} + +var _ File = &MemoryFile{} + +type MemoryFile struct { + *bytes.Reader +} + +func NewMemoryFile(data []byte) *MemoryFile { + return &MemoryFile{ + Reader: bytes.NewReader(data), + } +} + +func (d *MemoryFile) Size() int64 { + return int64(d.Reader.Len()) +} + +func (d *MemoryFile) IsDir() bool { + return false +} + +func (d *MemoryFile) Close() (err error) { + return +} diff --git a/fs/memory_test.go b/fs/memory_test.go new file mode 100644 index 0000000..fc06634 --- /dev/null +++ b/fs/memory_test.go @@ -0,0 +1,39 @@ +package fs + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMemory(t *testing.T) { + t.Parallel() + + require := require.New(t) + + mem := NewMemory() + + mem.Storage.Add(NewMemoryFile([]byte("Hello")), "/dir/here") + + fss := map[string]Filesystem{ + "/test": mem, + } + + c, err := NewContainerFs(fss) + require.NoError(err) + + f, err := c.Open("/test/dir/here") + require.NoError(err) + require.NotNil(f) + require.Equal(int64(5), f.Size()) + require.NoError(f.Close()) + + files, err := c.ReadDir("/") + require.NoError(err) + require.Len(files, 1) + + files, err = c.ReadDir("/test") + require.NoError(err) + require.Len(files, 1) + +} diff --git a/go.mod b/go.mod index cbbe0c8..f11491a 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,11 @@ go 1.17 require ( github.com/RoaringBitmap/roaring v0.9.4 // indirect - github.com/anacrolix/dht/v2 v2.12.1 + github.com/anacrolix/dht/v2 v2.13.0 github.com/anacrolix/log v0.10.0 github.com/anacrolix/missinggo/v2 v2.5.2 github.com/anacrolix/multiless v0.2.0 // indirect - github.com/anacrolix/torrent v1.37.0 + github.com/anacrolix/torrent v1.38.0 github.com/billziss-gh/cgofuse v1.5.0 github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/dgraph-io/badger/v3 v3.2103.2 @@ -16,6 +16,7 @@ require ( github.com/fsnotify/fsnotify v1.5.1 github.com/gin-gonic/gin v1.7.4 github.com/go-playground/validator/v10 v10.9.0 // indirect + github.com/goccy/go-yaml v1.9.4 github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect github.com/leodido/go-urn v1.2.1 // indirect @@ -30,12 +31,11 @@ require ( github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect - golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 - golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c // indirect + golang.org/x/net v0.0.0-20211118161319-6a13c67c3ce4 + golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) require ( @@ -60,20 +60,21 @@ require ( github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/snappy v0.0.3 // indirect - github.com/google/flatbuffers v1.12.1 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/json-iterator/go v1.1.10 // indirect - github.com/klauspost/compress v1.12.3 // indirect + github.com/klauspost/compress v1.13.6 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect @@ -95,14 +96,12 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/ugorji/go/codec v1.1.7 // indirect - github.com/willf/bitset v1.1.11 // indirect - github.com/willf/bloom v2.0.3+incompatible // indirect go.etcd.io/bbolt v1.3.6 // indirect - go.opencensus.io v0.22.5 // indirect + go.opencensus.io v0.23.0 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 2318ffb..4e96877 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,8 @@ github.com/anacrolix/chansync v0.3.0 h1:lRu9tbeuw3wl+PhMu/r+JJCRu5ArFXIluOgdF0ao github.com/anacrolix/chansync v0.3.0/go.mod h1:DZsatdsdXxD0WiwcGl0nJVwyjCKMDv+knl1q2iBjA2k= github.com/anacrolix/confluence v1.10.0 h1:xXhz98JUXzJ8223pdMRaVPwG7vrpScEzFzmbT30gCc0= github.com/anacrolix/confluence v1.10.0/go.mod h1:dEf1nWBXs0AV9w1u8m/UWugPtzgW2nTL7lYZ2cemA/Q= -github.com/anacrolix/dht/v2 v2.12.1 h1:fvE/JkfDVEHFImsrWCzUsBR/BwyZyLKxZRdUllyAZVU= -github.com/anacrolix/dht/v2 v2.12.1/go.mod h1:GfoiqvbDg0um5RVTJYf5HdPLDgk6TQeWKnPDfQ4bmWw= +github.com/anacrolix/dht/v2 v2.13.0 h1:nhEXbbwVL2fFEDqWJby+lSD0LEB06CW/Tgj74O5Ty9g= +github.com/anacrolix/dht/v2 v2.13.0/go.mod h1:zJgaiAU2yhzmchZE2mY8WyZ64LK/F/D9MAeN0ct73qQ= github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= @@ -66,8 +66,8 @@ github.com/anacrolix/sync v0.4.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DC github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= -github.com/anacrolix/torrent v1.37.0 h1:a6maknK6S0SkVrUUjgtr+w94XSdRM3xXm5mMpYZYhfg= -github.com/anacrolix/torrent v1.37.0/go.mod h1:Z/PwbVgpAthgWLagZoqip84l6XI2NUKr72M+BoF/6vg= +github.com/anacrolix/torrent v1.38.0 h1:HxSErXlfxjSnPRT9LJT4v7c6hGEFp9JLSS7eRcs3G14= +github.com/anacrolix/torrent v1.38.0/go.mod h1:QC5d3J5OQMZC67Ni2nKkvwph/vc/MA7ikHqtdiGSq7M= github.com/anacrolix/upnp v0.1.2-0.20200416075019-5e9378ed1425 h1:/Wi6l2ONI1FUFWN4cBwHOO90V4ylp4ud/eov6GUcVFk= github.com/anacrolix/upnp v0.1.2-0.20200416075019-5e9378ed1425/go.mod h1:Pz94W3kl8rf+wxH3IbCa9Sq+DTJr8OSbV2Q3/y51vYs= github.com/anacrolix/utp v0.1.0 h1:FOpQOmIwYsnENnz7tAGohA+r6iXpRjrq8ssKSre2Cp4= @@ -89,12 +89,14 @@ github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2w github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -124,6 +126,13 @@ github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A= github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -157,16 +166,20 @@ github.com/go-playground/validator/v10 v10.9.0 h1:NgTtmN58D0m8+UuxtYmGztBJB7VnPg github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/goccy/go-yaml v1.9.4 h1:S0GCYjwHKVI6IHqio7QWNKNThUl6NLzFd/g8Z65Axw8= +github.com/goccy/go-yaml v1.9.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -177,6 +190,7 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -184,23 +198,28 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.0+incompatible h1:dicJ2oXwypfwUGnB2/TYWYEKiuk9eYQlQO/AnOHl5mI= +github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -230,8 +249,9 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -247,6 +267,8 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -333,6 +355,7 @@ github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -397,10 +420,6 @@ github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= -github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -410,8 +429,9 @@ go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -446,6 +466,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -454,8 +475,8 @@ golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211118161319-6a13c67c3ce4 h1:DZshvxDdVoeKIbudAdFEKi+f70l51luSy/7b76ibTY0= +golang.org/x/net v0.0.0-20211118161319-6a13c67c3ce4/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -483,6 +504,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -498,8 +520,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c h1:DHcbWVXeY+0Y8HHKR+rbLwnoh2F4tNCY7rTiHJ30RmA= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI= +golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -514,6 +536,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -531,15 +554,24 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= @@ -569,3 +601,4 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/http/http.go b/http/http.go index 7cc4102..91fa838 100644 --- a/http/http.go +++ b/http/http.go @@ -13,7 +13,7 @@ import ( "github.com/shurcooL/httpfs/html/vfstemplate" ) -func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.Handler, tss []*torrent.Server, port int, logPath string) error { +func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config.Handler, tss []*torrent.Server, fs http.FileSystem, logPath string, cfg *config.HTTPGlobal) error { gin.SetMode(gin.ReleaseMode) r := gin.New() r.Use(gin.Recovery()) @@ -23,6 +23,14 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config. c.FileFromFS(c.Request.URL.Path, http.FS(distribyted.Assets)) }) + if cfg.HTTPFS { + log.Info().Str("host", fmt.Sprintf("%s:%d/fs", cfg.IP, cfg.Port)).Msg("starting HTTPFS") + r.GET("/fs/*filepath", func(c *gin.Context) { + path := c.Param("filepath") + c.FileFromFS(path, fs) + }) + } + t, err := vfstemplate.ParseGlob(http.FS(distribyted.Templates), nil, "/templates/*") if err != nil { return fmt.Errorf("error parsing html: %w", err) @@ -47,9 +55,9 @@ func New(fc *filecache.Cache, ss *torrent.Stats, s *torrent.Service, ch *config. } - log.Info().Str("host", fmt.Sprintf("0.0.0.0:%d", port)).Msg("starting webserver") + log.Info().Str("host", fmt.Sprintf("%s:%d", cfg.IP, cfg.Port)).Msg("starting webserver") - if err := r.Run(fmt.Sprintf(":%d", port)); err != nil { + if err := r.Run(fmt.Sprintf("%s:%d", cfg.IP, cfg.Port)); err != nil { return fmt.Errorf("error initializing server: %w", err) } diff --git a/iio/reader.go b/iio/reader.go index 998dd05..c7b45a8 100644 --- a/iio/reader.go +++ b/iio/reader.go @@ -7,3 +7,8 @@ type Reader interface { io.Closer io.Reader } + +type ReaderSeeker interface { + Reader + io.Seeker +} diff --git a/iio/wrapper.go b/iio/wrapper.go index cb25b64..47be270 100644 --- a/iio/wrapper.go +++ b/iio/wrapper.go @@ -38,3 +38,42 @@ func (rw *readAtWrapper) Close() error { return c.Close() } + +type seekerWrapper struct { + mu sync.Mutex + pos int64 + size int64 + + io.Seeker + Reader +} + +func NewSeekerWrapper(r Reader, size int64) *seekerWrapper { + return &seekerWrapper{Reader: r} +} + +func (r *seekerWrapper) Seek(offset int64, whence int) (int64, error) { + r.mu.Lock() + defer r.mu.Unlock() + + switch whence { + case io.SeekStart: + r.pos = offset + case io.SeekCurrent: + r.pos = r.pos + offset + case io.SeekEnd: + r.pos = r.size + offset + } + + return r.pos, nil +} + +func (r *seekerWrapper) Read(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + + n, err := r.ReadAt(p, r.pos) + r.pos += int64(n) + + return n, err +} diff --git a/iio/wrapper_test.go b/iio/wrapper_test.go new file mode 100644 index 0000000..de22b7c --- /dev/null +++ b/iio/wrapper_test.go @@ -0,0 +1,56 @@ +package iio_test + +import ( + "bytes" + "io" + "testing" + + "github.com/distribyted/distribyted/fs" + "github.com/distribyted/distribyted/iio" + "github.com/stretchr/testify/require" +) + +var testData []byte = []byte("Hello World") + +func TestReadAtWrapper(t *testing.T) { + t.Parallel() + + require := require.New(t) + + br := bytes.NewReader(testData) + + r := iio.NewReadAtWrapper(br) + defer r.Close() + + toRead := make([]byte, 5) + n, err := r.ReadAt(toRead, 6) + require.NoError(err) + require.Equal(5, n) + require.Equal("World", string(toRead)) + + n, err = r.ReadAt(toRead, 0) + require.NoError(err) + require.Equal(5, n) + require.Equal("Hello", string(toRead)) +} + +func TestSeekerWrapper(t *testing.T) { + t.Parallel() + + require := require.New(t) + + mf := fs.NewMemoryFile(testData) + + r := iio.NewSeekerWrapper(mf, mf.Size()) + defer r.Close() + + n, err := r.Seek(6, io.SeekStart) + require.NoError(err) + require.Equal(int64(6), n) + + toRead := make([]byte, 5) + nn, err := r.Read(toRead) + require.NoError(err) + require.Equal(5, nn) + require.Equal("World", string(toRead)) +} diff --git a/templates/config_template.yaml b/templates/config_template.yaml index 45c8b30..7480779 100644 --- a/templates/config_template.yaml +++ b/templates/config_template.yaml @@ -4,6 +4,11 @@ # HTTP specific configuration. http: port: 4444 + + ip: "0.0.0.0" + + # Serve all routes content over http on IP:PORT/fs + httpfs: true # WebDAV specific configuration. Remove this to disable WebDAV. webdav: diff --git a/torrent/http.go b/torrent/http.go new file mode 100644 index 0000000..c801ec7 --- /dev/null +++ b/torrent/http.go @@ -0,0 +1,109 @@ +package torrent + +import ( + "io" + "io/fs" + "net/http" + "os" + "sync" + + dfs "github.com/distribyted/distribyted/fs" + "github.com/distribyted/distribyted/iio" +) + +var _ http.FileSystem = &HTTPFS{} + +type HTTPFS struct { + fs dfs.Filesystem +} + +func NewHTTPFS(fs dfs.Filesystem) *HTTPFS { + return &HTTPFS{fs: fs} +} + +func (fs *HTTPFS) Open(name string) (http.File, error) { + f, err := fs.fs.Open(name) + if err != nil { + return nil, err + } + + fi := dfs.NewFileInfo(name, f.Size(), f.IsDir()) + + // TODO make this lazy + fis, err := fs.filesToFileInfo(name) + if err != nil { + return nil, err + } + + return newHTTPFile(f, fis, fi), nil +} + +func (fs *HTTPFS) filesToFileInfo(path string) ([]fs.FileInfo, error) { + files, err := fs.fs.ReadDir(path) + if err != nil { + return nil, err + } + + var out []os.FileInfo + for n, f := range files { + out = append(out, dfs.NewFileInfo(n, f.Size(), f.IsDir())) + } + + return out, nil +} + +var _ http.File = &httpFile{} + +type httpFile struct { + iio.ReaderSeeker + + mu sync.Mutex + // dirPos is protected by mu. + dirPos int + dirContent []os.FileInfo + + fi fs.FileInfo +} + +func newHTTPFile(f dfs.File, fis []fs.FileInfo, fi fs.FileInfo) *httpFile { + return &httpFile{ + dirContent: fis, + fi: fi, + + ReaderSeeker: iio.NewSeekerWrapper(f, f.Size()), + } +} + +func (f *httpFile) Readdir(count int) ([]fs.FileInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if !f.fi.IsDir() { + return nil, os.ErrInvalid + } + + old := f.dirPos + if old >= len(f.dirContent) { + // The os.File Readdir docs say that at the end of a directory, + // the error is io.EOF if count > 0 and nil if count <= 0. + if count > 0 { + return nil, io.EOF + } + return nil, nil + } + if count > 0 { + f.dirPos += count + if f.dirPos > len(f.dirContent) { + f.dirPos = len(f.dirContent) + } + } else { + f.dirPos = len(f.dirContent) + old = 0 + } + + return f.dirContent[old:f.dirPos], nil +} + +func (f *httpFile) Stat() (fs.FileInfo, error) { + return f.fi, nil +} diff --git a/torrent/service.go b/torrent/service.go index 2c2faa7..c2b13ac 100644 --- a/torrent/service.go +++ b/torrent/service.go @@ -32,10 +32,6 @@ type Service struct { func NewService(cfg loader.Loader, db loader.LoaderAdder, stats *Stats, c *torrent.Client, timeout int) *Service { l := log.Logger.With().Str("component", "torrent-service").Logger() - - if timeout == 0 { - timeout = 60 - } return &Service{ log: l, s: stats, diff --git a/webdav/fs_test.go b/webdav/fs_test.go new file mode 100644 index 0000000..0394ee5 --- /dev/null +++ b/webdav/fs_test.go @@ -0,0 +1,56 @@ +package webdav + +import ( + "context" + "io" + "os" + "testing" + + "github.com/distribyted/distribyted/fs" + "github.com/stretchr/testify/require" +) + +func TestWebDAVFilesystem(t *testing.T) { + t.Parallel() + + require := require.New(t) + + mfs := fs.NewMemory() + mf := fs.NewMemoryFile([]byte("test file content.")) + err := mfs.Storage.Add(mf, "/folder/file.txt") + require.NoError(err) + + wfs := newFS(mfs) + + dir, err := wfs.OpenFile(context.Background(), "/", 0, 0) + require.NoError(err) + + fi, err := dir.Readdir(0) + require.NoError(err) + require.Len(fi, 1) + require.Equal("folder", fi[0].Name()) + + file, err := wfs.OpenFile(context.Background(), "/folder/file.txt", 0, 0) + require.NoError(err) + _, err = file.Readdir(0) + require.ErrorIs(err, os.ErrInvalid) + + n, err := file.Seek(5, io.SeekStart) + require.NoError(err) + require.Equal(int64(5), n) + + br := make([]byte, 4) + nn, err := file.Read(br) + require.NoError(err) + require.Equal(4, nn) + require.Equal([]byte("file"), br) + + n, err = file.Seek(0, io.SeekStart) + require.NoError(err) + require.Equal(int64(0), n) + + nn, err = file.Read(br) + require.NoError(err) + require.Equal(4, nn) + require.Equal([]byte("test"), br) +}