royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

docker deploy

Browse source
This commit is contained in:
royalcat 2023-10-18 12:52:48 +03:00
parent d30ef6cc9c
commit ec83e3b08b
16 changed files with 236 additions and 335 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
.github/workflows/codeql-analysis.yml vendored
View file

@ -1,59 +0,0 @@
name: "CodeQL"
on:
push:
branches: [ master ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ master ]
schedule:
- cron: '26 14 * * 1'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'go', 'javascript' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://git.io/codeql-language-support
steps:
- name: Checkout repository
uses: actions/checkout@v4
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2

18
.github/workflows/docker.yaml vendored
View file

@ -9,6 +9,9 @@ on:
jobs:
build-docker:
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
@ -16,22 +19,23 @@ jobs:
- linux/amd64
- linux/arm64
- linux/386
- inux/arm/v6
- linux/arm/v5
- linux/arm/v7
- linux/arm64
- linux/arm64/v8
- linux/riscv64
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
- name: Login to Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -54,7 +58,7 @@ jobs:
type=sha
- name: Build and push Docker image
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
context: .
push: true

41
Dockerfile
View file

@ -1,33 +1,24 @@
#===============
# Stage 1: Build
#===============
FROM golang:1.21 as builder
FROM golang:1.20 as builder
WORKDIR /app
ENV BIN_REPO=git.kmsign.ru/royalcat/tstor
ENV BIN_PATH=$GOPATH/src/$BIN_REPO
COPY go.mod ./
COPY go.sum ./
RUN go mod download
COPY . $BIN_PATH
WORKDIR $BIN_PATH
COPY ./src ./src
COPY ./cmd ./cmd
COPY ./assets ./assets
COPY ./templates ./templates
COPY embed.go embed.go
RUN apk add fuse-dev git gcc libc-dev g++ make
RUN go generate ./...
RUN CGO_ENABLED=0 go build -tags timetzdata -o /tstor ./cmd/tstor/main.go
RUN BIN_OUTPUT=/bin/tstor make build
#===============
# Stage 2: Run
#===============
FROM scratch
FROM alpine:3
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=builder /tstor /tstor
RUN apk add gcc libc-dev fuse-dev
COPY --from=builder /bin/tstor /bin/tstor
RUN chmod +x /bin/tstor
RUN mkdir /tstor-data
RUN echo "user_allow_other" >> /etc/fuse.conf
ENV tstor_FUSE_ALLOW_OTHER=true
ENTRYPOINT ["./bin/tstor"]
ENTRYPOINT ["/tstor"]

19
build_tools/Dockerfile
View file

@ -1,19 +0,0 @@
FROM techknowlogick/xgo:go-1.17.x
# add 32-bit and 64-bit architectures and install 7zip
RUN \
dpkg --add-architecture i386 && \
dpkg --add-architecture amd64 && \
apt-get update && \
apt-get install -y --no-install-recommends p7zip-full
# install LIBFUSE
RUN \
apt-get update && \
apt-get install -y --no-install-recommends libfuse-dev:i386 && \
apt-get install -y --no-install-recommends libfuse-dev:amd64 && \
apt-get download libfuse-dev:i386 && \
dpkg -x libfuse-dev*i386*.deb /
ENV \
OSXCROSS_NO_INCLUDE_PATH_WARNINGS 1

5
cmd/tstor/main.go
View file

@ -27,7 +27,6 @@ import (
const (
configFlag = "config"
fuseAllowOther = "fuse-allow-other"
portFlag = "http-port"
webDAVPortFlag = "webdav-port"
)
@ -45,7 +44,7 @@ func main() {
},
Action: func(c *cli.Context) error {
err := load(c.String(configFlag), c.Int(portFlag), c.Int(webDAVPortFlag), c.Bool(fuseAllowOther))
err := load(c.String(configFlag))
// stop program execution on errors to avoid flashing consoles
if err != nil && runtime.GOOS == "windows" {
@ -108,7 +107,7 @@ func (s *stc) Close() error {
return nil
}
func load(configPath string, port, webDAVPort int, fuseAllowOther bool) error {
func load(configPath string) error {
conf, err := config.Load(configPath)
if err != nil {
return fmt.Errorf("error loading configuration: %w", err)

0
http_fs.go → embed.go
View file

4
src/host/torrent/service.go
View file

@ -47,6 +47,10 @@ func (s *Service) NewTorrentFs(f vfs.File) (vfs.Filesystem, error) {
}
<-t.GotInfo()
t.AllowDataDownload()
for _, f := range t.Files() {
f.SetPriority(torrent.PiecePriorityReadahead)
}
return vfs.NewTorrentFs(t, s.readTimeout), nil
}

61
src/host/vfs/resolver.go
View file

@ -2,6 +2,7 @@ package vfs
import (
"fmt"
"io/fs"
"strings"
"sync"
)
@ -88,11 +89,11 @@ PARTS_LOOP:
}
if nestOn == -1 {
return name, nil, "", nil
return clean(name), nil, "", nil
}
fsPath = Clean(strings.Join(parts[:nestOn], Separator))
nestedFsPath = Clean(strings.Join(parts[nestOn:], Separator))
fsPath = clean(strings.Join(parts[:nestOn], Separator))
nestedFsPath = clean(strings.Join(parts[nestOn:], Separator))
// we dont need lock until now
// it must be before fsmap read to exclude race condition:
@ -119,28 +120,42 @@ PARTS_LOOP:
}
// func (r *resolver) resolveFile(name string, fs Filesystem) (File, error) {
// fsPath, nestedFs, nestedFsPath, err := r.resolvePath(name, fs)
// if err != nil {
// return nil, err
// }
var ErrNotExist = fs.ErrNotExist
// if nestedFs == nil {
// return fs.Open(fsPath)
// }
func getFile[F File](m map[string]F, name string) (File, error) {
name = clean(name)
if name == Separator {
return &Dir{}, nil
}
// return nestedFs.Open(nestedFsPath)
// }
f, ok := m[name]
if ok {
return f, nil
}
// func (r *resolver) resolveDir(name string, fs Filesystem) (map[string]File, error) {
// fsPath, nestedFs, nestedFsPath, err := r.resolvePath(name, fs)
// if err != nil {
// return nil, err
// }
for p := range m {
if strings.HasPrefix(p, name) {
return &Dir{}, nil
}
}
// if nestedFs == nil {
// return fs.ReadDir(fsPath)
// }
return nil, ErrNotExist
}
// return nestedFs.ReadDir(nestedFsPath)
// }
func listFilesInDir[F File](m map[string]F, name string) (map[string]File, error) {
name = clean(name)
out := map[string]File{}
for p, f := range m {
if strings.HasPrefix(p, name) {
parts := strings.Split(trimRelPath(p, name), Separator)
if len(parts) == 1 {
out[parts[0]] = f
} else {
out[parts[0]] = &Dir{}
}
}
}
return out, nil
}

232
src/host/vfs/resolver_test.go
View file

@ -2,6 +2,9 @@ package vfs
import (
"os"
"testing"
"github.com/stretchr/testify/require"
)
type Dummy struct {
@ -49,144 +52,121 @@ func (d *DummyFs) ReadDir(path string) (map[string]File, error) {
var _ Filesystem = &DummyFs{}
// func TestDefaultFactories(t *testing.T) {
// t.Parallel()
func TestResolver(t *testing.T) {
t.Parallel()
resolver := newResolver(ArchiveFactories)
t.Run("nested fs", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// require := require.New(t)
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("/f1.rar/f2.rar", func(path string) (File, error) {
require.Equal("/f1.rar", path)
return &Dummy{}, nil
})
require.Nil(err)
require.Equal("/f1.rar", fsPath)
require.Equal("/f2.rar", nestedFsPath)
require.IsType(&archive{}, nestedFs)
})
t.Run("root", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// require.Contains(defaultFactories, ".zip")
// require.Contains(defaultFactories, ".rar")
// require.Contains(defaultFactories, ".7z")
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("/", func(path string) (File, error) {
require.Equal("/", path)
return &Dummy{}, nil
})
require.Nil(err)
require.Nil(nestedFs)
require.Equal("/", fsPath)
require.Equal("", nestedFsPath)
})
// fs, err := defaultFactories[".zip"](&Dummy{}, nil)
// require.NoError(err)
// require.NotNil(fs)
t.Run("root dirty", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// fs, err = defaultFactories[".rar"](&Dummy{}, nil)
// require.NoError(err)
// require.NotNil(fs)
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("//.//", func(path string) (File, error) {
require.Equal("/", path)
return &Dummy{}, nil
})
require.Nil(err)
require.Nil(nestedFs)
require.Equal("/", fsPath)
require.Equal("", nestedFsPath)
})
t.Run("fs dirty", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// fs, err = defaultFactories[".7z"](&Dummy{}, nil)
// require.NoError(err)
// require.NotNil(fs)
// }
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("//.//f1.rar", func(path string) (File, error) {
require.Equal("/f1.rar", path)
return &Dummy{}, nil
})
require.Nil(err)
require.Equal("/f1.rar", fsPath)
require.Equal("/", nestedFsPath)
require.IsType(&archive{}, nestedFs)
})
t.Run("inside folder", func(t *testing.T) {
t.Parallel()
require := require.New(t)
// func TestStorageAddFs(t *testing.T) {
// t.Parallel()
fsPath, nestedFs, nestedFsPath, err := resolver.resolvePath("//test1/f1.rar", func(path string) (File, error) {
require.Equal("/test1/f1.rar", path)
return &Dummy{}, nil
})
require.Nil(err)
require.IsType(&archive{}, nestedFs)
require.Equal("/test1/f1.rar", fsPath)
require.Equal("/", nestedFsPath)
})
}
// require := require.New(t)
func TestArchiveFactories(t *testing.T) {
t.Parallel()
// s := newStorage(dummyFactories)
require := require.New(t)
// err := s.AddFS(&DummyFs{}, "/test")
// require.NoError(err)
require.Contains(ArchiveFactories, ".zip")
require.Contains(ArchiveFactories, ".rar")
require.Contains(ArchiveFactories, ".7z")
// f, err := s.Get("/test/dir/here/file1.txt")
// require.NoError(err)
// require.NotNil(f)
fs, err := ArchiveFactories[".zip"](&Dummy{})
require.NoError(err)
require.NotNil(fs)
// err = s.AddFS(&DummyFs{}, "/test")
// require.Error(err)
// }
fs, err = ArchiveFactories[".rar"](&Dummy{})
require.NoError(err)
require.NotNil(fs)
// func TestStorageWindowsPath(t *testing.T) {
// t.Parallel()
fs, err = ArchiveFactories[".7z"](&Dummy{})
require.NoError(err)
require.NotNil(fs)
}
// require := require.New(t)
func TestFiles(t *testing.T) {
t.Parallel()
require := require.New(t)
// s := newStorage(dummyFactories)
files := map[string]*Dummy{
"/test/file.txt": &Dummy{},
}
{
file, err := getFile(files, "/test")
require.Nil(err)
require.Equal(&Dir{}, file)
}
{
file, err := getFile(files, "/test/file.txt")
require.Nil(err)
require.Equal(&Dummy{}, file)
}
// err := s.Add(&Dummy{}, "\\path\\to\\dummy\\file.txt")
// require.NoError(err)
// file, err := s.Get("\\path\\to\\dummy\\file.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// file, err = s.Get("/path/to/dummy/file.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// }
// var dummyFactories = map[string]vfs.FsFactory{
// ".test": func(f vfs.File, factories map[string]vfs.FsFactory) (vfs.Filesystem, error) {
// return &DummyFs{}, nil
// },
// }
// func TestStorage(t *testing.T) {
// t.Parallel()
// require := require.New(t)
// s := newStorage(dummyFactories)
// err := s.Add(&Dummy{}, "/path/to/dummy/file.txt")
// require.NoError(err)
// err = s.Add(&Dummy{}, "/path/to/dummy/file2.txt")
// require.NoError(err)
// contains := s.Has("/path")
// require.True(contains)
// contains = s.Has("/path/to/dummy/")
// require.True(contains)
// file, err := s.Get("/path/to/dummy/file.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// file, err = s.Get("/path/to/dummy/file3.txt")
// require.Error(err)
// require.Nil(file)
// files, err := s.Children("/path/to/dummy/")
// require.NoError(err)
// require.Len(files, 2)
// require.Contains(files, "file.txt")
// require.Contains(files, "file2.txt")
// err = s.Add(&Dummy{}, "/path/to/dummy/folder/file.txt")
// require.NoError(err)
// files, err = s.Children("/path/to/dummy/")
// require.NoError(err)
// require.Len(files, 3)
// require.Contains(files, "file.txt")
// require.Contains(files, "file2.txt")
// require.Contains(files, "folder")
// err = s.Add(&Dummy{}, "path/file4.txt")
// require.NoError(err)
// require.True(s.Has("/path/file4.txt"))
// files, err = s.Children("/")
// require.NoError(err)
// require.Len(files, 1)
// err = s.Add(&Dummy{}, "/path/special_file.test")
// require.NoError(err)
// file, err = s.Get("/path/special_file.test/dir/here/file1.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// files, err = s.Children("/path/special_file.test")
// require.NoError(err)
// require.NotNil(files)
// files, err = s.Children("/path/special_file.test/dir/here")
// require.NoError(err)
// require.Len(files, 2)
// err = s.Add(&Dummy{}, "/path/to/__special__path/file3.txt")
// require.NoError(err)
// file, err = s.Get("/path/to/__special__path/file3.txt")
// require.NoError(err)
// require.Equal(&Dummy{}, file)
// s.Clear()
// }
{
out, err := listFilesInDir(files, "/test")
require.Nil(err)
require.Contains(out, "file.txt")
require.Equal(&Dummy{}, out["file.txt"])
}
}

34
src/host/vfs/torrent.go
View file

@ -33,7 +33,11 @@ func (fs *TorrentFs) files() map[string]*torrentFile {
files := make(map[string]*torrentFile)
<-fs.t.GotInfo()
for _, file := range fs.t.Files() {
p := Clean(file.Path())
if file.Priority() == torrent.PiecePriorityNone {
continue
}
p := clean(file.Path())
files[p] = &torrentFile{
readerFunc: file.NewReader,
len: file.Length(),
@ -73,6 +77,12 @@ func (fs *TorrentFs) ReadDir(name string) (map[string]File, error) {
return listFilesInDir(fs.files(), fsPath)
}
func (fs *TorrentFs) Unlink(name string) error {
file := fs.t.Files()[0]
file.SetPriority(torrent.PiecePriorityNone)
return nil
}
type reader interface {
iio.Reader
missinggo.ReadContexter
@ -111,18 +121,11 @@ func readAtLeast(r missinggo.ReadContexter, timeout int, buf []byte, min int) (n
for n < min && err == nil {
var nn int
ctx, cancel := context.WithCancel(context.Background())
timer := time.AfterFunc(
time.Duration(timeout)*time.Second,
func() {
cancel()
},
)
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
defer cancel()
nn, err = r.ReadContext(ctx, buf[n:])
n += nn
timer.Stop()
}
if n >= min {
err = nil
@ -175,15 +178,8 @@ func (d *torrentFile) Close() error {
func (d *torrentFile) Read(p []byte) (n int, err error) {
d.load()
ctx, cancel := context.WithCancel(context.Background())
timer := time.AfterFunc(
time.Duration(d.timeout)*time.Second,
func() {
cancel()
},
)
defer timer.Stop()
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(d.timeout)*time.Second)
defer cancel()
return d.reader.ReadContext(ctx, p)
}

2
src/host/vfs/torrent_test.go
View file

@ -85,6 +85,8 @@ func TestMain(m *testing.M) {
// }
func TestReadAtTorrent(t *testing.T) {
t.Parallel()
require := require.New(t)
to, err := Cli.AddMagnet(testMagnet)

43
src/host/vfs/utils.go
View file

@ -1,55 +1,14 @@
package vfs
import (
"io/fs"
"path"
"strings"
)
var ErrNotExist = fs.ErrNotExist
func getFile[F File](m map[string]F, name string) (File, error) {
name = Clean(name)
if name == Separator {
return &Dir{}, nil
}
f, ok := m[name]
if ok {
return f, nil
}
for p := range m {
if strings.HasPrefix(p, name) {
return &Dir{}, nil
}
}
return nil, ErrNotExist
}
func listFilesInDir[F File](m map[string]F, name string) (map[string]File, error) {
name = Clean(name)
out := map[string]File{}
for p, f := range m {
if strings.HasPrefix(p, name) {
parts := strings.Split(trimRelPath(p, name), Separator)
if len(parts) == 1 {
out[parts[0]] = f
} else {
out[parts[0]] = &Dir{}
}
}
}
return out, nil
}
func trimRelPath(p, t string) string {
return strings.Trim(strings.TrimPrefix(p, t), "/")
}
func Clean(p string) string {
func clean(p string) string {
return path.Clean(Separator + strings.ReplaceAll(p, "\\", "/"))
}

4
src/mounts/fuse/handler.go
View file

@ -1,3 +1,5 @@
//go:build cgo
package fuse
import (
@ -38,7 +40,7 @@ func (s *Handler) Mount(vfs vfs.Filesystem) error {
}
}
host := fuse.NewFileSystemHost(NewFS(vfs))
host := fuse.NewFileSystemHost(newFuseFS(vfs))
// TODO improve error handling here
go func() {

23
src/mounts/fuse/handler_nocgo.go Normal file
View file

@ -0,0 +1,23 @@
//go:build !cgo
package fuse
import (
"fmt"
"git.kmsign.ru/royalcat/tstor/src/host/vfs"
)
type Handler struct{}
func NewHandler(fuseAllowOther bool, path string) *Handler {
return &Handler{}
}
func (s *Handler) Mount(vfs vfs.Filesystem) error {
return fmt.Errorf("tstor was build without fuse support")
}
func (s *Handler) Unmount() {
}

24
src/mounts/fuse/mount.go
View file

@ -1,3 +1,5 @@
//go:build cgo
package fuse
import (
@ -14,22 +16,22 @@ import (
"github.com/rs/zerolog/log"
)
type FS struct {
type fuseFS struct {
fuse.FileSystemBase
fh *fileHandler
log zerolog.Logger
}
func NewFS(fs vfs.Filesystem) fuse.FileSystemInterface {
func newFuseFS(fs vfs.Filesystem) fuse.FileSystemInterface {
l := log.Logger.With().Str("component", "fuse").Logger()
return &FS{
return &fuseFS{
fh: &fileHandler{fs: fs},
log: l,
}
}
func (fs *FS) Open(path string, flags int) (errc int, fh uint64) {
func (fs *fuseFS) Open(path string, flags int) (errc int, fh uint64) {
fh, err := fs.fh.OpenHolder(path)
if os.IsNotExist(err) {
fs.log.Debug().Str("path", path).Msg("file does not exists")
@ -46,15 +48,15 @@ func (fs *FS) Open(path string, flags int) (errc int, fh uint64) {
// Unlink removes a file.
// The FileSystemBase implementation returns -ENOSYS.
func (fs *FS) Unlink(path string) int {
func (fs *fuseFS) Unlink(path string) int {
return -fuse.ENOSYS
}
func (fs *FS) Opendir(path string) (errc int, fh uint64) {
func (fs *fuseFS) Opendir(path string) (errc int, fh uint64) {
return fs.Open(path, 0)
}
func (fs *FS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
func (fs *fuseFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
if path == "/" {
stat.Mode = fuse.S_IFDIR | 0555
return 0
@ -81,7 +83,7 @@ func (fs *FS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
return 0
}
func (fs *FS) Read(path string, dest []byte, off int64, fh uint64) int {
func (fs *fuseFS) Read(path string, dest []byte, off int64, fh uint64) int {
file, err := fs.fh.GetFile(path, fh)
if os.IsNotExist(err) {
fs.log.Error().Err(err).Str("path", path).Msg("file not found on READ operation")
@ -110,7 +112,7 @@ func (fs *FS) Read(path string, dest []byte, off int64, fh uint64) int {
return n
}
func (fs *FS) Release(path string, fh uint64) int {
func (fs *fuseFS) Release(path string, fh uint64) int {
if err := fs.fh.Remove(fh); err != nil {
fs.log.Error().Err(err).Str("path", path).Msg("error getting holder when releasing file")
return -fuse.EIO
@ -119,11 +121,11 @@ func (fs *FS) Release(path string, fh uint64) int {
return 0
}
func (fs *FS) Releasedir(path string, fh uint64) int {
func (fs *fuseFS) Releasedir(path string, fh uint64) int {
return fs.Release(path, fh)
}
func (fs *FS) Readdir(path string,
func (fs *fuseFS) Readdir(path string,
fill func(name string, stat *fuse.Stat_t, ofst int64) bool,
ofst int64,
fh uint64) (errc int) {

2
src/mounts/fuse/mount_test.go
View file

@ -1,3 +1,5 @@
//go:build cgo
package fuse
import (