From 1614af438e8ea5a7a0f635a51c782a435dce6bb4 Mon Sep 17 00:00:00 2001 From: Antonio Navarro Perez Date: Sun, 1 Nov 2020 12:23:39 +0100 Subject: [PATCH] Build on several platforms (#13) - Added a GitHub workflow to compile distribyted on linux, macOS and windows. - Fixed some problems on storage when running on windows. - Added more documentation about how to use on windows. - Now on every build, artifacts are generated and can be downloaded for testing before a release. - Only cross-compiling arm-7 for now. Fixes #6 Signed-off-by: Antonio Navarro Perez --- .github/workflows/build.yaml | 128 ++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 32 --------- .github/workflows/test.yaml | 27 ------- Makefile | 24 +++---- README.md | 7 +- build_tools/Dockerfile | 17 ----- fs/storage.go | 13 ++-- fs/storage_test.go | 19 +++++ 8 files changed, 170 insertions(+), 97 deletions(-) create mode 100644 .github/workflows/build.yaml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..6fe6522 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,128 @@ +--- +name: build + +# Trigger the workflow on push or pull request +on: + push: + branches: + - '*' + tags: + - '*' + pull_request: + +jobs: + build: + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + job_name: ['linux', 'mac', 'windows_amd64'] + + include: + - job_name: linux + os: ubuntu-latest + go: '1.15.x' + + - job_name: mac + os: macOS-latest + go: '1.15.x' + + - job_name: windows_amd64 + os: windows-latest + go: '1.15.x' + + name: ${{ matrix.job_name }} + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v2 + with: + stable: 'false' + go-version: ${{ matrix.go }} + + - name: Install Libraries on Linux + shell: bash + run: | + sudo modprobe fuse + sudo chmod 666 /dev/fuse + sudo chown root:$USER /etc/fuse.conf + sudo apt-get install fuse libfuse-dev rpm pkg-config + if: matrix.os == 'ubuntu-latest' + + - name: Install Libraries on macOS + shell: bash + run: | + brew untap local/homebrew-openssl # workaround for https://github.com/actions/virtual-environments/issues/1811 + brew untap local/homebrew-python2 # workaround for https://github.com/actions/virtual-environments/issues/1811 + brew update + brew cask install osxfuse + if: matrix.os == 'macOS-latest' + + - name: Install Libraries on Windows + shell: powershell + run: | + $ProgressPreference = 'SilentlyContinue' + choco install -y winfsp zip + echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + if ($env:GOARCH -eq "386") { + choco install -y mingw --forcex86 --force + echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + } + # Copy mingw32-make.exe to make.exe so the same command line + # can be used on Windows as on macOS and Linux + $path = (get-command mingw32-make.exe).Path + Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe') + if: matrix.os == 'windows-latest' + + - name: Print Go version and environment + shell: bash + run: | + printf "Using go at: $(which go)\n" + printf "Go version: $(go version)\n" + printf "\n\nGo environment:\n\n" + go env + printf "\n\nSystem environment:\n\n" + env + + - name: Go module cache + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Build + shell: bash + run: | + make build + - name: Cross-compile + shell: bash + run: | + make cross-compile + if: matrix.os == 'ubuntu-latest' + + - name: Run tests + shell: bash + run: | + make test + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + if-no-files-found: error + name: build-${{ matrix.job_name }} + path: bin/* + + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + files: bin/* \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 2bb7b86..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Release - -on: - push: - tags: - - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 - -jobs: - release: - name: Upload Release Assets - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Install fuse - run: sudo apt-get install libfuse-dev gcc - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.15.x - - name: Compile project - run: make compile - - name: Upload Release - uses: softprops/action-gh-release@v0.1.5 - with: - draft: true - prerelease: false - fail_on_unmatched_files: true - files: | - bin/* - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml deleted file mode 100644 index c40b2d9..0000000 --- a/.github/workflows/test.yaml +++ /dev/null @@ -1,27 +0,0 @@ -on: [push, pull_request] -name: Test -jobs: - test: - strategy: - matrix: - go-version: [1.15.x] - platform: [ubuntu-latest] - # platform: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.platform }} - steps: - - name: Install fuse - run: sudo apt-get install libfuse-dev gcc - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - name: Checkout code - uses: actions/checkout@v2 - - name: Test - run: make test - - name: Code Coverage - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: true - - name: Build - run: make build diff --git a/Makefile b/Makefile index a0f7a15..622e8b8 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,10 @@ VERSION := $(shell git describe --tags) BUILD := $(shell git rev-parse --short HEAD) PROJECTNAME := $(shell basename "$(PWD)") +go-cross-compile: GOPATH=~/go +go-cross-compile: ORGPATH=$(GOPATH)/src/github.com/distribyted +go-cross-compile: REPOPATH=$(ORGPATH)/distribyted + # Use linker flags to provide version/build settings LDFLAGS=-X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -linkmode external @@ -21,33 +25,25 @@ build: go-generate go-build test: go test -v --race -coverprofile=coverage.out ./... -compile: go-generate go-compile +## cross-compile: compile for other platforms using xgo. +cross-compile: go-generate go-cross-compile go-build: @echo " > Building binary..." - go build -o bin/distribyted -tags "release" cmd/distribyted/main.go + go build -o bin/distribyted-$(VERSION)-`go env GOOS`-`go env GOARCH``go env GOEXE` -tags "release" cmd/distribyted/main.go go-generate: @echo " > Generating code files..." go generate ./... -go-compile: - GOPATH=~/go - ORGPATH=$(GOPATH)/src/github.com/distribyted - REPOPATH=$(ORGPATH)/distribyted - +go-cross-compile: @echo " > Compiling for several platforms..." - go install src.techknowlogick.com/xgo + GO111MODULE=off go get -u src.techknowlogick.com/xgo docker build ./build_tools/ -t distribyted/xgo-cgofuse mkdir -p $(ORGPATH) ln -sfrnT . $(REPOPATH) - @echo " > Compiling for windows..." - GOPATH=$(GOPATH) xgo -out bin/distribyted-$(VERSION) -image=distribyted/xgo-cgofuse -ldflags='-extldflags "-static" $(LDFLAGS)' -tags="release" -targets=windows/amd64 $(REPOPATH)/cmd/distribyted/ - @echo " > Compiling for linux..." - GOPATH=$(GOPATH) xgo -out bin/distribyted-$(VERSION) -image=distribyted/xgo-cgofuse -ldflags='$(LDFLAGS)' -tags="release" -targets=linux/arm-7,linux/amd64 $(REPOPATH)/cmd/distribyted/ -# @echo " > Compiling for darwin..." -# GOPATH=$(GOPATH) xgo -out bin/distribyted-$(VERSION) -image=distribyted/xgo-cgofuse -ldflags='$(LDFLAGS)' -tags="release" -targets=darwin/* $(REPOPATH)/cmd/distribyted/ + GOPATH=$(GOPATH) xgo -out bin/distribyted-$(VERSION) -image=distribyted/xgo-cgofuse -ldflags='$(LDFLAGS)' -tags="release" -targets=linux/arm-7 $(REPOPATH)/cmd/distribyted/ .PHONY: help all: help diff --git a/README.md b/README.md index 25dff11..99d1597 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ - [To Be Supported](#to-be-supported) - [Not Supported](#not-supported) - [Getting Started](#getting-started) + - [Prerequisites on windows](#prerequisites-on-windows) - [Usage](#usage) - [Configuration File](#configuration-file) - [root](#root) @@ -83,6 +84,10 @@ Use the [example config file][example-config] and modify it as needed. Run the program: `distribyted-binary path/to/config/file.yaml` +### Prerequisites on windows + +Download and install [WinFsp](http://www.secfs.net/winfsp/). + ## Usage After executing and load all torrent or magnet files, a web interface will be available with information about the mounted routes and torrent files like download/upload speed, leechers, seeders... @@ -101,7 +106,7 @@ After executing and load all torrent or magnet files, a web interface will be av |Config key|Description| |-|-| -|path|Path where a new fuse mount will be initialized.| +|path|Path where a new fuse mount will be initialized. On Windows you can use a drive letter (`X:` per example) or a folder path that **does not exist**.| |torrents|List of `magnetUri`s or/and `torrentPath`s to be loaded on this fuse mount.| ## Contributing diff --git a/build_tools/Dockerfile b/build_tools/Dockerfile index f84ba10..91b63d5 100644 --- a/build_tools/Dockerfile +++ b/build_tools/Dockerfile @@ -7,17 +7,6 @@ RUN \ apt-get update && \ apt-get install -y --no-install-recommends p7zip-full -# install OSXFUSE -# (TODO) not compatible anymore. Find a new way to install osxfuse -# RUN wget -q -O osxfuse.dmg https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.8.3/osxfuse-3.8.3.dmg -# RUN 7z e osxfuse.dmg 0.hfs -# RUN 7z e 0.hfs "FUSE for macOS/Extras/FUSE for macOS 3.8.3.pkg" -# RUN 7z e "FUSE for macOS 3.8.3.pkg" Core.pkg/Payload -# RUN 7z e Payload -# RUN 7z x Payload~ -o/tmp -# RUN cp -R /tmp/usr/local/include/osxfuse /usr/local/include -# RUN cp /tmp/usr/local/lib/libosxfuse_i64.2.dylib /usr/local/lib/libosxfuse.dylib - # install LIBFUSE RUN \ apt-get update && \ @@ -26,11 +15,5 @@ RUN \ apt-get download libfuse-dev:i386 && \ dpkg -x libfuse-dev*i386*.deb / -# install WinFsp-FUSE -RUN \ - wget -q -O winfsp.zip \ - https://github.com/billziss-gh/winfsp/archive/release/1.4.zip && \ - 7z e winfsp.zip 'winfsp-release-1.4/inc/fuse/*' -o/usr/local/include/winfsp - ENV \ OSXCROSS_NO_INCLUDE_PATH_WARNINGS 1 diff --git a/fs/storage.go b/fs/storage.go index 04f9d1c..0ab0042 100644 --- a/fs/storage.go +++ b/fs/storage.go @@ -3,10 +3,11 @@ package fs import ( "os" "path" - "path/filepath" "strings" ) +const separator = "/" + type FsFactory func(f File) (Filesystem, error) var SupportedFactories = map[string]FsFactory{ @@ -76,8 +77,8 @@ func (s *storage) Add(f File, p string) error { return nil } -func (s *storage) createParent(path string, f File) error { - base, filename := filepath.Split(path) +func (s *storage) createParent(p string, f File) error { + base, filename := path.Split(p) base = clean(base) if err := s.Add(&Dir{}, base); err != nil { @@ -128,7 +129,7 @@ func (s *storage) Get(path string) (File, error) { func (s *storage) getFileFromFs(p string) (File, error) { for fsp, fs := range s.filesystems { if strings.HasPrefix(p, fsp) { - return fs.Open(string(os.PathSeparator) + strings.TrimPrefix(p, fsp)) + return fs.Open(separator + strings.TrimPrefix(p, fsp)) } } @@ -146,6 +147,6 @@ func (s *storage) getDirFromFs(p string) (map[string]File, error) { return nil, os.ErrNotExist } -func clean(path string) string { - return filepath.Clean(string(os.PathSeparator) + filepath.FromSlash(path)) +func clean(p string) string { + return path.Clean(separator + strings.ReplaceAll(p, "\\", "/")) } diff --git a/fs/storage_test.go b/fs/storage_test.go index ba84d03..20e5821 100644 --- a/fs/storage_test.go +++ b/fs/storage_test.go @@ -83,6 +83,25 @@ func TestStorage(t *testing.T) { require.Equal(&Dummy{}, file) } +func TestStorageWindowsPath(t *testing.T) { + t.Parallel() + + require := require.New(t) + + s := newStorage(dummyFactories) + + err := s.Add(&Dummy{}, "\\path\\to\\dummy\\file.txt") + require.NoError(err) + + file, err := s.Get("\\path\\to\\dummy\\file.txt") + require.NoError(err) + require.Equal(&Dummy{}, file) + + file, err = s.Get("/path/to/dummy/file.txt") + require.NoError(err) + require.Equal(&Dummy{}, file) +} + var _ Filesystem = &DummyFs{} type DummyFs struct {