Build on several platforms (#13)

- Added a GitHub workflow to compile distribyted on linux, macOS and windows.
- Fixed some problems on storage when running on windows.
- Added more documentation about how to use on windows.
- Now on every build, artifacts are generated and can be downloaded for testing before a release.
- Only cross-compiling arm-7 for now.

Fixes #6 

Signed-off-by: Antonio Navarro Perez <antnavper@gmail.com>

.github/workflows/build.yaml

@@ -0,0 +1,128 @@
+---
+name: build
+
+# Trigger the workflow on push or pull request
+on:
+  push:
+    branches:
+      - '*'
+    tags:
+      - '*'
+  pull_request:
+
+jobs:
+  build:
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: ['linux', 'mac', 'windows_amd64']
+
+        include:
+          - job_name: linux
+            os: ubuntu-latest
+            go: '1.15.x'
+
+          - job_name: mac
+            os: macOS-latest
+            go: '1.15.x'
+
+          - job_name: windows_amd64
+            os: windows-latest
+            go: '1.15.x'
+            
+    name: ${{ matrix.job_name }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          stable: 'false'
+          go-version: ${{ matrix.go }}
+
+      - name: Install Libraries on Linux
+        shell: bash
+        run: |
+          sudo modprobe fuse
+          sudo chmod 666 /dev/fuse
+          sudo chown root:$USER /etc/fuse.conf
+          sudo apt-get install fuse libfuse-dev rpm pkg-config
+        if: matrix.os == 'ubuntu-latest'
+
+      - name: Install Libraries on macOS
+        shell: bash
+        run: |
+          brew untap local/homebrew-openssl  # workaround for https://github.com/actions/virtual-environments/issues/1811
+          brew untap local/homebrew-python2  # workaround for https://github.com/actions/virtual-environments/issues/1811
+          brew update
+          brew cask install osxfuse
+        if: matrix.os == 'macOS-latest'
+
+      - name: Install Libraries on Windows
+        shell: powershell
+        run: |
+          $ProgressPreference = 'SilentlyContinue'
+          choco install -y winfsp zip
+          echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          if ($env:GOARCH -eq "386") {
+            choco install -y mingw --forcex86 --force
+            echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          }
+          # Copy mingw32-make.exe to make.exe so the same command line
+          # can be used on Windows as on macOS and Linux
+          $path = (get-command mingw32-make.exe).Path
+          Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe')
+        if: matrix.os == 'windows-latest'
+
+      - name: Print Go version and environment
+        shell: bash
+        run: |
+          printf "Using go at: $(which go)\n"
+          printf "Go version: $(go version)\n"
+          printf "\n\nGo environment:\n\n"
+          go env
+          printf "\n\nSystem environment:\n\n"
+          env
+
+      - name: Go module cache
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Build
+        shell: bash
+        run: |
+          make build
+      - name: Cross-compile
+        shell: bash
+        run: |
+          make cross-compile
+        if: matrix.os == 'ubuntu-latest'
+
+      - name: Run tests
+        shell: bash
+        run: |
+          make test
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          if-no-files-found: error
+          name: build-${{ matrix.job_name }}
+          path: bin/*
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          files: bin/*

.github/workflows/release.yml

@@ -1,32 +0,0 @@
-name: Release
-
-on:
-  push:
-    tags:
-      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
-
-jobs:
-  release:
-    name: Upload Release Assets
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Install fuse
-        run: sudo apt-get install libfuse-dev gcc
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.15.x
-      - name: Compile project 
-        run: make compile
-      - name: Upload Release
-        uses: softprops/action-gh-release@v0.1.5
-        with:
-          draft: true
-          prerelease: false
-          fail_on_unmatched_files: true
-          files: |
-            bin/*
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yaml

@@ -1,27 +0,0 @@
-on: [push, pull_request]
-name: Test
-jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.15.x]
-        platform: [ubuntu-latest]
-        # platform: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.platform }}
-    steps:
-    - name: Install fuse
-      run: sudo apt-get install libfuse-dev gcc
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ matrix.go-version }}
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: Test
-      run: make test
-    - name: Code Coverage
-      uses: codecov/codecov-action@v1
-      with:
-        fail_ci_if_error: true
-    - name: Build
-      run: make build

Makefile

@@ -4,6 +4,10 @@ VERSION := $(shell git describe --tags)
 BUILD := $(shell git rev-parse --short HEAD)
 PROJECTNAME := $(shell basename "$(PWD)")
 
+go-cross-compile: GOPATH=~/go
+go-cross-compile: ORGPATH=$(GOPATH)/src/github.com/distribyted
+go-cross-compile: REPOPATH=$(ORGPATH)/distribyted
+
 # Use linker flags to provide version/build settings
 LDFLAGS=-X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -linkmode external
 
@@ -21,33 +25,25 @@ build: go-generate go-build
 test:
 	go test -v --race -coverprofile=coverage.out ./...
 
-compile: go-generate go-compile
+## cross-compile: compile for other platforms using xgo.
+cross-compile: go-generate go-cross-compile
 
 go-build:
 	@echo "  >  Building binary..."
-	go build -o bin/distribyted -tags "release" cmd/distribyted/main.go
+	go build -o bin/distribyted-$(VERSION)-`go env GOOS`-`go env GOARCH``go env GOEXE` -tags "release" cmd/distribyted/main.go
 
 go-generate:
 	@echo "  >  Generating code files..."
 	go generate ./...
 
-go-compile:
-	GOPATH=~/go
-	ORGPATH=$(GOPATH)/src/github.com/distribyted
-	REPOPATH=$(ORGPATH)/distribyted
-
+go-cross-compile:
 	@echo "  >  Compiling for several platforms..."
-	go install src.techknowlogick.com/xgo
+	GO111MODULE=off go get -u src.techknowlogick.com/xgo
 	docker build ./build_tools/ -t distribyted/xgo-cgofuse
 	mkdir -p $(ORGPATH)
 	ln -sfrnT . $(REPOPATH)
 
-	@echo "  >  Compiling for windows..."
-	GOPATH=$(GOPATH) xgo -out bin/distribyted-$(VERSION) -image=distribyted/xgo-cgofuse -ldflags='-extldflags "-static" $(LDFLAGS)' -tags="release" -targets=windows/amd64 $(REPOPATH)/cmd/distribyted/
-	@echo "  >  Compiling for linux..."
-	GOPATH=$(GOPATH) xgo -out bin/distribyted-$(VERSION) -image=distribyted/xgo-cgofuse -ldflags='$(LDFLAGS)' -tags="release" -targets=linux/arm-7,linux/amd64 $(REPOPATH)/cmd/distribyted/
-#	@echo "  >  Compiling for darwin..."
-#	GOPATH=$(GOPATH) xgo -out bin/distribyted-$(VERSION) -image=distribyted/xgo-cgofuse -ldflags='$(LDFLAGS)' -tags="release" -targets=darwin/* $(REPOPATH)/cmd/distribyted/
+	GOPATH=$(GOPATH) xgo -out bin/distribyted-$(VERSION) -image=distribyted/xgo-cgofuse -ldflags='$(LDFLAGS)' -tags="release" -targets=linux/arm-7 $(REPOPATH)/cmd/distribyted/
 
 .PHONY: help
 all: help

README.md

@@ -35,6 +35,7 @@
     - [To Be Supported](#to-be-supported)
     - [Not Supported](#not-supported)
 - [Getting Started](#getting-started)
+  - [Prerequisites on windows](#prerequisites-on-windows)
 - [Usage](#usage)
   - [Configuration File](#configuration-file)
     - [root](#root)
@@ -83,6 +84,10 @@ Use the [example config file][example-config] and modify it as needed.
 
 Run the program: `distribyted-binary path/to/config/file.yaml`
 
+### Prerequisites on windows
+
+Download and install [WinFsp](http://www.secfs.net/winfsp/).
+
 ## Usage
 
 After executing and load all torrent or magnet files, a web interface will be available with information about the mounted routes and torrent files like download/upload speed, leechers, seeders...
@@ -101,7 +106,7 @@ After executing and load all torrent or magnet files, a web interface will be av
 
 |Config key|Description|
 |-|-|
-|path|Path where a new fuse mount will be initialized.|
+|path|Path where a new fuse mount will be initialized. On Windows you can use a drive letter (`X:` per example) or a folder path that **does not exist**.|
 |torrents|List of `magnetUri`s or/and `torrentPath`s to be loaded on this fuse mount.|
 
 ## Contributing

build_tools/Dockerfile

@@ -7,17 +7,6 @@ RUN \
     apt-get update && \
     apt-get install -y --no-install-recommends p7zip-full
 
-# install OSXFUSE
-# (TODO) not compatible anymore. Find a new way to install osxfuse
-# RUN wget -q -O osxfuse.dmg https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.8.3/osxfuse-3.8.3.dmg
-# RUN 7z e osxfuse.dmg 0.hfs
-# RUN 7z e 0.hfs "FUSE for macOS/Extras/FUSE for macOS 3.8.3.pkg"
-# RUN 7z e "FUSE for macOS 3.8.3.pkg" Core.pkg/Payload
-# RUN 7z e Payload
-# RUN 7z x Payload~ -o/tmp
-# RUN cp -R /tmp/usr/local/include/osxfuse /usr/local/include
-# RUN cp /tmp/usr/local/lib/libosxfuse_i64.2.dylib /usr/local/lib/libosxfuse.dylib
-
 # install LIBFUSE
 RUN \
     apt-get update && \
@@ -26,11 +15,5 @@ RUN \
     apt-get download libfuse-dev:i386 && \
     dpkg -x libfuse-dev*i386*.deb /
 
-# install WinFsp-FUSE
-RUN \
-    wget -q -O winfsp.zip \
-        https://github.com/billziss-gh/winfsp/archive/release/1.4.zip && \
-    7z e winfsp.zip 'winfsp-release-1.4/inc/fuse/*' -o/usr/local/include/winfsp
-
 ENV \
     OSXCROSS_NO_INCLUDE_PATH_WARNINGS 1

fs/storage.go

@@ -3,10 +3,11 @@ package fs
 import (
 	"os"
 	"path"
-	"path/filepath"
 	"strings"
 )
 
+const separator = "/"
+
 type FsFactory func(f File) (Filesystem, error)
 
 var SupportedFactories = map[string]FsFactory{
@@ -76,8 +77,8 @@ func (s *storage) Add(f File, p string) error {
 	return nil
 }
 
-func (s *storage) createParent(path string, f File) error {
-	base, filename := filepath.Split(path)
+func (s *storage) createParent(p string, f File) error {
+	base, filename := path.Split(p)
 	base = clean(base)
 
 	if err := s.Add(&Dir{}, base); err != nil {
@@ -128,7 +129,7 @@ func (s *storage) Get(path string) (File, error) {
 func (s *storage) getFileFromFs(p string) (File, error) {
 	for fsp, fs := range s.filesystems {
 		if strings.HasPrefix(p, fsp) {
-			return fs.Open(string(os.PathSeparator) + strings.TrimPrefix(p, fsp))
+			return fs.Open(separator + strings.TrimPrefix(p, fsp))
 		}
 	}
 
@@ -146,6 +147,6 @@ func (s *storage) getDirFromFs(p string) (map[string]File, error) {
 	return nil, os.ErrNotExist
 }
 
-func clean(path string) string {
-	return filepath.Clean(string(os.PathSeparator) + filepath.FromSlash(path))
+func clean(p string) string {
+	return path.Clean(separator + strings.ReplaceAll(p, "\\", "/"))
 }

fs/storage_test.go

@@ -83,6 +83,25 @@ func TestStorage(t *testing.T) {
 	require.Equal(&Dummy{}, file)
 }
 
+func TestStorageWindowsPath(t *testing.T) {
+	t.Parallel()
+
+	require := require.New(t)
+
+	s := newStorage(dummyFactories)
+
+	err := s.Add(&Dummy{}, "\\path\\to\\dummy\\file.txt")
+	require.NoError(err)
+
+	file, err := s.Get("\\path\\to\\dummy\\file.txt")
+	require.NoError(err)
+	require.Equal(&Dummy{}, file)
+
+	file, err = s.Get("/path/to/dummy/file.txt")
+	require.NoError(err)
+	require.Equal(&Dummy{}, file)
+}
+
 var _ Filesystem = &DummyFs{}
 
 type DummyFs struct {