From 709011c2bd72368482f282c55cf2665fe2476a6d Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 10:36:36 -0700 Subject: [PATCH 01/10] Initial dockerization --- .gitignore | 4 +- Dockerfile | 12 +++++ Dockerfile.dev | 7 +++ Makefile | 84 ++++++++++++++++++++++++++++++++++ VERSION | 1 + api/api.go | 2 +- config/client.go | 29 +++++------- config/server.go | 46 +++++++++++-------- docker-compose.yml | 30 ++++++++++++ gsync/gsync.go | 48 +++++--------------- index/index.go | 6 +-- vars/vars.go | 111 +++++++++++++++++++++++++++++++++++++++++++++ 12 files changed, 302 insertions(+), 78 deletions(-) create mode 100644 Dockerfile create mode 100644 Dockerfile.dev create mode 100644 Makefile create mode 100644 VERSION create mode 100644 docker-compose.yml create mode 100644 vars/vars.go diff --git a/.gitignore b/.gitignore index 2f18747..0dcd3f0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ bin pkg out .idea -gsyncd.iml \ No newline at end of file +gsyncd.iml +build +.env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8b89705 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM alpine:edge +MAINTAINER CausticLab + +ENV FILESYNC_RELEASE v0.0.1 + +ADD https://github.com/CausticLab/filesync/releases/download/${RGON_EXEC_RELEASE}/filesync-linux-amd64.tar.gz /tmp/filesync.tar.gz +RUN tar -zxvf /tmp/filesync -C /usr/local/bin \ + && mv /usr/local/bin/filesync-linux-amd64 /usr/local/bin/filesync \ + && chmod +x /usr/local/bin/filesync \ + && rm /tmp/filesync + +ENTRYPOINT ["/usr/local/bin/filesync"] \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..51d2c84 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,7 @@ +FROM alpine:edge +MAINTAINER CausticLab + +ADD build/filesync-linux-amd64 /usr/local/bin/filesync +RUN chmod +x /usr/local/bin/filesync + +ENTRYPOINT ["/usr/local/bin/filesync"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2fcc703 --- /dev/null +++ b/Makefile @@ -0,0 +1,84 @@ +# These env vars have to be set in the CI +# GITHUB_TOKEN +# DOCKER_HUB_TOKEN + +.PHONY: build deps test release clean push image ci-compile build-dir ci-dist dist-dir ci-release version help + +PROJECT := filesync +PLATFORM := linux +ARCH := amd64 +DOCKER_IMAGE := causticlab/$(PROJECT) + +VERSION := $(shell cat VERSION) +GITSHA := $(shell git rev-parse --short HEAD) + +all: help + +help: + @echo "make build - build binary for the target environment" + @echo "make deps - install build dependencies" + @echo "make vet - run vet & gofmt checks" + @echo "make test - run tests" + @echo "make clean - Duh!" + @echo "make release - tag with version and trigger CI release build" + @echo "make image - build release image" + @echo "make dev-image - build development image" + @echo "make dockerhub - build and push image to Docker Hub" + @echo "make version - show app version" + +build: build-dir + CGO_ENABLED=1 GOOS=$(PLATFORM) GOARCH=$(ARCH) godep go build -ldflags "-X main.Version=$(VERSION) -X main.GitSHA=$(GITSHA)" -o build/$(PROJECT)-$(PLATFORM)-$(ARCH) ./gsync + +deps: + go get github.com/tools/godep + go get github.com/mattn/go-sqlite3 + go get github.com/bitly/go-simplejson + +release: + git tag `cat VERSION` + git push origin master --tags + +clean: + go clean + rm -fr ./build + rm -fr ./dist + +dockerhub: image + @echo "Pushing $(DOCKER_IMAGE):$(VERSION)" + docker push $(DOCKER_IMAGE):$(VERSION) + +image: + docker build -t $(DOCKER_IMAGE):$(VERSION) -f Dockerfile . + +dev-image: + docker build -t $(DOCKER_IMAGE):dev -f Dockerfile.dev . + +version: + @echo $(VERSION) $(GITSHA) + +ci-compile: build-dir + CGO_ENABLED=0 GOOS=$(PLATFORM) GOARCH=$(ARCH) go build -ldflags "-X main.Version=$(VERSION) -X main.GitSHA=$(GITSHA) -w -s" -a -o build/$(PROJECT)-$(PLATFORM)-$(ARCH)/$(PROJECT) ./gsync + +build-dir: + @rm -rf build && mkdir build + +dist-dir: + @rm -rf dist && mkdir dist + +ci-dist: ci-compile dist-dir + $(eval FILES := $(shell ls build)) + @for f in $(FILES); do \ + (cd $(shell pwd)/build/$$f && tar -cvzf ../../dist/$$f.tar.gz *); \ + (cd $(shell pwd)/dist && shasum -a 256 $$f.tar.gz > $$f.sha256); \ + (cd $(shell pwd)/dist && md5sum $$f.tar.gz > $$f.md5); \ + echo $$f; \ + done + @cp -r $(shell pwd)/dist/* $(CIRCLE_ARTIFACTS) + ls $(CIRCLE_ARTIFACTS) + +ci-release: + @previous_tag=$$(git describe --abbrev=0 --tags $(VERSION)^); \ + comparison="$$previous_tag..HEAD"; \ + if [ -z "$$previous_tag" ]; then comparison=""; fi; \ + changelog=$$(git log $$comparison --oneline --no-merges --reverse); \ + github-release $(CIRCLE_PROJECT_USERNAME)/$(CIRCLE_PROJECT_REPONAME) $(VERSION) master "**Changelog**
$$changelog" 'dist/*' diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..95e94cd --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v0.0.1 \ No newline at end of file diff --git a/api/api.go b/api/api.go index 6a4cc15..8d1c188 100644 --- a/api/api.go +++ b/api/api.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/codegangsta/martini" "github.com/codegangsta/martini-contrib/encoder" - "github.com/elgs/filesync/index" + "filesync/index" "io" "net/http" "os" diff --git a/config/client.go b/config/client.go index a45cde7..c207a10 100644 --- a/config/client.go +++ b/config/client.go @@ -4,7 +4,8 @@ import ( "encoding/json" "fmt" simplejson "github.com/bitly/go-simplejson" - "github.com/elgs/filesync/index" + "filesync/index" + vars "filesync/vars" "hash/crc32" "io" "io/ioutil" @@ -16,24 +17,18 @@ import ( var monitorFilePart bool = false -func StartClient(configFile string, done chan bool) { - b, err := ioutil.ReadFile(configFile) - if err != nil { - fmt.Println(configFile, " not found") - go func() { - done <- false - }() - return - } - json, _ := simplejson.NewJson(b) - ip := json.Get("ip").MustString("127.0.0.1") - port := json.Get("port").MustInt(6776) +func StartClient(done chan bool) { + vars := vars.GetConfig(); + fmt.Println("Starting Filesync client") + for k, v := range vars.Monitors { + monitored, _ := v.(string) - monitors := json.Get("monitors").MustMap() + if(!index.Exists(monitored)){ + fmt.Println("Path does not exist: ", monitored) + continue + } - for k, v := range monitors { - monitored, _ := v.(string) - go startWork(ip, port, k, monitored, time.Minute) + go startWork(vars.Ip, vars.Port, k, monitored, time.Minute) } } diff --git a/config/server.go b/config/server.go index aeafa84..f3d8352 100644 --- a/config/server.go +++ b/config/server.go @@ -1,41 +1,47 @@ package config import ( + "os" + "log" "database/sql" - "fmt" - simplejson "github.com/bitly/go-simplejson" - "github.com/elgs/filesync/api" - "github.com/elgs/filesync/index" + "filesync/api" + "filesync/index" + vars "filesync/vars" "github.com/howeyc/fsnotify" _ "github.com/mattn/go-sqlite3" - "io/ioutil" ) -func StartServer(configFile string) { - b, err := ioutil.ReadFile(configFile) - if err != nil { - fmt.Println(configFile, " not found") - return - } - json, _ := simplejson.NewJson(b) - ip := json.Get("ip").MustString("127.0.0.1") - port := json.Get("port").MustInt(6776) - - monitors := json.Get("monitors").MustMap() +func StartServer() { + vars := vars.GetConfig(); - for _, v := range monitors { + for _, v := range vars.Monitors { watcher, _ := fsnotify.NewWatcher() monitored, _ := v.(string) monitored = index.PathSafe(monitored) - db, _ := sql.Open("sqlite3", index.SlashSuffix(monitored)+".sync/index.db") + + if(!index.Exists(monitored)){ + log.Println("Path does not exist, creating: ", monitored) + + if os.MkdirAll(monitored, os.ModePerm) != nil { + log.Fatal("Could not create directory: ", monitored) + continue + } + } + + db, err := sql.Open("sqlite3", index.SlashSuffix(monitored)+".sync/index.db") + if err != nil { + log.Fatal(err) + } defer db.Close() db.Exec("VACUUM;") + index.InitIndex(monitored, db) go index.ProcessEvent(watcher, monitored) index.WatchRecursively(watcher, monitored, monitored) + } - fmt.Println("Serving now...") - api.RunWeb(ip, port, monitors) + log.Println("Serving now...") + api.RunWeb(vars.Ip, vars.Port, vars.Monitors) //watcher.Close() } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0d7eef0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +version: '2' + +services: + fs-server: + image: "causticlab/filesync:dev" + tty: true + stdin_open: true + labels: + io.rancher.container.create_agent: 'true' + io.rancher.container.agent.role: 'environment' + io.rancher.scheduler.affinity:host_label: rgon=primary + environment: + FILESYNC_MODE: 'server' + FILESYNC_PATH: '/share' +# fs-client: +# image: "causticlab/filesync:dev" +# tty: true +# stdin_open: true +# links: +# - fs-server:fs-server +# labels: +# io.rancher.scheduler.global: 'true' +# io.rancher.container.create_agent: 'true' +# io.rancher.container.agent.role: 'environment' +# io.rancher.scheduler.affinity:host_label_ne: rgon=primary +# environment: +# FILESYNC_MODE: 'client' +# FILESYNC_IP: 'fs-server' +# FILESYNC_PORT: '6776' +# FILESYNC_PATH: '/share' \ No newline at end of file diff --git a/gsync/gsync.go b/gsync/gsync.go index f740c4a..4b9fb4c 100644 --- a/gsync/gsync.go +++ b/gsync/gsync.go @@ -1,51 +1,27 @@ package main import ( - "fmt" - simplejson "github.com/bitly/go-simplejson" - "github.com/elgs/filesync/config" - "io/ioutil" - "os" + "log" + vars "filesync/vars" + "filesync/config" "runtime" ) func main() { runtime.GOMAXPROCS(runtime.NumCPU()) - fmt.Println("CPUs: ", runtime.NumCPU()) - input := args() + log.Println("CPUs: ", runtime.NumCPU()) done := make(chan bool) - if len(input) >= 1 { - start(input[0], done) - } + start(done) <-done } -func start(configFile string, done chan bool) { - b, err := ioutil.ReadFile(configFile) - if err != nil { - fmt.Println(configFile, " not found") - go func() { - done <- false - }() - return - } - json, _ := simplejson.NewJson(b) - mode := json.Get("mode").MustString("server") - if mode == "server" { - config.StartServer(configFile) - } else if mode == "client" { - config.StartClient(configFile, done) - } -} +func start(done chan bool) { + vars := vars.GetConfig(); + log.Printf("Fileshare Config:\n%+v\n", vars) -func args() []string { - ret := []string{} - if len(os.Args) <= 1 { - ret = append(ret, "gsync.json") - } else { - for i := 1; i < len(os.Args); i++ { - ret = append(ret, os.Args[i]) - } + if vars.Mode == "server" { + config.StartServer() + } else if vars.Mode == "client" { + config.StartClient(done) } - return ret } diff --git a/index/index.go b/index/index.go index e6b08ef..e95e838 100644 --- a/index/index.go +++ b/index/index.go @@ -329,7 +329,7 @@ func LikeSafe(path string) string { func InitIndex(monitored string, db *sql.DB) error { var ret error = nil - exists := exists(SlashSuffix(monitored) + ".sync/index.db") + exists := Exists(SlashSuffix(monitored) + ".sync/index.db") if !exists { os.MkdirAll(SlashSuffix(monitored)+".sync/", (os.FileMode)(0755)) if monitorFilePart { @@ -362,7 +362,7 @@ func InitIndex(monitored string, db *sql.DB) error { } // exists returns whether the given file or directory exists or not -func exists(path string) bool { +func Exists(path string) bool { _, err := os.Stat(path) if err == nil { return true @@ -407,7 +407,7 @@ func ProcessEvent(watcher *fsnotify.Watcher, monitored string) { ProcessFileDelete(ev.Name, monitored) //fmt.Println("Deleted: " + ev.Name) } else if ev.IsRename() { - if exists(ev.Name) { + if Exists(ev.Name) { if info.IsDir() { WatchRecursively(watcher, ev.Name, monitored) //fmt.Println("Created dir: " + ev.Name) diff --git a/vars/vars.go b/vars/vars.go new file mode 100644 index 0000000..3fcf3d6 --- /dev/null +++ b/vars/vars.go @@ -0,0 +1,111 @@ +package vars + +import ( + "os" + "fmt" + "log" + "strconv" + "io/ioutil" + simplejson "github.com/bitly/go-simplejson" +) + +type ConfigVars struct{ + Mode string + Ip string + Port int + Monitors map[string]interface{} +} + +func Args() []string { + ret := []string{} + if len(os.Args) >= 1 { + for i := 1; i < len(os.Args); i++ { + ret = append(ret, os.Args[i]) + } + } + return ret +} + +func GetConfig() ConfigVars{ + var config ConfigVars + var configFile string + + input := Args() + if len(input) >= 1 { + configFile = input[0] + } + + if(configFile != ""){ + fmt.Println("HERE ", configFile) + b, err := ioutil.ReadFile(configFile) + if err != nil { + fmt.Println(configFile, " not found") + //return + } else { + json, _ := simplejson.NewJson(b) + config.Mode = json.Get("mode").MustString() + config.Ip = json.Get("ip").MustString() + config.Port = json.Get("port").MustInt() + config.Monitors = json.Get("monitors").MustMap() + } + } else { + config.Mode = os.Getenv("FILESYNC_MODE") + config.Ip = os.Getenv("FILESYNC_IP") + config.Port, _ = strconv.Atoi(os.Getenv("FILESYNC_PORT")) + + config.Monitors = make(map[string]interface{}) + config.Monitors["default"] = os.Getenv("FILESYNC_PATH") + } + + if(config.Mode == ""){ + config.Mode = "server" + } + + if(config.Ip == ""){ + config.Ip = "0.0.0.0" + } + + if(config.Port <= 0){ + config.Port = 6776 + } + + if(len(config.Monitors) == 0){ + log.Println("No paths to monitor - defaulting to /share") + + config.Monitors = make(map[string]interface{}) + config.Monitors["default"] = "/share" + + /* + var err error + + if _, err := os.Stat("/share"); os.IsNotExist(err) { + os.Mkdir("/share", os.ModePerm) + } + + if err != nil { + log.Fatal("Could not create default /share directory") + } else { + log.Println("Created /share directory") + } + */ + } + + + return config +} + + + + + + + + + + + + + + + + From cc735b816bf34d9bb9e0ae08ce8e16e2442a035c Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 10:58:56 -0700 Subject: [PATCH 02/10] Attempting linux build process --- .gitignore | 4 +++- Makefile | 5 +++-- gsync/gsync.go => gsync.go | 0 3 files changed, 6 insertions(+), 3 deletions(-) rename gsync/gsync.go => gsync.go (100%) diff --git a/.gitignore b/.gitignore index 0dcd3f0..ecd735d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ out .idea gsyncd.iml build -.env \ No newline at end of file +.env +/Godeps +/vendor \ No newline at end of file diff --git a/Makefile b/Makefile index 2fcc703..b64e302 100644 --- a/Makefile +++ b/Makefile @@ -27,12 +27,13 @@ help: @echo "make version - show app version" build: build-dir - CGO_ENABLED=1 GOOS=$(PLATFORM) GOARCH=$(ARCH) godep go build -ldflags "-X main.Version=$(VERSION) -X main.GitSHA=$(GITSHA)" -o build/$(PROJECT)-$(PLATFORM)-$(ARCH) ./gsync + CGO_ENABLED=1 GOOS=$(PLATFORM) GOARCH=$(ARCH) godep go build -ldflags "-X main.Version=$(VERSION) -X main.GitSHA=$(GITSHA)" -o build/$(PROJECT)-$(PLATFORM)-$(ARCH) -v deps: go get github.com/tools/godep go get github.com/mattn/go-sqlite3 go get github.com/bitly/go-simplejson + godep save release: git tag `cat VERSION` @@ -57,7 +58,7 @@ version: @echo $(VERSION) $(GITSHA) ci-compile: build-dir - CGO_ENABLED=0 GOOS=$(PLATFORM) GOARCH=$(ARCH) go build -ldflags "-X main.Version=$(VERSION) -X main.GitSHA=$(GITSHA) -w -s" -a -o build/$(PROJECT)-$(PLATFORM)-$(ARCH)/$(PROJECT) ./gsync + CGO_ENABLED=0 GOOS=$(PLATFORM) GOARCH=$(ARCH) go build -ldflags "-X main.Version=$(VERSION) -X main.GitSHA=$(GITSHA) -w -s" -a -o build/$(PROJECT)-$(PLATFORM)-$(ARCH)/$(PROJECT) build-dir: @rm -rf build && mkdir build diff --git a/gsync/gsync.go b/gsync.go similarity index 100% rename from gsync/gsync.go rename to gsync.go From 86465913708d64ae6a0b0fbad9eac623d9842a10 Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 12:08:40 -0700 Subject: [PATCH 03/10] Separating vars init method --- Dockerfile | 1 + Dockerfile.dev | 1 + Makefile | 3 +++ gsync.go | 1 + vars/vars.go | 43 ++++++++++++++----------------------------- 5 files changed, 20 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8b89705..f4f8a2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,5 +8,6 @@ RUN tar -zxvf /tmp/filesync -C /usr/local/bin \ && mv /usr/local/bin/filesync-linux-amd64 /usr/local/bin/filesync \ && chmod +x /usr/local/bin/filesync \ && rm /tmp/filesync +RUN mkdir /share ENTRYPOINT ["/usr/local/bin/filesync"] \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile.dev index 51d2c84..6c8c2ad 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -3,5 +3,6 @@ MAINTAINER CausticLab ADD build/filesync-linux-amd64 /usr/local/bin/filesync RUN chmod +x /usr/local/bin/filesync +RUN mkdir /share ENTRYPOINT ["/usr/local/bin/filesync"] \ No newline at end of file diff --git a/Makefile b/Makefile index b64e302..33ae7d1 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,9 @@ deps: go get github.com/tools/godep go get github.com/mattn/go-sqlite3 go get github.com/bitly/go-simplejson + go get github.com/howeyc/fsnotify + go get github.com/codegangsta/martini + go get github.com/codegangsta/martini-contrib/encoder godep save release: diff --git a/gsync.go b/gsync.go index 4b9fb4c..a06a5f4 100644 --- a/gsync.go +++ b/gsync.go @@ -16,6 +16,7 @@ func main() { } func start(done chan bool) { + vars.Init(); vars := vars.GetConfig(); log.Printf("Fileshare Config:\n%+v\n", vars) diff --git a/vars/vars.go b/vars/vars.go index 3fcf3d6..df476ce 100644 --- a/vars/vars.go +++ b/vars/vars.go @@ -15,19 +15,9 @@ type ConfigVars struct{ Port int Monitors map[string]interface{} } +var config ConfigVars -func Args() []string { - ret := []string{} - if len(os.Args) >= 1 { - for i := 1; i < len(os.Args); i++ { - ret = append(ret, os.Args[i]) - } - } - return ret -} - -func GetConfig() ConfigVars{ - var config ConfigVars +func Init() { var configFile string input := Args() @@ -69,32 +59,27 @@ func GetConfig() ConfigVars{ config.Port = 6776 } - if(len(config.Monitors) == 0){ + if (len(config.Monitors) == 1) && (config.Monitors["default"] == ""){ log.Println("No paths to monitor - defaulting to /share") config.Monitors = make(map[string]interface{}) config.Monitors["default"] = "/share" - - /* - var err error - - if _, err := os.Stat("/share"); os.IsNotExist(err) { - os.Mkdir("/share", os.ModePerm) - } - - if err != nil { - log.Fatal("Could not create default /share directory") - } else { - log.Println("Created /share directory") - } - */ } +} - +func GetConfig() ConfigVars{ return config } - +func Args() []string { + ret := []string{} + if len(os.Args) >= 1 { + for i := 1; i < len(os.Args); i++ { + ret = append(ret, os.Args[i]) + } + } + return ret +} From 9e88c1f9324e2673d0b162b5ee30cb84ea9d3d8c Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 13:07:44 -0700 Subject: [PATCH 04/10] Adding directory write check for server --- config/server.go | 16 +++++++++++----- index/index.go | 5 +++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/config/server.go b/config/server.go index f3d8352..f25cf16 100644 --- a/config/server.go +++ b/config/server.go @@ -18,17 +18,23 @@ func StartServer() { watcher, _ := fsnotify.NewWatcher() monitored, _ := v.(string) monitored = index.PathSafe(monitored) + targetPath := index.SlashSuffix(monitored) + dbPath := targetPath + ".sync/index.db" if(!index.Exists(monitored)){ log.Println("Path does not exist, creating: ", monitored) - if os.MkdirAll(monitored, os.ModePerm) != nil { - log.Fatal("Could not create directory: ", monitored) - continue - } + if os.MkdirAll(monitored, os.ModePerm) != nil { + log.Fatal("Could not create directory: ", monitored) + continue + } } - db, err := sql.Open("sqlite3", index.SlashSuffix(monitored)+".sync/index.db") + if(!index.Writable(targetPath)){ + log.Fatal("Path is not writeable: ", targetPath) + } + + db, err := sql.Open("sqlite3", dbPath) if err != nil { log.Fatal(err) } diff --git a/index/index.go b/index/index.go index e95e838..fdefc4e 100644 --- a/index/index.go +++ b/index/index.go @@ -11,6 +11,7 @@ import ( "regexp" "strings" "time" + "golang.org/x/sys/unix" ) type IndexedFile struct { @@ -427,3 +428,7 @@ func ProcessEvent(watcher *fsnotify.Watcher, monitored string) { } } } + +func Writable(path string) bool { + return unix.Access(path, unix.W_OK) == nil +} \ No newline at end of file From 805a9a935503a24087118f6648d36c474deae394 Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 13:49:25 -0700 Subject: [PATCH 05/10] Defining busybox:ubutnu image --- Dockerfile | 2 +- Dockerfile.dev | 2 +- Makefile | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f4f8a2c..a58eb1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:edge +FROM busybox:ubuntu-14.04 MAINTAINER CausticLab ENV FILESYNC_RELEASE v0.0.1 diff --git a/Dockerfile.dev b/Dockerfile.dev index 6c8c2ad..9370a1a 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM alpine:edge +FROM busybox:ubuntu-14.04 MAINTAINER CausticLab ADD build/filesync-linux-amd64 /usr/local/bin/filesync diff --git a/Makefile b/Makefile index 33ae7d1..264072f 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ build: build-dir CGO_ENABLED=1 GOOS=$(PLATFORM) GOARCH=$(ARCH) godep go build -ldflags "-X main.Version=$(VERSION) -X main.GitSHA=$(GITSHA)" -o build/$(PROJECT)-$(PLATFORM)-$(ARCH) -v deps: + go get golang.org/x/sys/unix go get github.com/tools/godep go get github.com/mattn/go-sqlite3 go get github.com/bitly/go-simplejson From 6d67559158959eb163d1e70c82769edb075d6202 Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 13:53:12 -0700 Subject: [PATCH 06/10] Adding client to docker-compose --- docker-compose.yml | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0d7eef0..337ef45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,19 +12,18 @@ services: environment: FILESYNC_MODE: 'server' FILESYNC_PATH: '/share' -# fs-client: -# image: "causticlab/filesync:dev" -# tty: true -# stdin_open: true -# links: -# - fs-server:fs-server -# labels: -# io.rancher.scheduler.global: 'true' -# io.rancher.container.create_agent: 'true' -# io.rancher.container.agent.role: 'environment' -# io.rancher.scheduler.affinity:host_label_ne: rgon=primary -# environment: -# FILESYNC_MODE: 'client' -# FILESYNC_IP: 'fs-server' -# FILESYNC_PORT: '6776' -# FILESYNC_PATH: '/share' \ No newline at end of file + fs-client: + image: "causticlab/filesync:dev" + tty: true + stdin_open: true + links: + - fs-server:fs-server + labels: + io.rancher.scheduler.global: 'true' + io.rancher.container.create_agent: 'true' + io.rancher.container.agent.role: 'environment' + io.rancher.scheduler.affinity:host_label_ne: rgon=primary + environment: + FILESYNC_MODE: 'client' + FILESYNC_IP: 'fs-server' + FILESYNC_PATH: '/share' \ No newline at end of file From 3e90adc354fe04fd0a94b97d1e3e746a3765f3c5 Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 14:54:26 -0700 Subject: [PATCH 07/10] Converting print statements to logs --- api/api.go | 5 +++-- config/client.go | 19 ++++++++++--------- index/index.go | 25 +++++++++++++------------ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/api/api.go b/api/api.go index 8d1c188..8d12d46 100644 --- a/api/api.go +++ b/api/api.go @@ -3,6 +3,7 @@ package api import ( "database/sql" "fmt" + "log" "github.com/codegangsta/martini" "github.com/codegangsta/martini-contrib/encoder" "filesync/index" @@ -37,7 +38,7 @@ func RunWeb(ip string, port int, monitors map[string]interface{}) { route.Get("/dirs", func(enc encoder.Encoder, req *http.Request) (int, []byte) { defer func() { if err := recover(); err != nil { - fmt.Println(err) + log.Println(err) } }() monitored := req.Header.Get("MONITORED") @@ -114,5 +115,5 @@ func RunWeb(ip string, port int, monitors map[string]interface{}) { }) m.Action(route.Handle) - fmt.Println(http.ListenAndServe(fmt.Sprint(ip, ":", port), m)) + log.Println(http.ListenAndServe(fmt.Sprint(ip, ":", port), m)) } diff --git a/config/client.go b/config/client.go index c207a10..773a3f8 100644 --- a/config/client.go +++ b/config/client.go @@ -3,6 +3,7 @@ package config import ( "encoding/json" "fmt" + "log" simplejson "github.com/bitly/go-simplejson" "filesync/index" vars "filesync/vars" @@ -19,12 +20,12 @@ var monitorFilePart bool = false func StartClient(done chan bool) { vars := vars.GetConfig(); - fmt.Println("Starting Filesync client") + log.Println("Starting Filesync client") for k, v := range vars.Monitors { monitored, _ := v.(string) if(!index.Exists(monitored)){ - fmt.Println("Path does not exist: ", monitored) + log.Println("Path does not exist: ", monitored) continue } @@ -47,7 +48,7 @@ func startWork(ip string, port int, key string, monitored string, maxInterval ti if dirStatus == "deleted" { err := os.RemoveAll(dir) if err != nil { - fmt.Println(err) + log.Println(err) } continue } @@ -55,7 +56,7 @@ func startWork(ip string, port int, key string, monitored string, maxInterval ti dirMode, _ := mode.Int64() err := os.MkdirAll(dir, os.FileMode(dirMode)) if err != nil { - fmt.Println(err) + log.Println(err) } } @@ -74,7 +75,7 @@ func startWork(ip string, port int, key string, monitored string, maxInterval ti if fileStatus == "deleted" { err := os.RemoveAll(f) if err != nil { - fmt.Println(err) + log.Println(err) } continue } @@ -145,7 +146,7 @@ func startWork(ip string, port int, key string, monitored string, maxInterval ti func downloadFromServer(ip string, port int, key string, filePath string, start int64, length int64, file *os.File) int64 { defer func() { if err := recover(); err != nil { - fmt.Println(err) + log.Println(err) } }() client := &http.Client{} @@ -162,7 +163,7 @@ func downloadFromServer(ip string, port int, key string, filePath string, start func filePartsFromServer(ip string, port int, key string, filePath string) []interface{} { defer func() { if err := recover(); err != nil { - fmt.Println(err) + log.Println(err) } }() client := &http.Client{} @@ -180,7 +181,7 @@ func filePartsFromServer(ip string, port int, key string, filePath string) []int func filesFromServer(ip string, port int, key string, filePath string, lastIndexed int64) []interface{} { defer func() { if err := recover(); err != nil { - fmt.Println(err) + log.Println(err) } }() client := &http.Client{} @@ -198,7 +199,7 @@ func filesFromServer(ip string, port int, key string, filePath string, lastIndex func dirsFromServer(ip string, port int, key string, lastIndexed int64) []interface{} { defer func() { if err := recover(); err != nil { - fmt.Println(err) + log.Println(err) } }() client := &http.Client{} diff --git a/index/index.go b/index/index.go index fdefc4e..33231a6 100644 --- a/index/index.go +++ b/index/index.go @@ -2,6 +2,7 @@ package index import ( "database/sql" + "log" "fmt" "github.com/howeyc/fsnotify" "hash/crc32" @@ -40,12 +41,12 @@ var monitorFilePart bool = false func ProcessFileDelete(thePath string, monitored string) { if ignore(thePath, monitored) { - fmt.Println("Ignored: ", thePath) + //log.Println("Ignored: ", thePath) return } defer func() { if err := recover(); err != nil { - fmt.Println(err) + log.Println(err) } }() thePath = PathSafe(thePath) @@ -83,16 +84,16 @@ func ProcessFileDelete(thePath string, monitored string) { func ProcessDirChange(thePath string, info os.FileInfo, monitored string) { if ignore(thePath, monitored) { - fmt.Println("Ignored: ", thePath) + //log.Println("Ignored: ", thePath) return } defer func() { if err := recover(); err != nil { - fmt.Println(err) + log.Println(err) } }() if info == nil { - fmt.Println("Dir no longer exists: " + thePath) + log.Println("Dir no longer exists: " + thePath) return } thePath = PathSafe(thePath) @@ -109,16 +110,16 @@ func ProcessDirChange(thePath string, info os.FileInfo, monitored string) { func ProcessFileChange(thePath string, info os.FileInfo, monitored string) { if ignore(thePath, monitored) { - fmt.Println("Ignored: ", thePath) + //log.Println("Ignored: ", thePath) return } defer func() { if err := recover(); err != nil { - fmt.Println(err) + log.Println(err) } }() if info == nil { - fmt.Println("File no longer exists: " + thePath) + log.Println("File no longer exists: " + thePath) return } thePath = PathSafe(thePath) @@ -239,7 +240,7 @@ func ProcessFileChange(thePath string, info os.FileInfo, monitored string) { func WatchRecursively(watcher *fsnotify.Watcher, root string, monitored string) error { if ignore(root, monitored) { - fmt.Println("Ignored: ", root) + //log.Println("Ignored: ", root) return nil } @@ -266,7 +267,7 @@ func WatchRecursively(watcher *fsnotify.Watcher, root string, monitored string) defer db.Close() //fmt.Println(path) if ignore(path, monitored) { - fmt.Println("Ignored: ", path) + //log.Println("Ignored: ", path) return nil } var thePath string @@ -303,7 +304,7 @@ func WatchRecursively(watcher *fsnotify.Watcher, root string, monitored string) // remove zombies for k, v := range mapFiles { if k != "/" && v.Status == "ready" { - fmt.Println("Zombie removed: ", v.FilePath) + log.Println("Zombie removed: ", v.FilePath) ProcessFileDelete(monitored+k, monitored) } } @@ -421,7 +422,7 @@ func ProcessEvent(watcher *fsnotify.Watcher, monitored string) { } } case err := <-watcher.Error: - fmt.Println("error:", err) + log.Println("error: ", err) case <-time.After(time.Minute): //fmt.Println("I'm idle, so I decided to do a patrol") go WatchRecursively(watcher, monitored, monitored) From 0a062e4b7a0905e5a058621e3c3940fb97eeccbe Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 15:23:29 -0700 Subject: [PATCH 08/10] Preparing for release --- Makefile | 2 +- README.md | 94 +++++++++++++++++++++++++--------------------- docker-compose.yml | 17 ++++++++- 3 files changed, 69 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index 264072f..835790c 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ version: @echo $(VERSION) $(GITSHA) ci-compile: build-dir - CGO_ENABLED=0 GOOS=$(PLATFORM) GOARCH=$(ARCH) go build -ldflags "-X main.Version=$(VERSION) -X main.GitSHA=$(GITSHA) -w -s" -a -o build/$(PROJECT)-$(PLATFORM)-$(ARCH)/$(PROJECT) + CGO_ENABLED=1 GOOS=$(PLATFORM) GOARCH=$(ARCH) go build -ldflags "-X main.Version=$(VERSION) -X main.GitSHA=$(GITSHA) -w -s" -a -o build/$(PROJECT)-$(PLATFORM)-$(ARCH)/$(PROJECT) build-dir: @rm -rf build && mkdir build diff --git a/README.md b/README.md index 34d3345..a5e7812 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,58 @@ -Filesync -=== +# Filesync + Filesync is a utility written in Golang which helps you to keep the files on the client up to date with the files on the server. Only the changed parts of files on the server are downloaded. Therefore it's great to synchronize your huge, and frequently changing files. -Installation -=== -`go get github.com/elgs/filesync/gsync` - -Server -=== -Run ---- -`gsync server.json` -Configuration ---- -server.json -```json -{ - "mode": "server", - "ip": "0.0.0.0", - "port": 6776, - "monitors": { - "home_elgs_desktop_a": "/home/elgs/Desktop/a", - "home_elgs_desktop_b": "/home/elgs/Desktop/b" - } -} +Forked from github.com/elgs/filesync/gsync + +## Requirements + +Needs access to `glibc` to compile properly, and so `busybox:ubuntu-14.04` is the selected base image. + +## Local Usage + +Install dependencies: + +```sh +make deps ``` +Run locally with config files (modify paths before running): + +```sh +# Server +go run gsync.go gsync/server.json -Client -=== -Run ---- -`gsync client.json` -Configuration ---- -client.json -```json -{ - "mode": "client", - "ip": "127.0.0.1", - "port": 6776, - "monitors": { - "home_elgs_desktop_a": "/home/elgs/Desktop/c", - "home_elgs_desktop_b": "/home/elgs/Desktop/d" - } -} +# Client +go run gsync.go gsync/client.json ``` + +Alternatively, set environment variables (examples in [/.env](/.env)): + +```sh +# Server +export FILESYNC_MODE=server +export FILESYNC_PORT=6776 +export FILESYNC_IP=0.0.0.0 +export FILESYNC_PATH=/tmp/share + +go run gsync.go +``` + +Build package (requires `glibc`, won't work on OSX): + +```sh +make build +``` + +Build Docker image: + +```sh +make dev-image +make image +``` + +Run Docker-Compose cluster: + +```sh +docker-compose up -d +``` \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 337ef45..6b8a083 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,22 @@ services: environment: FILESYNC_MODE: 'server' FILESYNC_PATH: '/share' - fs-client: + fs-client1: + image: "causticlab/filesync:dev" + tty: true + stdin_open: true + links: + - fs-server:fs-server + labels: + io.rancher.scheduler.global: 'true' + io.rancher.container.create_agent: 'true' + io.rancher.container.agent.role: 'environment' + io.rancher.scheduler.affinity:host_label_ne: rgon=primary + environment: + FILESYNC_MODE: 'client' + FILESYNC_IP: 'fs-server' + FILESYNC_PATH: '/share' + fs-client2: image: "causticlab/filesync:dev" tty: true stdin_open: true From faae59f31f842e1e140f57305dd60f6df0fa1dc6 Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 15:52:47 -0700 Subject: [PATCH 09/10] README update, demo --- .gitignore | 3 ++- README.md | 44 +++++++++++++++++++++++++++++++++++++++++++- docker-compose.yml | 8 +++++++- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index ecd735d..079b514 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ gsyncd.iml build .env /Godeps -/vendor \ No newline at end of file +/vendor +/demo \ No newline at end of file diff --git a/README.md b/README.md index a5e7812..ec2a1e1 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,50 @@ make dev-image make image ``` +## Demo + +The [docker-compose.yml](/docker-compose.yml) details a setup with a server and 2 clients. It will create a `./demo/` directory with 3 subfolders: `server`, `client1`, and `client2`. Each subfolder is volumed as `/share/` inside each container. + +As files in `./demo/server` are modified, they will be altered in `./demo/client1` and `./demo/client2`. + Run Docker-Compose cluster: ```sh docker-compose up -d -``` \ No newline at end of file +``` + +Check directories: + +```sh +ls -al ./demo/server +ls -al ./demo/client1 +ls -al ./demo/client2 +``` + +Add a file to server volume: + +```sh +echo "testing 123" > ./demo/server/test1 +``` + +Check directories again: + +```sh +ls -al ./demo/server +ls -al ./demo/client1 +ls -al ./demo/client2 +``` + +At this point, the `./demo/client*` directories should contain a `test1` file. + +## Notes + +### Docker Environment + +While the server configuration can be set to an IP of `0.0.0.0` (accepts traffic from anywhere), the clients need a specific address to connect to. If running locally, the clients can be set to connect to `127.0.0.1` - but this will not work in a Dockerized environment. + +The Docker-Compose.yml file contains `links: [fs-server:fs-server]` which enables the clients to contact the server container at `http://fs-server`. This is supported in a Rancher environment as well. + +### Rancher Environment + +- todo \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 6b8a083..bd20dea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,8 @@ services: environment: FILESYNC_MODE: 'server' FILESYNC_PATH: '/share' + volumes: + - ./demo/server:/share fs-client1: image: "causticlab/filesync:dev" tty: true @@ -27,6 +29,8 @@ services: FILESYNC_MODE: 'client' FILESYNC_IP: 'fs-server' FILESYNC_PATH: '/share' + volumes: + - ./demo/client1:/share fs-client2: image: "causticlab/filesync:dev" tty: true @@ -41,4 +45,6 @@ services: environment: FILESYNC_MODE: 'client' FILESYNC_IP: 'fs-server' - FILESYNC_PATH: '/share' \ No newline at end of file + FILESYNC_PATH: '/share' + volumes: + - ./demo/client2:/share \ No newline at end of file From ba1cdd569c94e8f1e57c83b40cbc8a8e5024f1dc Mon Sep 17 00:00:00 2001 From: Eric McNiece Date: Sun, 19 Mar 2017 16:00:15 -0700 Subject: [PATCH 10/10] Adding Rancher notes --- README.md | 17 ++++++++++++++++- docker-compose.yml | 6 ------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ec2a1e1..6078140 100644 --- a/README.md +++ b/README.md @@ -97,4 +97,19 @@ The Docker-Compose.yml file contains `links: [fs-server:fs-server]` which enable ### Rancher Environment -- todo \ No newline at end of file +Filesync can be used as a way to share files between hosts. Container deployment can be controlled by assigning labels and using the Rancher scheduling system. + +By labelling the primary host with `filesync=server`, these labels can be used to add a Filesync server and multiple clients: + +```yml +# Server + labels: + io.rancher.scheduler.affinity:host_label: filesync=server + +# Client + labels: + io.rancher.scheduler.global: 'true' + io.rancher.scheduler.affinity:host_label_ne: filesync=server +``` + +The host labelled with `filesync=server` will receive a Filesync server container, and all other hosts (`io.rancher.scheduler.global: 'true'`) not labelled with this (`host_label_ne: filesync=server`) will receive a client container. diff --git a/docker-compose.yml b/docker-compose.yml index bd20dea..77f2f27 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,6 @@ services: tty: true stdin_open: true labels: - io.rancher.container.create_agent: 'true' - io.rancher.container.agent.role: 'environment' io.rancher.scheduler.affinity:host_label: rgon=primary environment: FILESYNC_MODE: 'server' @@ -22,8 +20,6 @@ services: - fs-server:fs-server labels: io.rancher.scheduler.global: 'true' - io.rancher.container.create_agent: 'true' - io.rancher.container.agent.role: 'environment' io.rancher.scheduler.affinity:host_label_ne: rgon=primary environment: FILESYNC_MODE: 'client' @@ -39,8 +35,6 @@ services: - fs-server:fs-server labels: io.rancher.scheduler.global: 'true' - io.rancher.container.create_agent: 'true' - io.rancher.container.agent.role: 'environment' io.rancher.scheduler.affinity:host_label_ne: rgon=primary environment: FILESYNC_MODE: 'client'