initial commit

README.md

@@ -0,0 +1,74 @@
+# heptapod
+
+This is a command line application to manage and fine-tune
+[Time Machine](https://support.apple.com/en-us/HT201250) exclude paths.
+
+## This repository is a WIP! The advertised functionality is nonexistent yet :(
+
+### Repo state
+done:
+ - architecture the protocol/configs
+ - most of asimovs features are ported
+ - example rules added
+
+todos:
+ - command line interface
+ - option to dryrun, show inner states, write to file
+ - handle global deps (m2, ivy, nvm, npm) 
+ - support tmignore functionality
+ - support tmignore like funcionality with dockerignore
+ - regexp pattern
+ - brew package
+ - hook to rerun periodically/before tmbackups
+ - port asimov's issues (spotify, spotlight)
+ - docker support (at least tell if docker vm is persisted or not)
+   - this is kinda easy with tmutil.GetExcludeList() 
+ - android vms?
+   - this should be also easy
+ - preemptive search that tells you how many files with which size will be excluded/included
+   - nice to have, we can tell the sizes, but counting files need to actually count the files which could be slow AH 
+ - speedtest it
+ - purge option
+   - we should write down what paths excluded by us, and include them back
+ - modify the backup intervals and frequencies
+   - not sure it can be done with tmutil, but there are applications for this 
+ - probably adding `.` and `..` will break the execution shortcuts and need to fix them
+ - wildcards like `*.sh` not working right now, do we want to make them work?
+
+### Notes for migrating to a new machine
+`xcode-select --install` may be needed after a migration
+
+### Rules
+Every rule has a searchPaths, ignorePaths.
+ - `name` for categorization
+ - `enabled` for easier enable/disable
+ - `searchPaths` are the root of the rule search like `~`
+ - `ignorePaths` are subpaths that we want to ignore to make the run quicker
+   - like we want to parse dirs under `~`, but not `~/Downloads`
+ - `type` is the ruletype
+   - `file-trigger`
+   - `regexp`
+   - `ignore-files`
+ - `settings` other type setting see below
+
+#### Ignore (not yet implemented)
+Parses the `.gitignore` or `.dockerignore` files, and excludes its contents.
+ - `forceAddPaths` are files that we add even if they would be otherwise excluded
+   - like `.gitignore` ignores `.env` files but we forcefully want to add them back 
+ - `fileName`
+   - `.gitignore` or `.dockerignore`
+
+#### Regexp (not yet implemented)
+Ignores all files/folders with the given regexp. This can be slow!
+ - `regexp`
+
+#### File trigger
+Ignores files/dirs based on other files existence, made for easy language dep ignores.
+ - `fileTrigger` like `package.json` or `.git`
+ - `excludePaths` like `node-modules` or `.`
+
+### Credits
+ - [asimov](https://github.com/stevegrunwell/asimov)
+ - [tmignore](https://github.com/samuelmeuli/tmignore)
+ - [various stack exchange responses](https://superuser.com/questions/1161038/exclude-folders-by-regex-from-time-machine-backup)
+ - [tmutil](https://ss64.com/osx/tmutil.html)

go.mod

@@ -0,0 +1,5 @@
+module github.com/tg44/heptapod
+
+go 1.13
+
+require gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -0,0 +1,4 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"fmt"
+	"github.com/tg44/heptapod/pkg/tmutil"
+)
+
+func main() {
+	//res := pkg.GetExcludedPaths([]string{"test-rules/scala.yaml", "test-rules/node.yaml"}, 4, 2048)
+	//tmutil.AddPathsToTM(res, 2048)
+	//fmt.Println(len(res))
+
+	fmt.Println(tmutil.GetExcludeList())
+}

pkg/parser/file-trigger.go

@@ -0,0 +1,50 @@
+package parser
+
+import (
+	"errors"
+	"fmt"
+	"github.com/tg44/heptapod/pkg/utils"
+	"github.com/tg44/heptapod/pkg/walker"
+	"os"
+	"path/filepath"
+)
+
+type FileTriggerSettings struct {
+	FileTrigger       string
+	ExcludePaths []string
+}
+
+const FileTriggerType = ""
+
+func fileTriggerSettingsParse(i map[string]interface{}) (*FileTriggerSettings, error) {
+	fileTrigger, found1 := i["fileTrigger"].(string)
+	excludePathsI, found2 := i["excludePaths"].([]interface{})
+	excludePaths := make([]string, len(excludePathsI))
+	for i, v := range excludePathsI {
+		excludePaths[i] = fmt.Sprint(v)
+	}
+	if found1 && found2 {
+		return &FileTriggerSettings{fileTrigger, excludePaths}, nil
+	}
+	return nil, errors.New("The given input can't be parsed to FileTriggerSettings!")
+}
+
+func fileTriggerWalker(rule Rule, settings FileTriggerSettings) walker.Walker {
+	fixIgnorePaths := []string{}
+	for _, p := range rule.IgnorePaths {
+		path, err := utils.FixupPathsToHandleHome(p)
+		if err == nil {
+			fixIgnorePaths = append(fixIgnorePaths, path)
+		}
+	}
+	return func(path string, subfiles []os.FileInfo) ([]string, []string, []string) {
+		if utils.ContainsFIA(subfiles, settings.FileTrigger) {
+			var ret []string
+			for _, f := range settings.ExcludePaths {
+				ret = append(ret, filepath.Join(path, f))
+			}
+			return ret, []string{}, fixIgnorePaths
+		}
+		return []string{}, []string{}, fixIgnorePaths
+	}
+}

pkg/parser/file-trigger_test.go

@@ -0,0 +1,25 @@
+package parser
+
+import "testing"
+
+func TestFileTriggerSettingsParse(t *testing.T) {
+	rule, err := ruleParse("../test-rules/git.yaml")
+	if err != nil {
+		t.Errorf("Rule parser can't parse testRule!")
+	}
+	settings, err2 := fileTriggerSettingsParse(rule.RuleSettings)
+	//t.Log(rule.RuleSettings)
+	//t.Log(settings)
+	if err2 != nil {
+		t.Errorf("FileSettings parser can't parse testRule!")
+	}
+	if settings == nil {
+		t.Errorf("FileSettings returns nil without error!")
+	}
+	if len(settings.ExcludePaths) != 1 || settings.ExcludePaths[0] != "." {
+		t.Errorf("FileSettings parser ExcludePaths parser error!")
+	}
+	if settings.FileTrigger != ".git" {
+		t.Errorf("FileSettings parser FileTrigger parser error!")
+	}
+}

pkg/parser/parser.go

@@ -0,0 +1,62 @@
+package parser
+
+import (
+	"github.com/tg44/heptapod/pkg/walker"
+	"log"
+)
+
+func Parse(ruleFiles []string) []walker.WalkJob {
+	jobs := []walker.WalkJob{}
+	for _, f := range ruleFiles {
+		jobs = append(jobs, parse(f)...)
+	}
+	jobs = mergeJobs(jobs)
+	return jobs
+}
+
+func parse(ruleFile string) []walker.WalkJob {
+	rule, err := ruleParse(ruleFile)
+	if err != nil {
+		log.Println(err)
+		return []walker.WalkJob{}
+	}
+	if !rule.Enabled {
+		return []walker.WalkJob{}
+	}
+	if rule.RuleType == "file-trigger" {
+		settings, err2 := fileTriggerSettingsParse(rule.RuleSettings)
+		if err2 != nil {
+			log.Println(err2)
+			return []walker.WalkJob{}
+		}
+		tasks := []walker.WalkJob{}
+		walkerFun := fileTriggerWalker(*rule, *settings)
+		for _, p := range rule.SearchPaths {
+			tasks = append(tasks, walker.WalkJob{p, []walker.Walker{walkerFun}, []string{}})
+		}
+		return tasks
+	}
+	return []walker.WalkJob{}
+}
+
+func mergeJobs(works []walker.WalkJob) []walker.WalkJob {
+	paths := map[string]bool{}
+	for _, w := range works {
+		paths[w.Rootpath] = true
+	}
+	pathArr := []string{}
+	for k, _ := range paths {
+		pathArr = append(pathArr, k)
+	}
+	newJobs := []walker.WalkJob{}
+	for _, p := range pathArr {
+		walkers := []walker.Walker{}
+		for _, w := range works {
+			if w.Rootpath == p {
+				walkers = append(walkers, w.Walkers...)
+			}
+		}
+		newJobs = append(newJobs, walker.WalkJob{p, walkers, pathArr})
+	}
+	return newJobs
+}

pkg/parser/rule.go

@@ -0,0 +1,33 @@
+package parser
+
+import (
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+)
+
+type Rule struct {
+	Name       string `yaml:"name"`
+	Enabled bool `yaml:"enabled"`
+	SearchPaths []string `yaml:"searchPaths"`
+	IgnorePaths []string `yaml:"ignorePaths"`
+	RuleType string `yaml:"ruleType"`
+	RuleSettings map[string]interface{} `yaml:"ruleSettings"`
+}
+
+func ruleParse(fileName string) (*Rule, error) {
+	yfile, err := ioutil.ReadFile(fileName)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var rule Rule
+
+	err2 := yaml.Unmarshal(yfile, &rule)
+
+	if err2 != nil {
+		return nil, err
+	}
+
+	return &rule, nil
+}

pkg/parser/rule_test.go

@@ -0,0 +1,31 @@
+package parser
+
+import (
+	"testing"
+)
+
+func TestRuleParse(t *testing.T) {
+	res, err := ruleParse("../test-rules/git.yaml")
+	//t.Log(res)
+	if err != nil {
+		t.Errorf("Rule parser can't parse testRule!")
+	}
+	if res == nil {
+		t.Errorf("Rule parser returned with a nil but not with an error!")
+	}
+	if res.Name != "git" {
+		t.Errorf("Rule parser name parse error!")
+	}
+	if res.Enabled != true {
+		t.Errorf("Rule parser enabled parse error!")
+	}
+	if len(res.SearchPaths) != 1 || res.SearchPaths[0] != "~" {
+		t.Errorf("Rule parser SearchPaths parse error!")
+	}
+	if len(res.IgnorePaths) != 0 {
+		t.Errorf("Rule parser IgnorePaths parse error!")
+	}
+	if res.RuleType != "file-trigger" {
+		t.Errorf("Rule parser name parse error!")
+	}
+}

pkg/runner.go

@@ -0,0 +1,12 @@
+package pkg
+
+import (
+	"github.com/tg44/heptapod/pkg/parser"
+	"github.com/tg44/heptapod/pkg/walker"
+)
+
+func GetExcludedPaths(ruleFiles []string, par int, bufferSize int) []string {
+	jobs := parser.Parse(ruleFiles)
+	res := walker.Run(jobs, par, bufferSize)
+	return res
+}