diff --git a/.gitignore b/.gitignore index 4b7675d..0dc307b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ .vscode .idea -*/**/node_modules \ No newline at end of file +*/**/node_modules +gcs-credentials.json \ No newline at end of file diff --git a/Makefile b/Makefile index 3eff58b..c26a318 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ dev-dependencies: go install github.com/canthefason/go-watcher/cmd/watcher watcher: dev-dependencies - watcher # github.com/canthefason/go-watcher + GOOGLE_APPLICATION_CREDENTIALS=`pwd`/gcs-credentials.json watcher # github.com/canthefason/go-watcher run: docker run -ti -v`pwd`:/go/src/github.com/rodrigo-brito/gocity -p80:4000 -d -w /go/src/github.com/rodrigo-brito/gocity golang go run main.go \ No newline at end of file diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go index 60d9014..da7f920 100644 --- a/analyzer/analyzer.go +++ b/analyzer/analyzer.go @@ -1,11 +1,11 @@ package analyzer import ( - "errors" "fmt" "go/ast" "go/parser" "go/token" + "log" "os" "path/filepath" "strings" @@ -15,8 +15,6 @@ import ( "github.com/rodrigo-brito/gocity/lib" ) -var ErrInvalidPackage = errors.New("invalid package") - type Analyzer interface { FetchPackage() error Analyze() (map[string]*NodeInfo, error) @@ -25,6 +23,7 @@ type Analyzer interface { type analyzer struct { PackageName string IgnoreNodes []string + fetcher lib.Fetcher } type Option func(a *analyzer) @@ -32,6 +31,7 @@ type Option func(a *analyzer) func NewAnalyzer(packageName string, options ...Option) Analyzer { analyzer := &analyzer{ PackageName: packageName, + fetcher: lib.NewFetcher(), } for _, option := range options { @@ -48,17 +48,7 @@ func WithIgnoreList(files ...string) Option { } func (p *analyzer) FetchPackage() error { - fetcher := lib.NewFetcher() - ok, err := fetcher.Fetch(p.PackageName) - if err != nil { - return err - } - - if !ok { - return ErrInvalidPackage - } - - return nil + return p.fetcher.Fetch(p.PackageName) } func (p *analyzer) IsInvalidPath(path string) bool { @@ -84,7 +74,9 @@ func (a *analyzer) Analyze() (map[string]*NodeInfo, error) { file, err := parser.ParseFile(fileSet, path, nil, parser.ParseComments) if err != nil { - return fmt.Errorf("invalid input %s: %s", path, err) + // TODO: Logo with project information + log.Printf("invalid file %s: %s", path, err) + return nil } v := &Visitor{ diff --git a/analyzer/visitor.go b/analyzer/visitor.go index 58458a8..91fa4c6 100644 --- a/analyzer/visitor.go +++ b/analyzer/visitor.go @@ -63,7 +63,11 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor { if ident, ok := typeObj.(*ast.Ident); ok { structName = ident.Name } else { - structName = typeObj.(*ast.StarExpr).X.(*ast.Ident).Name + if ident, ok := typeObj.(*ast.StarExpr).X.(*ast.Ident); ok { + structName = ident.Name + } else if ident, ok := typeObj.(*ast.StarExpr).X.(*ast.SelectorExpr); ok { + structName = ident.Sel.Name + } } } diff --git a/lib/cache.go b/lib/cache.go new file mode 100644 index 0000000..f456c99 --- /dev/null +++ b/lib/cache.go @@ -0,0 +1,52 @@ +package lib + +import ( + "time" + + "github.com/karlseguin/ccache" +) + +type Cache interface { + Get(key string) (bool, []byte) + Set(key string, value []byte, TTL time.Duration) + GetSet(key string, set func() ([]byte, error), TTL time.Duration) ([]byte, error) +} + +func NewCache() Cache { + return &cache{ + client: ccache.New(ccache.Configure()), + } +} + +type cache struct { + client *ccache.Cache +} + +func (c *cache) Get(key string) (bool, []byte) { + item := c.client.Get(key) + if item != nil { + return true, item.Value().([]byte) + } + + return false, nil +} + +func (c *cache) Set(key string, value []byte, TTL time.Duration) { + c.client.Set(key, value, TTL) +} + +func (c *cache) GetSet(key string, getValue func() ([]byte, error), TTL time.Duration) ([]byte, error) { + hit, value := c.Get(key) + if hit { + return value, nil + } + + value, err := getValue() + if err != nil { + return nil, err + } + + c.Set(key, value, TTL) + + return value, nil +} diff --git a/lib/fetch.go b/lib/fetch.go index 1f75bb8..8f875be 100644 --- a/lib/fetch.go +++ b/lib/fetch.go @@ -2,13 +2,14 @@ package lib import ( "fmt" + "log" "os" git "gopkg.in/src-d/go-git.v4" ) type Fetcher interface { - Fetch(packageName string) (bool, error) + Fetch(packageName string) error } func NewFetcher() Fetcher { @@ -25,7 +26,7 @@ func (fetcher) packageFound(name string) bool { return true } -func (f *fetcher) Fetch(name string) (bool, error) { +func (f *fetcher) Fetch(name string) error { gitAddress := fmt.Sprintf("https://%s", name) folder := fmt.Sprintf("%s/src/%s", os.Getenv("GOPATH"), name) @@ -36,8 +37,14 @@ func (f *fetcher) Fetch(name string) (bool, error) { }) if err != nil && err != git.ErrRepositoryAlreadyExists { - return false, err + go func() { + if err := os.RemoveAll(folder); err != nil { + log.Printf("error on remove: %s", err) + } + }() + + return err } - return f.packageFound(name), nil + return nil } diff --git a/lib/storage.go b/lib/storage.go new file mode 100644 index 0000000..b982976 --- /dev/null +++ b/lib/storage.go @@ -0,0 +1,86 @@ +package lib + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "log" + + "cloud.google.com/go/storage" +) + +// Sets the name for the new bucket. +const bucketName = "gocity" + +type Storage interface { + Get(projectName string) (bool, []byte, error) + Save(projectName string, content []byte) error + Delete(projectName string) error +} + +type GCS struct { + ctx context.Context + client *storage.Client +} + +func NewGCS(ctx context.Context) (Storage, error) { + client, err := storage.NewClient(ctx) + if err != nil { + return nil, err + } + + return &GCS{ + ctx: ctx, + client: client, + }, nil +} + +func getObjectName(name string) string { + return fmt.Sprintf("%s.json", name) +} + +func (g *GCS) Get(projectName string) (bool, []byte, error) { + object := g.client.Bucket(bucketName).Object(getObjectName(projectName)) + if object == nil { + return false, nil, nil + } + + reader, err := object.NewReader(g.ctx) + if err != nil { + if err == storage.ErrObjectNotExist { + log.Print("file not exists...") + return false, nil, nil + } + + return false, nil, err + } + defer reader.Close() + + data, err := ioutil.ReadAll(reader) + if err != nil { + return false, nil, err + } + + return true, data, nil +} + +func (g *GCS) Save(projectName string, content []byte) error { + client, err := storage.NewClient(g.ctx) + if err != nil { + return err + } + + buffer := bytes.NewBuffer(content) + wc := client.Bucket(bucketName).Object(getObjectName(projectName)).NewWriter(g.ctx) + if _, err = io.Copy(wc, buffer); err != nil { + return err + } + + return wc.Close() +} + +func (g *GCS) Delete(projectName string) error { + return nil +} diff --git a/main.go b/main.go index a391d59..ee6c511 100644 --- a/main.go +++ b/main.go @@ -1,22 +1,29 @@ package main import ( + "context" "encoding/json" "fmt" "log" "net/http" "os" "path/filepath" - - "github.com/rodrigo-brito/gocity/model" + "time" "github.com/go-chi/chi" "github.com/go-chi/cors" "github.com/rodrigo-brito/gocity/analyzer" + "github.com/rodrigo-brito/gocity/lib" + "github.com/rodrigo-brito/gocity/model" ) func main() { router := chi.NewRouter() + cache := lib.NewCache() + storage, err := lib.NewGCS(context.Background()) + if err != nil { + log.Fatal(err) + } cors := cors.New(cors.Options{ AllowedOrigins: []string{"*"}, @@ -31,29 +38,59 @@ func main() { router.Get("/api", func(w http.ResponseWriter, r *http.Request) { projectName := r.URL.Query().Get("q") if len(projectName) == 0 { + w.WriteHeader(http.StatusNotFound) return } - analyzer := analyzer.NewAnalyzer(projectName, analyzer.WithIgnoreList("/vendor/")) - err := analyzer.FetchPackage() + result, err := cache.GetSet(projectName, func() ([]byte, error) { + ok, data, err := storage.Get(projectName) + if err != nil { + return nil, err + } + + if ok && len(data) > 0 { + return data, nil + } + + analyzer := analyzer.NewAnalyzer(projectName, analyzer.WithIgnoreList("/vendor/")) + err = analyzer.FetchPackage() + if err != nil { + return nil, err + } + + summary, err := analyzer.Analyze() + if err != nil { + return nil, err + } + + body, err := json.Marshal(model.New(summary, projectName)) + if err != nil { + return nil, err + } + + // store result on Google Cloud Storage + go func() { + if err := storage.Save(projectName, body); err != nil { + log.Print(err) + } + }() + + return body, nil + }, time.Hour*48) + if err != nil { w.WriteHeader(http.StatusServiceUnavailable) log.Print(err) + return } - summary, err := analyzer.Analyze() - if err != nil { - w.WriteHeader(http.StatusServiceUnavailable) - log.Printf("error on analyzetion %s", err) + if len(result) == 0 { + w.WriteHeader(http.StatusNotFound) + return } - body, err := json.Marshal(model.New(summary, projectName)) - if err != nil { - w.WriteHeader(http.StatusServiceUnavailable) - log.Print(err) - } w.Header().Set("Content-Type", "application/json") - w.Write(body) + w.Write(result) }) workDir, _ := os.Getwd() diff --git a/utils/utils.go b/utils/utils.go index 0e0014a..ff86865 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -7,14 +7,17 @@ import ( "strings" ) -var pattern = regexp.MustCompile(`(\w+\.go)(?:\.\((\w+)\))?$`) +var ( + regexpFile = regexp.MustCompile(`([^/]+\.go)(?:\.\((\w+)\))?$`) + regexpGithub = regexp.MustCompile(`github\.com\/([^\/]+)\/([^\/]+)`) +) func TrimGoPath(path, repository string) string { return strings.TrimPrefix(path, fmt.Sprintf("%s/src/%s", os.Getenv("GOPATH"), repository)) } func GetFileAndStruct(identifier string) (fileName, structName string) { - result := pattern.FindStringSubmatch(identifier) + result := regexpFile.FindStringSubmatch(identifier) if len(result) > 1 { fileName = result[1] } @@ -37,3 +40,11 @@ func GetIdentifier(path, pkg, name string) string { func IsGoFile(name string) bool { return strings.HasSuffix(name, ".go") } + +func GetGithubBaseURL(path string) (string, bool) { + result := regexpGithub.FindStringSubmatch(path) + if len(result) > 2 { + return fmt.Sprintf("github.com/%s/%s", result[1], result[2]), true + } + return "", false +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 38c3b4a..17af167 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -14,6 +14,8 @@ func TestGetFileAndStruct(t *testing.T) { StructName string }{ {Input: "foo/bar/file.go", FileName: "file.go", StructName: ""}, + {Input: "foo/bar/file.pb.go", FileName: "file.pb.go", StructName: ""}, + {Input: "foo/bar/file.pb_test.go", FileName: "file.pb_test.go", StructName: ""}, {Input: "foo/bar/file.go.(Test)", FileName: "file.go", StructName: "Test"}, {Input: "foo/bar/test.go.(test)", FileName: "test.go", StructName: "test"}, {Input: "foo/bar/9999", FileName: "", StructName: ""}, @@ -27,3 +29,27 @@ func TestGetFileAndStruct(t *testing.T) { }) } } + +func TestGetGithubBaseURL(t *testing.T) { + tt := []struct { + Input string + Output string + IsValid bool + }{ + {"github.com/foo/bar", "github.com/foo/bar", true}, + {"https://github.com/foo/bar", "github.com/foo/bar", true}, + {"github.com/foo/bar/subpackage", "github.com/foo/bar", true}, + {"www.github.com/foo/bar/subpackage", "github.com/foo/bar", true}, + {"www.gitlab.com/foo/bar/subpackage", "", false}, + {"github.com/foo", "", false}, + {"invalid", "", false}, + } + + for _, tc := range tt { + t.Run(fmt.Sprintf("given the input %s", tc.Input), func(t *testing.T) { + output, valid := GetGithubBaseURL(tc.Input) + assert.Equal(t, tc.IsValid, valid) + assert.Equal(t, tc.Output, output) + }) + } +}