Todo parser

pkg/parser/fetcher.go

@@ -0,0 +1,26 @@
+package parser
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+	"io"
+	"net/http"
+)
+
+// DownloadFile downloads repository files. Http clients should have access
+// tokens already set. With this, parser does not need knowledge of private/public repos
+func DownloadFile(client *http.Client, u string) (io.ReadCloser, error) {
+	resp, err := client.Get(u)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to download file to parse")
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		if resp.Body != nil {
+			resp.Body.Close()
+		}
+		return nil, fmt.Errorf("failed to retrieve download file: status code %d", resp.StatusCode)
+	}
+
+	return resp.Body, nil
+}

pkg/parser/fetcher_test.go

@@ -0,0 +1,56 @@
+package parser
+
+import (
+	"github.com/stretchr/testify/require"
+	"io/ioutil"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func TestGetGithubFile(t *testing.T) {
+	a := require.New(t)
+	u := "https://github.com/while-loop/todo/raw/08b3e2fad64e54c061d1ba6324a382e968212a6c/pkg/vcs/github/github_test.go"
+	exp := `package github
+
+import (
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestTestValidBody(t *testing.T) {
+	secret := "mysecret"
+	payload := []byte("mypayload")
+	computed := "sha1=57852ac1e3fd8e66063cd9d4cb05ea87355bb0b8"
+
+	require.True(t, validBody(payload, secret, computed))
+}
+`
+
+	rc, err := DownloadFile(&http.Client{Timeout: 5 * time.Second}, u)
+	a.NoError(err)
+	defer rc.Close()
+
+	content, err := ioutil.ReadAll(rc)
+	a.NoError(err)
+
+	a.Equal(exp, string(content))
+}
+
+func TestFileNotFound(t *testing.T) {
+	a := require.New(t)
+	u := "https://github.com/while-loop/todo/raw/ezas123/pkg/vcs/github/github_test.go"
+
+	rc, err := DownloadFile(&http.Client{Timeout: 5 * time.Second}, u)
+	a.Contains(err.Error(), "status code 404")
+	a.Nil(rc)
+}
+
+func TestServerDown(t *testing.T) {
+	a := require.New(t)
+	u := "https://fakegithuburlTestServerDown.com/who"
+
+	rc, err := DownloadFile(&http.Client{Timeout: 5 * time.Second}, u)
+	a.Contains(err.Error(), "failed to download file to parse")
+	a.Nil(rc)
+}

pkg/parser/parser.go

@@ -1,8 +1,101 @@
 package parser
 
-import "github.com/while-loop/todo/pkg/tracker"
+import (
+	"github.com/while-loop/todo/pkg/log"
+	"github.com/while-loop/todo/pkg/tracker"
+	"net/http"
+	"sync"
+	"time"
+)
+
+const (
+	maxGoroutines = 5
+)
+
+var (
+	client = &http.Client{Timeout: 30 * time.Second}
+)
+
+type TodoParser interface {
+	GetTodos(urls ...string) []tracker.Issue
+}
+
+type todoParser struct {
+}
+
+func New() TodoParser {
+	return &todoParser{}
+}
+
+func (p *todoParser) GetTodos(urls ...string) []tracker.Issue {
+
+	jobs := make(chan string, 100)
+	results := make(chan tracker.Issue, 100)
+	finished := make(chan struct{})
+
+	wg := &sync.WaitGroup{}
+
+	log.Debugf("spinning %d goroutines for todoParser", maxGoroutines)
+	for i := 0; i < maxGoroutines; i++ {
+		wg.Add(1) // track how many goroutines we spin up
+		go worker(wg, jobs, results)
+	}
+
+	go func() {
+		log.Debug("sending urls to worker routines")
+		for _, u := range urls {
+			jobs <- u
+		}
+		close(jobs) // close the jobs channel so goroutines gracefully stop when no jobs are left
+	}()
 
-func GetTodos(files ...string) []tracker.Issue {
 	issues := make([]tracker.Issue, 0)
+	go func() {
+		log.Debug("collecting results from workers")
+		for issue := range results {
+			issues = append(issues, issue)
+		}
+		finished <- struct{}{} // let the main thread know we're done collecting issues
+	}()
+
+	log.Debug("waiting for goroutines to finish")
+	// wait for all the goroutines to gracefully finish
+	wg.Wait()
+
+	// tell the result collector that we're done waiting on worker goroutines
+	close(results)
+
+	log.Debug("waiting for collector routine to finish")
+	// wait for the result collector to finish appending issues
+	<-finished
+
+	log.Debug("done waiting for collector")
 	return issues
 }
+
+// worker downloads and parses a file given a url
+func worker(wg *sync.WaitGroup, urlChan <-chan string, results chan<- tracker.Issue) {
+	for u := range urlChan {
+		log.Info("worker recvd ", u)
+		rc, err := DownloadFile(client, u)
+		if err != nil {
+			log.Error("worker failed to download file", err)
+			continue
+		}
+
+		log.Debug("working parsing file")
+		iss, err := ParseFile(u, rc)
+		rc.Close()
+		if err != nil {
+			log.Error("worker failed to parse file", err)
+			// don't return because we could have recvd partial issues w/ an error
+		}
+
+		if len(iss) > 0 {
+			for _, is := range iss {
+				results <- is
+			}
+		}
+	}
+	wg.Done()
+}

pkg/parser/parser_test.go

@@ -0,0 +1,63 @@
+package parser
+
+import (
+	"github.com/stretchr/testify/require"
+	"github.com/while-loop/todo/pkg/tracker"
+	"reflect"
+	"testing"
+)
+
+func TestDownloadTwoFiles(t *testing.T) {
+	a := require.New(t)
+	urls := []string{"https://github.com/while-loop/todo/raw/08b3e2fad64e54c061d1ba6324a382e968212a6c/pkg/vcs/github/event_push_test.go",
+		"https://github.com/ansible/ansible/raw/781fd7099a0278d3d91557b94da1083f19fad329/test/legacy/roles/test_gce_labels/tasks/test.yml"}
+
+	iss := []tracker.Issue{
+		{Assignee: "erjohnso", Title: "write more tests", File: urls[1], Line: 28},
+		{Title: "test when parser has been impl", File: urls[0], Line: 18},
+	}
+
+	p := New()
+	issues := p.GetTodos(urls...)
+	a.Equal(len(iss), len(issues))
+	ttl := 0 // keep a count of total issues matched. since order of received issues are random due to goroutines
+
+	for _, expIs := range iss {
+		for i, actIs := range issues {
+			if reflect.DeepEqual(expIs, actIs) {
+				ttl++
+				issues = remove(issues, i)
+				break
+			}
+		}
+	}
+
+	a.Equal(len(iss), ttl)
+}
+
+func TestDownloadWhenServerIsNotReachable(t *testing.T) {
+	a := require.New(t)
+	urls := []string{"https://gitakjshdgfasjhgdfhub.com"}
+
+	p := New()
+	issues := p.GetTodos(urls...)
+
+	a.Equal(0, len(issues))
+}
+
+func TestParsingUnsupportedExtension(t *testing.T) {
+	a := require.New(t)
+
+	// readme.md has valid todo comments, but md is an supp extension atm
+	urls := []string{"https://github.com/while-loop/todo/raw/cc6b554cccfd3598f6b6342d69c78abcbc5d0128/README.md"}
+
+	p := New()
+	issues := p.GetTodos(urls...)
+
+	a.Equal(0, len(issues))
+}
+
+func remove(s []tracker.Issue, i int) []tracker.Issue {
+	s[i] = s[len(s)-1]
+	return s[:len(s)-1]
+}