Web crawler with concurrent crowling capability

Author: aditya43

06-web-crawler-concurrent/main.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"runtime"
+	"time"
+
+	"golang.org/x/net/html"
+)
+
+var fetched map[string]bool
+
+type result struct {
+	url   string
+	urls  []string
+	err   error
+	depth int
+}
+
+// Crawl uses findLinks to recursively crawl
+// pages starting with url, to a maximum of depth.
+func Crawl(url string, depth int) {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	results := make(chan *result)
+
+	fetch := func(url string, depth int) {
+		urls, err := findLinks(url)
+		results <- &result{url, urls, err, depth}
+	}
+
+	go fetch(url, depth)
+	fetched[url] = true
+
+	for fetching := 1; fetching > 0; fetching-- {
+		res := <-results
+		if res.err != nil {
+			// fmt.Println(res.err)
+			continue
+		}
+
+		fmt.Printf("found: %s\n", res.url)
+		if res.depth > 0 {
+			for _, u := range res.urls {
+				if !fetched[u] {
+					fetching++
+					go fetch(u, res.depth-1)
+					fetched[u] = true
+				}
+			}
+		}
+	}
+	close(results)
+}
+
+func main() {
+	fetched = make(map[string]bool)
+	now := time.Now()
+	Crawl("http://github.com/aditya43", 2)
+	fmt.Println("time taken:", time.Since(now))
+}
+
+func findLinks(url string) ([]string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, fmt.Errorf("getting %s: %s", url, resp.Status)
+	}
+	doc, err := html.Parse(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		return nil, fmt.Errorf("parsing %s as HTML: %v", url, err)
+	}
+	return visit(nil, doc), nil
+}
+
+// visit appends to links each link found in n, and returns the result.
+func visit(links []string, n *html.Node) []string {
+	if n.Type == html.ElementNode && n.Data == "a" {
+		for _, a := range n.Attr {
+			if a.Key == "href" {
+				links = append(links, a.Val)
+			}
+		}
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		links = visit(links, c)
+	}
+	return links
+}

go.mod

@@ -1,3 +1,5 @@
 module github.com/aditya43/golang_concurrency
 
 go 1.16
+
+require golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect

go.sum

@@ -0,0 +1,7 @@
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=