Use JSON API instead of scraping golang.org/dl (#46)

cmd/godeb/main.go

@@ -10,15 +10,12 @@
 package main
 
 import (
-	"bytes"
+	"encoding/json"
 	"fmt"
 	"go/build"
-	"gopkg.in/xmlpath.v1"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
-	"path"
 	"sort"
 	"strings"
 )
@@ -116,13 +113,6 @@ func actionCommand(version string, install bool) error {
 				break
 			}
 		}
-		if url == "" {
-			var urls []string
-			for _, source := range tarballSources {
-				urls = append(urls, source.url)
-			}
-			return fmt.Errorf("version %s not available at %s", version, strings.Join(urls, " or "))
-		}
 	}
 
 	installed, err := installedDebVersion()
@@ -179,109 +169,47 @@ type Tarball struct {
 	Version string
 }
 
-type tarballSource struct {
-	url, xpath string
+type GolangDlFile struct {
+	Arch     string `json:"arch"`
+	Filename string `json:"filename"`
+	Os       string `json:"os"`
+	Version  string `json:"version"`
 }
 
-var tarballSources = []tarballSource{
-	{"https://golang.org/dl/", "//a[@class='download']/@href"},
+type GolangDlVersion struct {
+	Version string         `json:"version"`
+	Files   []GolangDlFile `json:"files"`
 }
 
+// REST API described in https://github.com/golang/website/blob/master/internal/dl/dl.go
 func tarballs() ([]*Tarball, error) {
-	type result struct {
-		tarballs []*Tarball
-		err      error
-	}
-	results := make(chan result)
-	for _, source := range tarballSources {
-		source := source
-		go func() {
-			tbs, err := tarballsFrom(source)
-			results <- result{tbs, err}
-		}()
-	}
+	url := "https://golang.org/dl/?mode=json&include=all"
+	downloadBaseURL := "https://dl.google.com/go/"
 
-	var tbs []*Tarball
-	var err error
-	for _ = range tarballSources {
-		r := <-results
-		if r.err != nil {
-			err = r.err
-		} else {
-			tbs = append(tbs, r.tarballs...)
-		}
-	}
+	resp, err := http.Get(url)
 	if err != nil {
 		return nil, err
 	}
-	sort.Sort(sort.Reverse(tarballSlice(tbs)))
-	return tbs, nil
-}
 
-func tarballsFrom(source tarballSource) ([]*Tarball, error) {
-	resp, err := http.Get(source.url)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read http response: %v", err)
-	}
-	clearScripts(data)
-	root, err := xmlpath.ParseHTML(bytes.NewBuffer(data))
+	var versions []GolangDlVersion
+	err = json.NewDecoder(resp.Body).Decode(&versions)
 	if err != nil {
 		return nil, err
 	}
+
 	var tbs []*Tarball
-	iter := xmlpath.MustCompile(source.xpath).Iter(root)
-	seen := make(map[string]bool)
-	for iter.Next() {
-		s := iter.Node().String()
-		if strings.HasPrefix(s, "//") {
-			s = "https:" + s
-		}
-		if strings.HasPrefix(s, "/dl/") {
-			s = source.url + s[4:]
-		}
-		if tb, ok := parseURL(s); ok && !seen[tb.Version] {
-			seen[tb.Version] = true
-			tbs = append(tbs, tb)
+	for _, v := range versions {
+		for _, f := range v.Files {
+			if f.Os == build.Default.GOOS && f.Arch == build.Default.GOARCH {
+				t := Tarball{
+					Version: strings.TrimPrefix(f.Version, "go"),
+					URL:     downloadBaseURL + f.Filename}
+				tbs = append(tbs, &t)
+				break
+			}
 		}
 	}
-	if len(tbs) == 0 {
-		return nil, fmt.Errorf("no downloads available at " + source.url)
-	}
-	return tbs, nil
-}
 
-func parseURL(url string) (tb *Tarball, ok bool) {
-	// url looks like https://.../go1.1beta2.linux-amd64.tar.gz
-	_, s := path.Split(url)
-	if len(s) < 3 || !strings.HasPrefix(s, "go") || !(s[2] >= '1' && s[2] <= '9') {
-		return nil, false
-	}
-	suffix := fmt.Sprintf(".linux-%s.tar.gz", build.Default.GOARCH)
-	if !strings.HasSuffix(s, suffix) {
-		return nil, false
-	}
-	return &Tarball{url, s[2 : len(s)-len(suffix)]}, true
-}
-
-func clearScripts(data []byte) {
-	startTag := []byte("<script")
-	closeTag := []byte("</script>")
-	var i, j int
-	for {
-		i = j + bytes.Index(data[j:], startTag)
-		if i < j {
-			break
-		}
-		i = i + bytes.IndexByte(data[i:], '>') + 1
-		j = i + bytes.Index(data[i:], closeTag)
-		for i < j {
-			data[i] = ' '
-			i++
-		}
-	}
+	sort.Sort(sort.Reverse(tarballSlice(tbs)))
+	return tbs, nil
 }