check: fall back to heuristically scanning directory index

distr1 · Mar 1, 2020 · 0807389 · 0807389
1 parent 8250aa3
commit 0807389
Show file tree

Hide file tree

Showing 3 changed files with 100 additions and 2 deletions.
diff --git a/go.mod b/go.mod
@@ -16,12 +16,13 @@ require (
 	github.com/protocolbuffers/txtpbfmt v0.0.0-20191018194151-ab9b9b21328a
 	github.com/s-urbaniak/uevent v1.0.0
 	golang.org/x/exp v0.0.0-20190221220918-438050ddec5e
+	golang.org/x/mod v0.2.0
 	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b
 	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7
+	golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898
 	gonum.org/v1/gonum v0.0.0-20181012194325-406984d37414
 	gonum.org/v1/netlib v0.0.0-20181018051557-57e1e4db57a7 // indirect
 	google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 // indirect

diff --git a/go.sum b/go.sum
@@ -41,17 +41,23 @@ github.com/s-urbaniak/uevent v1.0.0 h1:y1e40TmDfCU0l1o/MBpimGJgo2UzTTqBl/0GF6vx/
 github.com/s-urbaniak/uevent v1.0.0/go.mod h1:h83SVc1NQj/EubPaK4hWyiG217xLYr+fS+Jf10COrG0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190221220918-438050ddec5e h1:dVreTP5bOOWt5GFwwvgTE2iU0TkIqi2x3r0b8qGlp6k=
 golang.org/x/exp v0.0.0-20190221220918-438050ddec5e/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
@@ -65,6 +71,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
@@ -76,8 +83,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.0.0-20181012194325-406984d37414 h1:I9CLYgoY02/pOK+fYZ5PSRwoGafdCYnUMtiiQ8cS3Co=
 gonum.org/v1/gonum v0.0.0-20181012194325-406984d37414/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/netlib v0.0.0-20181018051557-57e1e4db57a7 h1:biGE8p3sxXv+JdArdlsXEJZBPPFk51mQHVwOv6h1gnY=

diff --git a/internal/checkupstream/check.go b/internal/checkupstream/check.go
@@ -4,14 +4,19 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"path"
+	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/distr1/distri"
 	"github.com/protocolbuffers/txtpbfmt/ast"
+	"golang.org/x/mod/semver"
 )
 
 func checkDebian(debianPackagesURL, source string) (remoteSource, remoteHash, remoteVersion string, _ error) {
@@ -71,6 +76,80 @@ func checkDebian(debianPackagesURL, source string) (remoteSource, remoteHash, re
 	return "", "", "", fmt.Errorf("package not found in Debian Packages file")
 }
 
+func maybeV(v string) string {
+	if strings.HasPrefix(v, "v") {
+		return v
+	}
+	return "v" + v
+}
+
+func checkHeuristic(upstreamVersion, source string) (remoteSource, remoteHash, remoteVersion string, _ error) {
+	u, err := url.Parse(source)
+	u.Path = path.Dir(u.Path)
+	resp, err := http.Get(u.String())
+	if err != nil {
+		return "", "", "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", "", "", fmt.Errorf("unexpected HTTP response: got %v, want 200 OK", resp.Status)
+	}
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", "", "", err
+	}
+	// TODO: add special casing for parsing apache directory index?
+	log.Printf("base: %v", path.Base(source))
+	pattern := strings.Replace(path.Base(source), upstreamVersion, `([^"]*)`, 1)
+	if pattern == path.Base(source) {
+		return "", "", "", fmt.Errorf("could not derive regexp pattern, specify manually")
+	}
+	log.Printf("pattern: %v", pattern)
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return "", "", "", err
+	}
+	log.Printf("re: %v", re)
+	matches := re.FindAllStringSubmatch(string(b), -1)
+	log.Printf("matches: %v", matches)
+	versions := func() []string {
+		v := make(map[string]bool)
+		for _, m := range matches {
+			if m[1] == "latest" {
+				continue // skip common latest symlink
+			}
+			v[m[1]] = true
+		}
+		result := make([]string, 0, len(v))
+		for version := range v {
+			result = append(result, version)
+		}
+		valid := true
+		for _, r := range result {
+			if !semver.IsValid(maybeV(r)) {
+				valid = false
+				break
+			}
+		}
+		if !valid {
+			// Prefer a string sort when the versions aren’t semver, it’s better
+			// than semver.Compare.
+			sort.Sort(sort.Reverse(sort.StringSlice(result)))
+		} else {
+			sort.Slice(result, func(i, j int) bool {
+				v, w := result[i], result[j]
+				v, w = maybeV(v), maybeV(w)
+				return semver.Compare(v, w) >= 0 // reverse
+			})
+		}
+		return result
+	}()
+	if len(versions) == 0 {
+		return "", "", "", fmt.Errorf("not yet implemented")
+	}
+	u.Path = path.Join(u.Path, strings.Replace(path.Base(source), upstreamVersion, versions[0], 1))
+	return u.String(), "TODO", versions[0], nil
+}
+
 func Check(build []*ast.Node) (source, hash, version string, _ error) {
 	stringVal := func(path ...string) (string, error) {
 		nodes := ast.GetFromPath(build, path)
@@ -95,5 +174,13 @@ func Check(build []*ast.Node) (source, hash, version string, _ error) {
 		}
 		return checkDebian(u, source)
 	}
-	return "", "", "", fmt.Errorf("not yet implemented")
+
+	version, err = stringVal("version")
+	if err != nil {
+		return "", "", "", err
+	}
+
+	pv := distri.ParseVersion(version)
+	// fall back: see if we can obtain an index page
+	return checkHeuristic(pv.Upstream, source)
 }