Skip to content
Permalink
Browse files
check: fall back to heuristically scanning directory index
  • Loading branch information
stapelberg committed Mar 1, 2020
1 parent 8250aa3 commit 08073898f33e12d9bec824bb3bd29581efe3dd94
Showing 3 changed files with 100 additions and 2 deletions.
3 go.mod
@@ -16,12 +16,13 @@ require (
github.com/protocolbuffers/txtpbfmt v0.0.0-20191018194151-ab9b9b21328a
github.com/s-urbaniak/uevent v1.0.0
golang.org/x/exp v0.0.0-20190221220918-438050ddec5e
golang.org/x/mod v0.2.0
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b
golang.org/x/text v0.3.2 // indirect
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898
gonum.org/v1/gonum v0.0.0-20181012194325-406984d37414
gonum.org/v1/netlib v0.0.0-20181018051557-57e1e4db57a7 // indirect
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 // indirect
10 go.sum
@@ -41,17 +41,23 @@ github.com/s-urbaniak/uevent v1.0.0 h1:y1e40TmDfCU0l1o/MBpimGJgo2UzTTqBl/0GF6vx/
github.com/s-urbaniak/uevent v1.0.0/go.mod h1:h83SVc1NQj/EubPaK4hWyiG217xLYr+fS+Jf10COrG0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190221220918-438050ddec5e h1:dVreTP5bOOWt5GFwwvgTE2iU0TkIqi2x3r0b8qGlp6k=
golang.org/x/exp v0.0.0-20190221220918-438050ddec5e/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
@@ -65,6 +71,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
@@ -76,8 +83,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.0.0-20181012194325-406984d37414 h1:I9CLYgoY02/pOK+fYZ5PSRwoGafdCYnUMtiiQ8cS3Co=
gonum.org/v1/gonum v0.0.0-20181012194325-406984d37414/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/netlib v0.0.0-20181018051557-57e1e4db57a7 h1:biGE8p3sxXv+JdArdlsXEJZBPPFk51mQHVwOv6h1gnY=
@@ -4,14 +4,19 @@ import (
"context"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"path"
"regexp"
"sort"
"strconv"
"strings"
"time"

"github.com/distr1/distri"
"github.com/protocolbuffers/txtpbfmt/ast"
"golang.org/x/mod/semver"
)

func checkDebian(debianPackagesURL, source string) (remoteSource, remoteHash, remoteVersion string, _ error) {
@@ -71,6 +76,80 @@ func checkDebian(debianPackagesURL, source string) (remoteSource, remoteHash, re
return "", "", "", fmt.Errorf("package not found in Debian Packages file")
}

func maybeV(v string) string {
if strings.HasPrefix(v, "v") {
return v
}
return "v" + v
}

func checkHeuristic(upstreamVersion, source string) (remoteSource, remoteHash, remoteVersion string, _ error) {
u, err := url.Parse(source)
u.Path = path.Dir(u.Path)
resp, err := http.Get(u.String())
if err != nil {
return "", "", "", err
}
if resp.StatusCode != http.StatusOK {
return "", "", "", fmt.Errorf("unexpected HTTP response: got %v, want 200 OK", resp.Status)
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", "", "", err
}
// TODO: add special casing for parsing apache directory index?
log.Printf("base: %v", path.Base(source))
pattern := strings.Replace(path.Base(source), upstreamVersion, `([^"]*)`, 1)
if pattern == path.Base(source) {
return "", "", "", fmt.Errorf("could not derive regexp pattern, specify manually")
}
log.Printf("pattern: %v", pattern)
re, err := regexp.Compile(pattern)
if err != nil {
return "", "", "", err
}
log.Printf("re: %v", re)
matches := re.FindAllStringSubmatch(string(b), -1)
log.Printf("matches: %v", matches)
versions := func() []string {
v := make(map[string]bool)
for _, m := range matches {
if m[1] == "latest" {
continue // skip common latest symlink
}
v[m[1]] = true
}
result := make([]string, 0, len(v))
for version := range v {
result = append(result, version)
}
valid := true
for _, r := range result {
if !semver.IsValid(maybeV(r)) {
valid = false
break
}
}
if !valid {
// Prefer a string sort when the versions aren’t semver, it’s better
// than semver.Compare.
sort.Sort(sort.Reverse(sort.StringSlice(result)))
} else {
sort.Slice(result, func(i, j int) bool {
v, w := result[i], result[j]
v, w = maybeV(v), maybeV(w)
return semver.Compare(v, w) >= 0 // reverse
})
}
return result
}()
if len(versions) == 0 {
return "", "", "", fmt.Errorf("not yet implemented")
}
u.Path = path.Join(u.Path, strings.Replace(path.Base(source), upstreamVersion, versions[0], 1))
return u.String(), "TODO", versions[0], nil
}

func Check(build []*ast.Node) (source, hash, version string, _ error) {
stringVal := func(path ...string) (string, error) {
nodes := ast.GetFromPath(build, path)
@@ -95,5 +174,13 @@ func Check(build []*ast.Node) (source, hash, version string, _ error) {
}
return checkDebian(u, source)
}
return "", "", "", fmt.Errorf("not yet implemented")

version, err = stringVal("version")
if err != nil {
return "", "", "", err
}

pv := distri.ParseVersion(version)
// fall back: see if we can obtain an index page
return checkHeuristic(pv.Upstream, source)
}

0 comments on commit 0807389

Please sign in to comment.