Skip to content

Commit

Permalink
git: update parsing to clarify between scp-style urls
Browse files Browse the repository at this point in the history
This should also resolve the ports parsing issue.

Signed-off-by: Justin Chadwell <me@jedevc.com>
  • Loading branch information
jedevc committed Aug 14, 2023
1 parent 7173c15 commit 6c78917
Show file tree
Hide file tree
Showing 5 changed files with 160 additions and 70 deletions.
30 changes: 9 additions & 21 deletions client/llb/source.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,27 +237,15 @@ type ImageInfo struct {
//
// By default the git repository is cloned with `--depth=1` to reduce the amount of data downloaded.
// Additionally the ".git" directory is removed after the clone, you can keep ith with the [KeepGitDir] [GitOption].
func Git(remote, ref string, opts ...GitOption) State {
url := strings.Split(remote, "#")[0]

var protocolType int
remote, protocolType = gitutil.ParseProtocol(remote)

var sshHost string
if protocolType == gitutil.SSHProtocol {
parts := strings.SplitN(remote, ":", 2)
if len(parts) == 2 {
sshHost = parts[0]
// keep remote consistent with http(s) version
remote = parts[0] + "/" + parts[1]
}
}
if protocolType == gitutil.UnknownProtocol {
func Git(url, ref string, opts ...GitOption) State {
url = strings.Split(url, "#")[0]
proto, remote, path, err := gitutil.ParseProtocol(url)
if err != nil && proto == gitutil.UnknownProtocol {
url = "https://" + url
proto, remote, path, _ = gitutil.ParseProtocol(url)
}

id := remote

id := remote + "/" + strings.TrimPrefix(path, "/")
if ref != "" {
id += "#" + ref
}
Expand Down Expand Up @@ -290,11 +278,11 @@ func Git(remote, ref string, opts ...GitOption) State {
addCap(&gi.Constraints, pb.CapSourceGitHTTPAuth)
}
}
if protocolType == gitutil.SSHProtocol {
if proto == gitutil.SSHProtocol {
if gi.KnownSSHHosts != "" {
attrs[pb.AttrKnownSSHHosts] = gi.KnownSSHHosts
} else if sshHost != "" {
keyscan, err := sshutil.SSHKeyScan(sshHost)
} else if remote != "" {
keyscan, err := sshutil.SSHKeyScan(remote)
if err == nil {
// best effort
attrs[pb.AttrKnownSSHHosts] = keyscan
Expand Down
83 changes: 55 additions & 28 deletions util/gitutil/git_protocol.go
Original file line number Diff line number Diff line change
@@ -1,46 +1,73 @@
package gitutil

import (
"net/url"
"regexp"
"strings"

"github.com/containerd/containerd/errdefs"
"github.com/moby/buildkit/util/sshutil"
)

type Protocol string

const (
HTTPProtocol = iota + 1
HTTPSProtocol
SSHProtocol
GitProtocol
UnknownProtocol
HTTPProtocol Protocol = "http"
HTTPSProtocol Protocol = "https"
SSHProtocol Protocol = "ssh"
GitProtocol Protocol = "git"
UnknownProtocol Protocol = "unknown"
)

// ParseProtocol parses a git URL and returns the remote url and protocol type
func ParseProtocol(remote string) (string, int) {
prefixes := map[string]int{
"http://": HTTPProtocol,
"https://": HTTPSProtocol,
"git://": GitProtocol,
"ssh://": SSHProtocol,
}
protocolType := UnknownProtocol
for prefix, potentialType := range prefixes {
if strings.HasPrefix(remote, prefix) {
remote = strings.TrimPrefix(remote, prefix)
protocolType = potentialType
var supportedProtos = map[Protocol]struct{}{
HTTPProtocol: {},
HTTPSProtocol: {},
SSHProtocol: {},
GitProtocol: {},
}

var protoRegexp = regexp.MustCompile(`^[a-z0-9]+://`)

// ParseProtocol parses a git URL and returns the protocol type, the remote,
// and the path.
//
// ParseProtocol understands implicit ssh URLs such as "git@host:repo", and
// returns the same response as if the URL were "ssh://git@host/repo".
func ParseProtocol(remote string) (Protocol, string, string, error) {
if proto := protoRegexp.FindString(remote); proto != "" {
proto := Protocol(strings.TrimSuffix(proto, "://"))
_, ok := supportedProtos[proto]
if !ok {
return proto, remote, "", errdefs.ErrInvalidArgument
}
}

if protocolType == UnknownProtocol && sshutil.IsImplicitSSHTransport(remote) {
protocolType = SSHProtocol
}
u, err := url.Parse(remote)
if err != nil {
return proto, remote, "", err
}
if Protocol(u.Scheme) != proto {
// this should never happen
return proto, remote, "", errdefs.ErrInvalidArgument
}

host := u.Host
if u.User != nil {
host = u.User.String() + "@" + host
}

// remove name from ssh
if protocolType == SSHProtocol {
parts := strings.SplitN(remote, "@", 2)
if len(parts) == 2 {
remote = parts[1]
path := u.Path
path = strings.TrimPrefix(path, "/")
if u.Fragment != "" {
path += "#" + u.Fragment
}

return proto, host, path, nil
}

if sshutil.IsImplicitSSHTransport(remote) {
remote, path, _ := strings.Cut(remote, ":")
return SSHProtocol, remote, path, nil
}

return remote, protocolType
return UnknownProtocol, remote, "", errdefs.ErrInvalidArgument
}
68 changes: 58 additions & 10 deletions util/gitutil/git_protocol_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,45 +9,93 @@ import (
func TestParseProtocol(t *testing.T) {
tests := []struct {
url string
protocol int
protocol Protocol
remote string
path string
err bool
}{
{
url: "http://github.com/moby/buildkit",
protocol: HTTPProtocol,
remote: "github.com/moby/buildkit",
remote: "github.com",
path: "moby/buildkit",
},
{
url: "https://github.com/moby/buildkit",
protocol: HTTPSProtocol,
remote: "github.com/moby/buildkit",
remote: "github.com",
path: "moby/buildkit",
},
{
url: "http://github.com/moby/buildkit#v1.0.0",
protocol: HTTPProtocol,
remote: "github.com",
path: "moby/buildkit#v1.0.0",
},
{
url: "http://foo:bar@github.com/moby/buildkit#v1.0.0",
protocol: HTTPProtocol,
remote: "foo:bar@github.com",
path: "moby/buildkit#v1.0.0",
},
{
url: "ssh://git@github.com/moby/buildkit.git",
protocol: SSHProtocol,
remote: "git@github.com",
path: "moby/buildkit.git",
},
{
url: "ssh://git@github.com:22/moby/buildkit.git",
protocol: SSHProtocol,
remote: "git@github.com:22",
path: "moby/buildkit.git",
},
{
url: "git@github.com:moby/buildkit.git",
protocol: SSHProtocol,
remote: "github.com:moby/buildkit.git",
remote: "git@github.com",
path: "moby/buildkit.git",
},
{
url: "git@github.com:moby/buildkit.git#v1.0.0",
protocol: SSHProtocol,
remote: "git@github.com",
path: "moby/buildkit.git#v1.0.0",
},
{
url: "nonstandarduser@example.com:/srv/repos/weird/project.git",
protocol: SSHProtocol,
remote: "example.com:/srv/repos/weird/project.git",
remote: "nonstandarduser@example.com",
path: "/srv/repos/weird/project.git",
},
{
url: "ssh://root@subdomain.example.hostname:2222/root/my/really/weird/path/foo.git",
protocol: SSHProtocol,
remote: "subdomain.example.hostname:2222/root/my/really/weird/path/foo.git",
remote: "root@subdomain.example.hostname:2222",
path: "root/my/really/weird/path/foo.git",
},
{
url: "git://host.xz:1234/path/to/repo.git",
protocol: GitProtocol,
remote: "host.xz:1234/path/to/repo.git",
remote: "host.xz:1234",
path: "path/to/repo.git",
},
{
url: "httpx://github.com/moby/buildkit",
err: true,
},
}
for _, test := range tests {
t.Run(test.url, func(t *testing.T) {
protocol, remote, path := ParseProtocol(test.url)
require.Equal(t, test.protocol, protocol)
require.Equal(t, test.remote, remote)
protocol, remote, path, err := ParseProtocol(test.url)
if test.err {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, test.protocol, protocol)
require.Equal(t, test.remote, remote)
require.Equal(t, test.path, path)
}
})
}
}
29 changes: 21 additions & 8 deletions util/gitutil/git_ref.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,18 +51,30 @@ type GitRef struct {
func ParseGitRef(ref string) (*GitRef, error) {
res := &GitRef{}

var (
proto Protocol
remote string
path string
err error
)

if strings.HasPrefix(ref, "github.com/") {
res.IndistinguishableFromLocal = true // Deprecated
proto, remote, path = HTTPSProtocol, "github.com", strings.TrimPrefix(ref, "github.com/")
} else {
_, proto := ParseProtocol(ref)
switch proto {
case UnknownProtocol:
return nil, errdefs.ErrInvalidArgument
proto, remote, path, err = ParseProtocol(ref)
if err != nil && proto == UnknownProtocol {
proto, remote, path, err = ParseProtocol("https://" + ref)
}
if err != nil {
return nil, err
}

switch proto {
case HTTPProtocol, GitProtocol:
res.UnencryptedTCP = true // Discouraged, but not deprecated
}

switch proto {
// An HTTP(S) URL is considered to be a valid git ref only when it has the ".git[...]" suffix.
case HTTPProtocol, HTTPSProtocol:
Expand All @@ -73,11 +85,12 @@ func ParseGitRef(ref string) (*GitRef, error) {
}
}

var fragment string
res.Remote, fragment, _ = strings.Cut(ref, "#")
if len(res.Remote) == 0 {
return res, errdefs.ErrInvalidArgument
path, fragment, _ := strings.Cut(path, "#")
res.Remote = remote + "/" + strings.TrimPrefix(path, "/")
if !res.IndistinguishableFromLocal {
res.Remote = string(proto) + "://" + res.Remote
}

res.Commit, res.SubDir, _ = strings.Cut(fragment, ":")
repoSplitBySlash := strings.Split(res.Remote, "/")
res.ShortName = strings.TrimSuffix(repoSplitBySlash[len(repoSplitBySlash)-1], ".git")
Expand Down
20 changes: 17 additions & 3 deletions util/gitutil/git_ref_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,13 @@ func TestParseGitRef(t *testing.T) {
IndistinguishableFromLocal: true,
},
},
{
ref: "custom.xyz/moby/buildkit.git",
expected: &GitRef{
Remote: "https://custom.xyz/moby/buildkit.git",
ShortName: "buildkit",
},
},
{
ref: "https://github.com/moby/buildkit",
expected: nil,
Expand All @@ -77,24 +84,31 @@ func TestParseGitRef(t *testing.T) {
ShortName: "buildkit",
},
},
{
ref: "https://foo:bar@github.com/moby/buildkit.git",
expected: &GitRef{
Remote: "https://foo:bar@github.com/moby/buildkit.git",
ShortName: "buildkit",
},
},
{
ref: "git@github.com:moby/buildkit",
expected: &GitRef{
Remote: "git@github.com:moby/buildkit",
Remote: "ssh://git@github.com/moby/buildkit",
ShortName: "buildkit",
},
},
{
ref: "git@github.com:moby/buildkit.git",
expected: &GitRef{
Remote: "git@github.com:moby/buildkit.git",
Remote: "ssh://git@github.com/moby/buildkit.git",
ShortName: "buildkit",
},
},
{
ref: "git@bitbucket.org:atlassianlabs/atlassian-docker.git",
expected: &GitRef{
Remote: "git@bitbucket.org:atlassianlabs/atlassian-docker.git",
Remote: "ssh://git@bitbucket.org/atlassianlabs/atlassian-docker.git",
ShortName: "atlassian-docker",
},
},
Expand Down

0 comments on commit 6c78917

Please sign in to comment.