From 711d49770f878f62031ee352058789af6cabf594 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Fri, 5 Sep 2025 20:27:45 +0100 Subject: [PATCH 1/7] Improve logging to include proxy vars --- go/extractor/util/registryproxy.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 301d45896d2e..fb140cba7283 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -113,11 +113,6 @@ func getEnvVars() []string { // Applies private package proxy related environment variables to `cmd`. func ApplyProxyEnvVars(cmd *exec.Cmd) { - slog.Debug( - "Applying private registry proxy environment variables", - slog.String("cmd_args", strings.Join(cmd.Args, " ")), - ) - // If we haven't done so yet, check whether the proxy environment variables are set // and extract information from them. if !proxy_vars_checked { @@ -131,4 +126,10 @@ func ApplyProxyEnvVars(cmd *exec.Cmd) { if proxy_vars != nil { cmd.Env = append(os.Environ(), proxy_vars...) } + + slog.Debug( + "Applying private registry proxy environment variables", + slog.String("cmd_args", strings.Join(cmd.Args, " ")), + slog.String("proxy_vars", strings.Join(proxy_vars, ",")), + ) } From 23a04613c08ef2c5ef75309c4336c4561321461b Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Thu, 4 Sep 2025 17:40:38 +0100 Subject: [PATCH 2/7] Set lower-case variants of `HTTP_PROXY` and `HTTPS_PROXY` --- go/extractor/util/registryproxy.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index fb140cba7283..c3faab4900a9 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -53,7 +53,13 @@ func getEnvVars() []string { if proxy_host, proxy_host_set := os.LookupEnv(PROXY_HOST); proxy_host_set && proxy_host != "" { if proxy_port, proxy_port_set := os.LookupEnv(PROXY_PORT); proxy_port_set && proxy_port != "" { proxy_address = fmt.Sprintf("http://%s:%s", proxy_host, proxy_port) - result = append(result, fmt.Sprintf("HTTP_PROXY=%s", proxy_address), fmt.Sprintf("HTTPS_PROXY=%s", proxy_address)) + result = append( + result, + fmt.Sprintf("HTTP_PROXY=%s", proxy_address), + fmt.Sprintf("HTTPS_PROXY=%s", proxy_address), + fmt.Sprintf("http_proxy=%s", proxy_address), + fmt.Sprintf("https_proxy=%s", proxy_address), + ) slog.Info("Found private registry proxy", slog.String("proxy_address", proxy_address)) } From 895399ff0569686e0b7995e7fadc7ab82e811716 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Thu, 4 Sep 2025 15:31:59 +0100 Subject: [PATCH 3/7] Rename `proxy_configs` to `goproxy_servers` and only store URLs --- go/extractor/util/registryproxy.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index c3faab4900a9..7ebc3cb508db 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -26,9 +26,8 @@ var proxy_address string // The path to the temporary file that stores the proxy certificate, if any. var proxy_cert_file string -// An array of registry configurations that are relevant to Go. -// This excludes other registry configurations that may be available, but are not relevant to Go. -var proxy_configs []RegistryConfig +// An array of goproxy server URLs. +var goproxy_servers []string // Stores the environment variables that we wish to pass on to `go` commands. var proxy_vars []string = nil @@ -97,16 +96,16 @@ func getEnvVars() []string { // filter others out at this point. for _, cfg := range val { if cfg.Type == GOPROXY_SERVER { - proxy_configs = append(proxy_configs, cfg) + goproxy_servers = append(goproxy_servers, cfg.URL) slog.Info("Found GOPROXY server", slog.String("url", cfg.URL)) } } - if len(proxy_configs) > 0 { + if len(goproxy_servers) > 0 { goproxy_val := "https://proxy.golang.org,direct" - for _, cfg := range proxy_configs { - goproxy_val = cfg.URL + "," + goproxy_val + for _, url := range goproxy_servers { + goproxy_val = url + "," + goproxy_val } result = append(result, fmt.Sprintf("GOPROXY=%s", goproxy_val), "GOPRIVATE=", "GONOPROXY=") From a8fa1a76c482146e9cc59c1d58309e6c0eb02f90 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Thu, 4 Sep 2025 15:52:53 +0100 Subject: [PATCH 4/7] Use `git_source` configurations for `GOPRIVATE` --- go/extractor/util/registryproxy.go | 17 ++++++++++++++- go/extractor/util/registryproxy_test.go | 28 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 7ebc3cb508db..9aa50ca4cc37 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -14,6 +14,7 @@ const PROXY_PORT = "CODEQL_PROXY_PORT" const PROXY_CA_CERTIFICATE = "CODEQL_PROXY_CA_CERTIFICATE" const PROXY_URLS = "CODEQL_PROXY_URLS" const GOPROXY_SERVER = "goproxy_server" +const GIT_SOURCE = "git_source" type RegistryConfig struct { Type string `json:"type"` @@ -29,6 +30,9 @@ var proxy_cert_file string // An array of goproxy server URLs. var goproxy_servers []string +// An array of Git URLs. +var git_sources []string + // Stores the environment variables that we wish to pass on to `go` commands. var proxy_vars []string = nil @@ -98,9 +102,14 @@ func getEnvVars() []string { if cfg.Type == GOPROXY_SERVER { goproxy_servers = append(goproxy_servers, cfg.URL) slog.Info("Found GOPROXY server", slog.String("url", cfg.URL)) + } else if cfg.Type == GIT_SOURCE { + git_sources = append(git_sources, cfg.URL) + slog.Info("Found Git source", slog.String("url", cfg.URL)) } } + goprivate := []string{} + if len(goproxy_servers) > 0 { goproxy_val := "https://proxy.golang.org,direct" @@ -108,8 +117,14 @@ func getEnvVars() []string { goproxy_val = url + "," + goproxy_val } - result = append(result, fmt.Sprintf("GOPROXY=%s", goproxy_val), "GOPRIVATE=", "GONOPROXY=") + result = append(result, fmt.Sprintf("GOPROXY=%s", goproxy_val), "GONOPROXY=") } + + if len(git_sources) > 0 { + goprivate = append(goprivate, git_sources...) + } + + result = append(result, fmt.Sprintf("GOPRIVATE=%s", strings.Join(goprivate, ","))) } } diff --git a/go/extractor/util/registryproxy_test.go b/go/extractor/util/registryproxy_test.go index a21b1a215c11..ef63bd9d3f87 100644 --- a/go/extractor/util/registryproxy_test.go +++ b/go/extractor/util/registryproxy_test.go @@ -47,3 +47,31 @@ func TestParseRegistryConfigs(t *testing.T) { t.Fatalf("Expected `URL` to be `https://proxy.example.com/mod`, but got `%s`", first.URL) } } + +func TestParseRegistryConfigsMultiple(t *testing.T) { + multiple := parseRegistryConfigsSuccess(t, "[{ \"type\": \"git_source\", \"url\": \"https://github.com/github\" }, { \"type\": \"goproxy_server\", \"url\": \"https://proxy.example.com/mod\" }]") + + if len(multiple) != 2 { + t.Fatalf("Expected `parseRegistryConfigs` to return two configurations, but got %d.", len(multiple)) + } + + first := multiple[0] + + if first.Type != "git_source" { + t.Fatalf("Expected `Type` to be `git_source`, but got `%s`", first.Type) + } + + if first.URL != "https://github.com/github" { + t.Fatalf("Expected `URL` to be `https://github.com/github`, but got `%s`", first.URL) + } + + second := multiple[1] + + if second.Type != "goproxy_server" { + t.Fatalf("Expected `Type` to be `goproxy_server`, but got `%s`", second.Type) + } + + if second.URL != "https://proxy.example.com/mod" { + t.Fatalf("Expected `URL` to be `https://proxy.example.com/mod`, but got `%s`", second.URL) + } +} From 4ef8ff9a0fe87940510a2dffe2e1defdfd64c5f2 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Fri, 5 Sep 2025 20:35:14 +0100 Subject: [PATCH 5/7] Append `*` to `git_source` URL if not present Since `GOPRIVATE` / `GONOPROXY` expect a glob pattern --- go/extractor/util/registryproxy.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 9aa50ca4cc37..0ba49cb55d65 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -103,7 +103,11 @@ func getEnvVars() []string { goproxy_servers = append(goproxy_servers, cfg.URL) slog.Info("Found GOPROXY server", slog.String("url", cfg.URL)) } else if cfg.Type == GIT_SOURCE { - git_sources = append(git_sources, cfg.URL) + if strings.HasSuffix(cfg.URL, "*") { + git_sources = append(git_sources, cfg.URL) + } else { + git_sources = append(git_sources, cfg.URL+"*") + } slog.Info("Found Git source", slog.String("url", cfg.URL)) } } From bc38b79c9a185d589e16f38f634f878d2995c029 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Fri, 5 Sep 2025 22:38:06 +0100 Subject: [PATCH 6/7] Convert URLs to expected format --- go/extractor/util/registryproxy.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 0ba49cb55d65..3909f9e5cf17 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "log/slog" + "net/url" "os" "os/exec" "strings" @@ -103,12 +104,14 @@ func getEnvVars() []string { goproxy_servers = append(goproxy_servers, cfg.URL) slog.Info("Found GOPROXY server", slog.String("url", cfg.URL)) } else if cfg.Type == GIT_SOURCE { - if strings.HasSuffix(cfg.URL, "*") { - git_sources = append(git_sources, cfg.URL) + parsed, err := url.Parse(cfg.URL) + if err == nil && parsed.Hostname() != "" { + git_source := parsed.Hostname() + parsed.Path + "*" + git_sources = append(git_sources, git_source) + slog.Info("Found Git source", slog.String("source", git_source)) } else { - git_sources = append(git_sources, cfg.URL+"*") + slog.Warn("Not a valid URL for Git source", slog.String("url", cfg.URL)) } - slog.Info("Found Git source", slog.String("url", cfg.URL)) } } From 8c8499229d3dd32440a39909b069235dbbd21e83 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Thu, 4 Sep 2025 18:05:38 +0100 Subject: [PATCH 7/7] Configure `git` to use the certificate, if needed --- go/extractor/util/registryproxy.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 3909f9e5cf17..1f20832e8d81 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -129,6 +129,18 @@ func getEnvVars() []string { if len(git_sources) > 0 { goprivate = append(goprivate, git_sources...) + + if proxy_cert_file != "" { + slog.Info("Configuring `git` to use proxy certificate", slog.String("path", proxy_cert_file)) + cmd := exec.Command("git", "config", "--global", "http.sslCAInfo", proxy_cert_file) + + out, cmdErr := cmd.CombinedOutput() + slog.Info(string(out)) + + if cmdErr != nil { + slog.Error("Failed to configure `git` to accept the certificate file", slog.String("error", cmdErr.Error())) + } + } } result = append(result, fmt.Sprintf("GOPRIVATE=%s", strings.Join(goprivate, ",")))