From 185f44d507d5ba997486ec405f002dfb3995e961 Mon Sep 17 00:00:00 2001 From: Helge Bruegner Date: Sat, 24 Feb 2024 22:18:24 +0100 Subject: [PATCH 1/4] add a first draft for sp auth --- .vscode/launch.json | 15 ++++ azure-devops-client/main.go | 165 +++++++++++++++++++++++++++++------- config/opts.go | 5 +- go.mod | 2 +- main.go | 9 +- 5 files changed, 161 insertions(+), 35 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..88254df --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}", + } + ] +} \ No newline at end of file diff --git a/azure-devops-client/main.go b/azure-devops-client/main.go index 704f959..8463b7a 100644 --- a/azure-devops-client/main.go +++ b/azure-devops-client/main.go @@ -1,6 +1,7 @@ package AzureDevopsClient import ( + "encoding/json" "errors" "fmt" "net/url" @@ -21,7 +22,17 @@ type AzureDevopsClient struct { organization *string collection *string - accessToken *string + + // we can either use a PAT token for authentication ... + accessToken *string + + // ... or client id and secret + tenantId *string + clientId *string + clientSecret *string + + entraIdToken *EntraIdToken + entraIdTokenLastRefreshed int64 HostUrl *string @@ -48,6 +59,13 @@ type AzureDevopsClient struct { } } +type EntraIdToken struct { + token_type *string + expires_in *int64 + ext_expires_in *int64 + access_token *string +} + func NewAzureDevopsClient() *AzureDevopsClient { c := AzureDevopsClient{} c.Init() @@ -62,6 +80,8 @@ func (c *AzureDevopsClient) Init() { c.SetRetries(3) c.SetConcurrency(10) + c.entraIdTokenLastRefreshed = 0 + c.LimitBuildsPerProject = 100 c.LimitBuildsPerDefinition = 10 c.LimitReleasesPerDefinition = 100 @@ -115,46 +135,131 @@ func (c *AzureDevopsClient) SetAccessToken(token string) { c.accessToken = &token } +func (c *AzureDevopsClient) SetTenantId(tenantId string) { + c.tenantId = &tenantId +} + +func (c *AzureDevopsClient) SetClientId(clientId string) { + c.clientId = &clientId +} + +func (c *AzureDevopsClient) SetClientSecret(clientSecret string) { + c.clientSecret = &clientSecret +} + +func (c *AzureDevopsClient) SupportsPatAuthentication() bool { + return c.accessToken != nil +} + +func (c *AzureDevopsClient) SupportsServicePrincipalAuthentication() bool { + return c.tenantId != nil && c.clientId != nil && c.clientSecret != nil +} + +func (c *AzureDevopsClient) HasExpiredEntraIdAccessToken() bool { + var currentUnix = time.Now().Unix() + + // subtract 60 seconds of offset (should be enough time to use fire all requests) + return (c.entraIdToken == nil || currentUnix >= c.entraIdTokenLastRefreshed+*c.entraIdToken.expires_in-60) +} + +func (c *AzureDevopsClient) RefreshEntraIdAccessToken() (string, error) { + var restClient = resty.New() + + restClient.SetBaseURL(fmt.Sprintf("https://login.microsoftonline.com/%v/oauth2/v2.0/token", *c.tenantId)) + + restClient.SetFormData(map[string]string{ + "client_id": *c.clientId, + "client_secret": *c.clientSecret, + "grant_type": "client_credentials", + "scope": "499b84ac-1321-427f-aa17-267ca6975798", // the scope is always the same for Azure DevOps + }) + + restClient.SetHeader("Content-Type", "application/x-www-form-urlencoded") + restClient.SetHeader("Accept", "application/json") + restClient.SetRetryCount(c.RequestRetries) + + var response, err = restClient.R().Post("") + + if err != nil { + return "", err + } + + err = json.Unmarshal(response.Body(), &c.entraIdToken) + + if err != nil { + return "", err + } + + c.entraIdTokenLastRefreshed = time.Now().Unix() + + return *c.entraIdToken.access_token, nil +} + func (c *AzureDevopsClient) rest() *resty.Client { + var client, err = c.restWithAuthentication("dev.azure.com") + + if err != nil { + // TODO handle error! + } + + return client +} + +func (c *AzureDevopsClient) restVsrm() *resty.Client { + var client, err = c.restWithAuthentication("vsrm.dev.azure.com") + + if err != nil { + // TODO handle error! + } + + return client +} + +func (c *AzureDevopsClient) restWithAuthentication(domain string) (*resty.Client, error) { if c.restClient == nil { - c.restClient = resty.New() - if c.HostUrl != nil { - c.restClient.SetBaseURL(*c.HostUrl + "/" + *c.organization + "/") - } else { - c.restClient.SetBaseURL(fmt.Sprintf("https://dev.azure.com/%v/", *c.organization)) - } - c.restClient.SetHeader("Accept", "application/json") + c.restClient = c.restWithoutToken(domain) + } + + if c.SupportsPatAuthentication() { c.restClient.SetBasicAuth("", *c.accessToken) - c.restClient.SetRetryCount(c.RequestRetries) - if c.delayUntil != nil { - c.restClient.OnBeforeRequest(c.restOnBeforeRequestDelay) - } else { - c.restClient.OnBeforeRequest(c.restOnBeforeRequest) - } + } else if c.SupportsServicePrincipalAuthentication() { + if c.HasExpiredEntraIdAccessToken() { + var accessToken, err = c.RefreshEntraIdAccessToken() - c.restClient.OnAfterResponse(c.restOnAfterResponse) + if err != nil { + return nil, err + } + c.restClient.SetBasicAuth("", accessToken) + } + } else { + return nil, errors.New("no valid authentication method provided") } - return c.restClient + return c.restClient, nil } -func (c *AzureDevopsClient) restVsrm() *resty.Client { - if c.restClientVsrm == nil { - c.restClientVsrm = resty.New() - if c.HostUrl != nil { - c.restClientVsrm.SetBaseURL(*c.HostUrl + "/" + *c.organization + "/") - } else { - c.restClientVsrm.SetBaseURL(fmt.Sprintf("https://vsrm.dev.azure.com/%v/", *c.organization)) - } - c.restClientVsrm.SetHeader("Accept", "application/json") - c.restClientVsrm.SetBasicAuth("", *c.accessToken) - c.restClientVsrm.SetRetryCount(c.RequestRetries) - c.restClientVsrm.OnBeforeRequest(c.restOnBeforeRequest) - c.restClientVsrm.OnAfterResponse(c.restOnAfterResponse) +func (c *AzureDevopsClient) restWithoutToken(domain string) *resty.Client { + var restClient = resty.New() + + if c.HostUrl != nil { + restClient.SetBaseURL(*c.HostUrl + "/" + *c.organization + "/") + } else { + restClient.SetBaseURL(fmt.Sprintf("https://%v/%v/", domain, *c.organization)) + } + + restClient.SetHeader("Accept", "application/json") + restClient.SetRetryCount(c.RequestRetries) + + if c.delayUntil != nil { + restClient.OnBeforeRequest(c.restOnBeforeRequestDelay) + } else { + restClient.OnBeforeRequest(c.restOnBeforeRequest) } - return c.restClientVsrm + restClient.OnAfterResponse(c.restOnAfterResponse) + + return restClient } func (c *AzureDevopsClient) concurrencyLock() { diff --git a/config/opts.go b/config/opts.go index 9929d81..7084576 100644 --- a/config/opts.go +++ b/config/opts.go @@ -38,9 +38,12 @@ type ( // azure settings AzureDevops struct { - Url *string `long:"azuredevops.url" env:"AZURE_DEVOPS_URL" description:"Azure DevOps url (empty if hosted by microsoft)"` + Url *string `long:"azuredevops.url" env:"AZURE_DEVOPS_URL" description:"Azure DevOps URL (empty if hosted by Microsoft)"` AccessToken string `long:"azuredevops.access-token" env:"AZURE_DEVOPS_ACCESS_TOKEN" description:"Azure DevOps access token" json:"-"` AccessTokenFile *string `long:"azuredevops.access-token-file" env:"AZURE_DEVOPS_ACCESS_TOKEN_FILE" description:"Azure DevOps access token (from file)"` + TenantId string `long:"azuredevops.tenant-id" env:"AZURE_TENANT_ID" description:"Azure tenant ID for Service Principal authentication" json:"-"` + ClientId string `long:"azuredevops.client-id" env:"AZURE_CLIENT_ID" description:"Client ID for Service Principal authentication" json:"-"` + ClientSecret string `long:"azuredevops.client-secret" env:"AZURE_CLIENT_SECRET" description:"Client secret for Service Principal authentication" json:"-"` Organisation string `long:"azuredevops.organisation" env:"AZURE_DEVOPS_ORGANISATION" description:"Azure DevOps organization" required:"true"` ApiVersion string `long:"azuredevops.apiversion" env:"AZURE_DEVOPS_APIVERSION" description:"Azure DevOps API version" default:"5.1"` diff --git a/go.mod b/go.mod index db85716..108e290 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/webdevops/azure-devops-exporter -go 1.22 +go 1.22.0 require ( github.com/go-resty/resty/v2 v2.11.0 diff --git a/main.go b/main.go index 31dccea..4b24a86 100644 --- a/main.go +++ b/main.go @@ -36,8 +36,8 @@ var ( ) func main() { - initArgparser() initLogger() + initArgparser() logger.Infof("starting azure-devops-exporter v%s (%s; %s; by %v)", gitTag, gitCommit, runtime.Version(), Author) logger.Info(string(opts.GetJson())) @@ -82,8 +82,8 @@ func initArgparser() { } } - if len(opts.AzureDevops.AccessToken) == 0 { - logger.Fatalf("no Azure DevOps access token specified") + if len(opts.AzureDevops.AccessToken) == 0 || (len(opts.AzureDevops.TenantId) == 0 && len(opts.AzureDevops.ClientId) == 0 && len(opts.AzureDevops.ClientSecret) == 0) { + logger.Fatalf("neither an Azure DevOps PAT token nor client credentials (tenant ID, client ID, client secret) for service principal authentication have been provided") } // ensure query paths and projects are splitted by '@' @@ -160,6 +160,9 @@ func initAzureDevOpsConnection() { AzureDevopsClient.SetOrganization(opts.AzureDevops.Organisation) AzureDevopsClient.SetAccessToken(opts.AzureDevops.AccessToken) + AzureDevopsClient.SetTenantId(opts.AzureDevops.TenantId) + AzureDevopsClient.SetClientId(opts.AzureDevops.ClientId) + AzureDevopsClient.SetClientSecret(opts.AzureDevops.ClientSecret) AzureDevopsClient.SetApiVersion(opts.AzureDevops.ApiVersion) AzureDevopsClient.SetConcurrency(opts.Request.ConcurrencyLimit) AzureDevopsClient.SetRetries(opts.Request.Retries) From 6fca85036064330621e0b7107b32bd1604d22d40 Mon Sep 17 00:00:00 2001 From: Helge Bruegner Date: Sun, 25 Feb 2024 16:47:18 +0100 Subject: [PATCH 2/4] fix a few bugs in the json serialization and the token resolution --- azure-devops-client/main.go | 58 ++++++++++++++++++++++++++++--------- main.go | 4 +-- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/azure-devops-client/main.go b/azure-devops-client/main.go index 8463b7a..ad9f2d0 100644 --- a/azure-devops-client/main.go +++ b/azure-devops-client/main.go @@ -12,9 +12,12 @@ import ( resty "github.com/go-resty/resty/v2" "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" ) type AzureDevopsClient struct { + logger *zap.SugaredLogger + // RequestCount has to be the first words // in order to be 64-aligned on 32-bit architectures. RequestCount uint64 @@ -60,14 +63,21 @@ type AzureDevopsClient struct { } type EntraIdToken struct { - token_type *string - expires_in *int64 - ext_expires_in *int64 - access_token *string + TokenType *string `json:"token_type"` + ExpiresIn *int64 `json:"expires_in"` + ExtExpiresIn *int64 `json:"ext_expires_in"` + AccessToken *string `json:"access_token"` +} + +type EntraIdErrorResponse struct { + Error *string `json:"error"` + ErrorDescription *string `json:"error_description"` } -func NewAzureDevopsClient() *AzureDevopsClient { - c := AzureDevopsClient{} +func NewAzureDevopsClient(logger *zap.SugaredLogger) *AzureDevopsClient { + c := AzureDevopsClient{ + logger: logger, + } c.Init() return &c @@ -148,18 +158,20 @@ func (c *AzureDevopsClient) SetClientSecret(clientSecret string) { } func (c *AzureDevopsClient) SupportsPatAuthentication() bool { - return c.accessToken != nil + return c.accessToken != nil && len(*c.accessToken) > 0 } func (c *AzureDevopsClient) SupportsServicePrincipalAuthentication() bool { - return c.tenantId != nil && c.clientId != nil && c.clientSecret != nil + return c.tenantId != nil && len(*c.tenantId) > 0 && + c.clientId != nil && len(*c.clientId) > 0 && + c.clientSecret != nil && len(*c.clientSecret) > 0 } func (c *AzureDevopsClient) HasExpiredEntraIdAccessToken() bool { var currentUnix = time.Now().Unix() // subtract 60 seconds of offset (should be enough time to use fire all requests) - return (c.entraIdToken == nil || currentUnix >= c.entraIdTokenLastRefreshed+*c.entraIdToken.expires_in-60) + return (c.entraIdToken == nil || currentUnix >= c.entraIdTokenLastRefreshed+*c.entraIdToken.ExpiresIn-60) } func (c *AzureDevopsClient) RefreshEntraIdAccessToken() (string, error) { @@ -171,7 +183,7 @@ func (c *AzureDevopsClient) RefreshEntraIdAccessToken() (string, error) { "client_id": *c.clientId, "client_secret": *c.clientSecret, "grant_type": "client_credentials", - "scope": "499b84ac-1321-427f-aa17-267ca6975798", // the scope is always the same for Azure DevOps + "scope": "499b84ac-1321-427f-aa17-267ca6975798/.default", // the scope is always the same for Azure DevOps }) restClient.SetHeader("Content-Type", "application/x-www-form-urlencoded") @@ -184,22 +196,40 @@ func (c *AzureDevopsClient) RefreshEntraIdAccessToken() (string, error) { return "", err } - err = json.Unmarshal(response.Body(), &c.entraIdToken) + var responseBody = response.Body() + + var errorResponse *EntraIdErrorResponse + + err = json.Unmarshal(responseBody, &errorResponse) if err != nil { return "", err } + if errorResponse.Error != nil && len(*errorResponse.Error) > 0 { + return "", fmt.Errorf("could not request a token, error: %v %v", *errorResponse.Error, *errorResponse.ErrorDescription) + } + + err = json.Unmarshal(responseBody, &c.entraIdToken) + + if err != nil { + return "", err + } + + if c.entraIdToken == nil || c.entraIdToken.AccessToken == nil { + return "", errors.New("could not request an access token") + } + c.entraIdTokenLastRefreshed = time.Now().Unix() - return *c.entraIdToken.access_token, nil + return *c.entraIdToken.AccessToken, nil } func (c *AzureDevopsClient) rest() *resty.Client { var client, err = c.restWithAuthentication("dev.azure.com") if err != nil { - // TODO handle error! + c.logger.Fatalf("could not create a rest client: %v", err) } return client @@ -209,7 +239,7 @@ func (c *AzureDevopsClient) restVsrm() *resty.Client { var client, err = c.restWithAuthentication("vsrm.dev.azure.com") if err != nil { - // TODO handle error! + c.logger.Fatalf("could not create a rest client: %v", err) } return client diff --git a/main.go b/main.go index 4b24a86..e52c66c 100644 --- a/main.go +++ b/main.go @@ -82,7 +82,7 @@ func initArgparser() { } } - if len(opts.AzureDevops.AccessToken) == 0 || (len(opts.AzureDevops.TenantId) == 0 && len(opts.AzureDevops.ClientId) == 0 && len(opts.AzureDevops.ClientSecret) == 0) { + if len(opts.AzureDevops.AccessToken) == 0 && (len(opts.AzureDevops.TenantId) == 0 || len(opts.AzureDevops.ClientId) == 0 || len(opts.AzureDevops.ClientSecret) == 0) { logger.Fatalf("neither an Azure DevOps PAT token nor client credentials (tenant ID, client ID, client secret) for service principal authentication have been provided") } @@ -148,7 +148,7 @@ func initArgparser() { // Init and build Azure authorzier func initAzureDevOpsConnection() { - AzureDevopsClient = AzureDevops.NewAzureDevopsClient() + AzureDevopsClient = AzureDevops.NewAzureDevopsClient(logger) if opts.AzureDevops.Url != nil { AzureDevopsClient.HostUrl = opts.AzureDevops.Url } From c491442197b2d777f24ab429877a6026560c17c1 Mon Sep 17 00:00:00 2001 From: Helge Bruegner Date: Mon, 4 Mar 2024 22:57:31 +0100 Subject: [PATCH 3/4] update the readme --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 12e1911..36898fa 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,12 @@ Application Options: --scrape.time.query= Scrape time for query results (time.duration) [$SCRAPE_TIME_QUERY] --scrape.time.live= Scrape time for live metrics (time.duration) (default: 30s) [$SCRAPE_TIME_LIVE] --stats.summary.maxage= Stats Summary metrics max age (time.duration) [$STATS_SUMMARY_MAX_AGE] - --azuredevops.url= Azure DevOps url (empty if hosted by microsoft) [$AZURE_DEVOPS_URL] + --azuredevops.url= Azure DevOps URL (empty if hosted by Microsoft) [$AZURE_DEVOPS_URL] --azuredevops.access-token= Azure DevOps access token [$AZURE_DEVOPS_ACCESS_TOKEN] --azuredevops.access-token-file= Azure DevOps access token (from file) [$AZURE_DEVOPS_ACCESS_TOKEN_FILE] + --azuredevops.tenant-id= Azure tenant ID for Service Principal authentication [$AZURE_TENANT_ID] + --azuredevops.client-id= Client ID for Service Principal authentication [$AZURE_CLIENT_ID] + --azuredevops.client-secret= Client secret for Service Principal authentication [$AZURE_CLIENT_SECRET] --azuredevops.organisation= Azure DevOps organization [$AZURE_DEVOPS_ORGANISATION] --azuredevops.apiversion= Azure DevOps API version (default: 5.1) [$AZURE_DEVOPS_APIVERSION] --azuredevops.agentpool= Enable scrape metrics for agent pool (IDs) [$AZURE_DEVOPS_AGENTPOOL] From a04bfd4e7f543b148fee3346416efe053a040633 Mon Sep 17 00:00:00 2001 From: Markus Blaschke Date: Fri, 29 Mar 2024 00:29:13 +0100 Subject: [PATCH 4/4] Delete .vscode/launch.json --- .vscode/launch.json | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 88254df..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Launch Package", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}", - } - ] -} \ No newline at end of file