feat(cdsctl): allow to set CDN url as env variable (#6322)

cli/cdsctl/config.go

@@ -125,6 +125,7 @@ func loadConfig(cmd *cobra.Command) (string, *cdsclient.Config, error) {
 
 	config := &cdsclient.Config{
 		Host:                              cdsctx.Host,
+		CDNHost:                           os.Getenv("CDS_CDN_URL"),
 		SessionToken:                      cdsctx.Session,
 		BuitinConsumerAuthenticationToken: cdsctx.Token,
 		Verbose:                           verbose,

cli/cdsctl/main.go

@@ -129,6 +129,10 @@ Advanced usages:
 
 	CDS_API_URL="https://instance.cds.api" CDS_SESSION_TOKEN="yourtoken" CDS_HTTP_MAX_RETRY=10 cdsctl [command]
 
+* you can override CDN url if needed using the variable CDS_CDN_URL:
+
+  CDS_API_URL="https://instance.cds.api" CDS_TOKEN="yourtoken" CDS_CDN_URL="https://instance.cds.cdn" cdsctl [command]
+
 `,
 }
 

cli/cdsctl/workflow_artifact.go

@@ -46,7 +46,7 @@ var workflowArtifactDownloadCmd = cli.Command{
 		},
 		{
 			Name:    "cdn-url",
-			Usage:   "overwrite cdn url",
+			Usage:   "overwrite cdn url (deprecated)",
 			Default: "",
 		},
 	},
@@ -59,12 +59,13 @@ func workflowArtifactDownloadRun(v cli.Values) error {
 	}
 
 	cdnURL := v.GetString("cdn-url")
-	if cdnURL == "" {
-		confCDN, err := client.ConfigCDN()
-		if err != nil {
-			return err
-		}
-		cdnURL = confCDN.HTTPURL
+	if cdnURL != "" {
+		fmt.Printf("Flag cdn-url is deprecated, use CDS_CDN_URL env variable instead\n")
+	}
+
+	cdnURL, err = client.CDNURL()
+	if err != nil {
+		return err
 	}
 
 	// Search in result

cli/cdsctl/workflow_log.go

@@ -286,13 +286,6 @@ func workflowLogDownloadRun(v cli.Values) error {
 		}
 	}
 
-	feature, err := client.FeatureEnabled(sdk.FeatureCDNJobLogs, map[string]string{
-		"project_key": projectKey,
-	})
-	if err != nil {
-		return err
-	}
-
 	var ok bool
 	for _, log := range logs {
 		if reg != nil && !reg.MatchString(log.getFilename()) {
@@ -301,15 +294,13 @@ func workflowLogDownloadRun(v cli.Values) error {
 
 		// If cdn logs is enabled for current project, first check if logs can be downloaded from it
 		var link *sdk.CDNLogLink
-		if !feature.Exists || feature.Enabled {
-			if log.detailType == workflowLogDetailTypeService {
-				link, err = client.WorkflowNodeRunJobServiceLink(context.Background(), projectKey, workflowName, log.runID, log.jobID, log.serviceName)
-			} else {
-				link, err = client.WorkflowNodeRunJobStepLink(context.Background(), projectKey, workflowName, log.runID, log.jobID, log.stepOrder)
-			}
-			if err != nil {
-				return err
-			}
+		if log.detailType == workflowLogDetailTypeService {
+			link, err = client.WorkflowNodeRunJobServiceLink(context.Background(), projectKey, workflowName, log.runID, log.jobID, log.serviceName)
+		} else {
+			link, err = client.WorkflowNodeRunJobStepLink(context.Background(), projectKey, workflowName, log.runID, log.jobID, log.stepOrder)
+		}
+		if err != nil {
+			return err
 		}
 
 		data, err := client.WorkflowLogDownload(context.Background(), *link)
@@ -353,6 +344,11 @@ func workflowLogStreamRun(v cli.Values) error {
 	projectKey := v.GetString(_ProjectKey)
 	workflowName := v.GetString(_WorkflowName)
 
+	cdnURL, err := client.CDNURL()
+	if err != nil {
+		return err
+	}
+
 	runNumber, err := workflowLogSearchNumber(v)
 	if err != nil {
 		return err
@@ -423,7 +419,7 @@ func workflowLogStreamRun(v cli.Values) error {
 	goRoutines := sdk.NewGoRoutines(ctx)
 	goRoutines.Exec(ctx, "WebsocketEventsListenCmd", func(ctx context.Context) {
 		for ctx.Err() == nil {
-			if err := client.RequestWebsocket(ctx, goRoutines, fmt.Sprintf("%s/item/stream", link.CDNURL), chanMessageToSend, chanMsgReceived, chanErrorReceived); err != nil {
+			if err := client.RequestWebsocket(ctx, goRoutines, fmt.Sprintf("%s/item/stream", cdnURL), chanMessageToSend, chanMsgReceived, chanErrorReceived); err != nil {
 				fmt.Printf("Error: %s\n", err)
 			}
 			time.Sleep(1 * time.Second)

cli/cdsctl/workflow_run_interactive.go

@@ -19,13 +19,6 @@ func workflowRunInteractive(v cli.Values, w *sdk.WorkflowRun, baseURL string) er
 	projectKey := v.GetString(_ProjectKey)
 	workflowName := v.GetString(_WorkflowName)
 
-	feature, err := client.FeatureEnabled(sdk.FeatureCDNJobLogs, map[string]string{
-		"project_key": projectKey,
-	})
-	if err != nil {
-		return err
-	}
-
 	for {
 		var errrg error
 		wo, errrg = client.WorkflowRunGet(projectKey, v.GetString(_WorkflowName), w.Number)
@@ -51,23 +44,20 @@ func workflowRunInteractive(v cli.Values, w *sdk.WorkflowRun, baseURL string) er
 
 						jobLine := fmt.Sprintf("%s  %s/%s/%s/%s %s %s \n", status, v.GetString(_WorkflowName), wnr.WorkflowNodeName, stage.Name, job.Job.Action.Name, start, end)
 						if job.Status == sdk.StatusFail {
-							newOutput += fmt.Sprintf(tm.Color(tm.Bold(jobLine), tm.RED))
+							newOutput += tm.Color(tm.Bold(jobLine), tm.RED)
 						} else {
-							newOutput += fmt.Sprintf(tm.Bold(jobLine))
+							newOutput += tm.Bold(jobLine)
 						}
 
 						for _, info := range job.SpawnInfos {
 							newOutput += fmt.Sprintf("\nInformations: %s - %s", info.APITime, info.UserMessage)
 						}
-						newOutput += fmt.Sprintf("\n")
+						newOutput += "\n"
 
 						for _, step := range job.Job.StepStatus {
-							var link *sdk.CDNLogLink
-							if feature.Enabled {
-								link, err = client.WorkflowNodeRunJobStepLink(context.Background(), projectKey, workflowName, wnr.ID, job.ID, int64(step.StepOrder))
-								if err != nil {
-									return err
-								}
+							link, err := client.WorkflowNodeRunJobStepLink(context.Background(), projectKey, workflowName, wnr.ID, job.ID, int64(step.StepOrder))
+							if err != nil {
+								return err
 							}
 
 							buf, err := client.WorkflowLogDownload(context.Background(), *link)

cli/cdsctl/workflow_run_result.go

@@ -65,7 +65,7 @@ func workflowRunResultGet(v cli.Values) error {
 	}
 	resultID := v.GetString("result-id")
 
-	confCDN, err := client.ConfigCDN()
+	cdnURL, err := client.CDNURL()
 	if err != nil {
 		return err
 	}
@@ -165,7 +165,7 @@ func workflowRunResultGet(v cli.Values) error {
 				return cli.NewError("unable to open file %s: %s", fileName, err)
 			}
 			fmt.Printf("Downloading %s...\n", fileName)
-			if err := client.CDNItemDownload(context.Background(), confCDN.HTTPURL, cdnHash, sdk.CDNTypeItemRunResult, md5, f); err != nil {
+			if err := client.CDNItemDownload(context.Background(), cdnURL, cdnHash, sdk.CDNTypeItemRunResult, md5, f); err != nil {
 				_ = f.Close()
 				return err
 			}

engine/api/workflow_run_log.go

@@ -134,11 +134,6 @@ func (api *API) getWorkflowNodeRunJobLogLinkHandler(ctx context.Context, w http.
 		return sdk.NewErrorFrom(err, "invalid node job id")
 	}
 
-	httpURL, err := services.GetCDNPublicHTTPAdress(ctx, api.mustDB())
-	if err != nil {
-		return err
-	}
-
 	nodeRun, err := workflow.LoadNodeRun(api.mustDB(), projectKey, workflowName, nodeRunID, workflow.LoadRunOptions{DisableDetailledNodeRun: true})
 	if err != nil {
 		return sdk.WrapError(err, "cannot find nodeRun %d for workflow %s in project %s", nodeRunID, workflowName, projectKey)
@@ -221,7 +216,6 @@ func (api *API) getWorkflowNodeRunJobLogLinkHandler(ctx context.Context, w http.
 	apiRefHash := strconv.FormatUint(apiRefHashU, 10)
 
 	return service.WriteJSON(w, sdk.CDNLogLink{
-		CDNURL:   httpURL,
 		ItemType: itemType,
 		APIRef:   apiRefHash,
 	}, http.StatusOK)

engine/api/workflow_run_log_test.go

@@ -53,7 +53,6 @@ func Test_getWorkflowNodeRunJobLinkHandler(t *testing.T) {
 
 	var link sdk.CDNLogLink
 	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &link))
-	require.Equal(t, "http://cdn.net:8080", link.CDNURL)
 	require.Equal(t, sdk.CDNTypeItemStepLog, link.ItemType)
 	require.NotEmpty(t, link.APIRef)
 }

engine/cdn/cdn_log_engine.go

@@ -2,11 +2,9 @@ package cdn
 
 import (
 	"context"
-	"fmt"
 	"strings"
 	"time"
 
-	gocache "github.com/patrickmn/go-cache"
 	"github.com/rockbears/log"
 
 	"github.com/ovh/cds/engine/cache"
@@ -180,23 +178,3 @@ func (s *Service) canDequeue(ctx context.Context, jobID string) (string, error)
 	}
 	return jobQueueKey, nil
 }
-
-// Check if storage on CDN is enabled
-func (s *Service) cdnEnabled(ctx context.Context, projectKey string) bool {
-	cacheKey := fmt.Sprintf("cdn-job-logs-enabled-project-%s", projectKey)
-	enabledI, has := runCache.Get(cacheKey)
-	if has {
-		return enabledI.(bool)
-	}
-
-	resp, err := s.Client.FeatureEnabled(sdk.FeatureCDNJobLogs, map[string]string{
-		"project_key": projectKey,
-	})
-	if err != nil {
-		log.Error(ctx, "unable to get job logs feature for project %s: %v", projectKey, err)
-		return false
-	}
-	enabled := !resp.Exists || resp.Enabled
-	runCache.Set(cacheKey, enabled, gocache.DefaultExpiration)
-	return enabled
-}

engine/worker/internal/types.go

@@ -28,6 +28,7 @@ const (
 
 	// CDS API URL
 	CDSApiUrl = "CDS_API_URL"
+	CDSCDNUrl = "CDS_CDN_URL"
 )
 
 type logger struct {
@@ -280,6 +281,7 @@ func (wk *CurrentWorker) Environ() []string {
 	newEnv = append(newEnv, "CDS_KEY=********") //We have to let it here for some legacy reason
 	newEnv = append(newEnv, fmt.Sprintf("%s=%d", WorkerServerPort, wk.HTTPPort()))
 	newEnv = append(newEnv, fmt.Sprintf("%s=%s", CDSApiUrl, wk.cfg.APIEndpoint))
+	newEnv = append(newEnv, fmt.Sprintf("%s=%s", CDSCDNUrl, wk.cfg.CDNEndpoint))
 
 	if wk.currentJob.wJob != nil {
 		data := []byte(wk.currentJob.wJob.Job.Job.Action.Name)

sdk/cdn.go

@@ -117,7 +117,6 @@ type CDNLogLinkData struct {
 }
 
 type CDNLogLink struct {
-	CDNURL   string      `json:"cdn_url,omitempty"`
 	ItemType CDNItemType `json:"item_type"`
 	APIRef   string      `json:"api_ref"`
 }

sdk/cdsclient/client.go

@@ -190,6 +190,17 @@ func (c *client) APIURL() string {
 	return c.config.Host
 }
 
+func (c *client) CDNURL() (string, error) {
+	if c.config.CDNHost == "" {
+		confCDN, err := c.ConfigCDN()
+		if err != nil {
+			return "", err
+		}
+		c.config.CDNHost = confCDN.HTTPURL
+	}
+	return c.config.CDNHost, nil
+}
+
 func (c *client) HTTPClient() *http.Client {
 	return c.httpClient
 }

sdk/cdsclient/client_workflow.go

@@ -227,7 +227,11 @@ func (c *client) WorkflowAccess(ctx context.Context, projectKey string, workflow
 }
 
 func (c *client) WorkflowLogDownload(ctx context.Context, link sdk.CDNLogLink) ([]byte, error) {
-	downloadURL := fmt.Sprintf("%s/item/%s/%s/download", link.CDNURL, link.ItemType, link.APIRef)
+	cdnURL, err := c.CDNURL()
+	if err != nil {
+		return nil, err
+	}
+	downloadURL := fmt.Sprintf("%s/item/%s/%s/download", cdnURL, link.ItemType, link.APIRef)
 	data, _, _, err := c.Request(context.Background(), http.MethodGet, downloadURL, nil, func(req *http.Request) {
 		auth := "Bearer " + c.config.SessionToken
 		req.Header.Add("Authorization", auth)

sdk/cdsclient/config.go

@@ -8,9 +8,10 @@ import (
 	"github.com/ovh/cds/sdk"
 )
 
-//Config is the configuration data used by the cdsclient interface implementation
+// Config is the configuration data used by the cdsclient interface implementation
 type Config struct {
 	Host                              string
+	CDNHost                           string
 	User                              string
 	SessionToken                      string
 	BuitinConsumerAuthenticationToken string
@@ -20,15 +21,15 @@ type Config struct {
 	Mutex                             *sync.Mutex
 }
 
-//ProviderConfig is the configuration data used by the cdsclient ProviderClient interface implementation
+// ProviderConfig is the configuration data used by the cdsclient ProviderClient interface implementation
 type ProviderConfig struct {
 	Host                  string
 	Token                 string
 	RequestSecondsTimeout int
 	InsecureSkipVerifyTLS bool
 }
 
-//ServiceConfig is the configuration data used by the cdsclient interface implementation
+// ServiceConfig is the configuration data used by the cdsclient interface implementation
 type ServiceConfig struct {
 	Host                  string
 	Token                 string

sdk/cdsclient/interface.go

@@ -418,6 +418,7 @@ type Interface interface {
 	ActionClient
 	Admin
 	APIURL() string
+	CDNURL() (string, error)
 	ApplicationClient
 	ConfigUser() (sdk.ConfigUser, error)
 	ConfigCDN() (sdk.CDNConfig, error)

sdk/featureflipping.go

@@ -3,7 +3,6 @@ package sdk
 type FeatureName string
 
 const (
-	FeatureCDNJobLogs   FeatureName = "cdn-job-logs"
 	FeatureMFARequired  FeatureName = "mfa_required"
 	FeaturePurgeName    FeatureName = "workflow-retention-policy"
 	FeaturePurgeMaxRuns FeatureName = "workflow-retention-maxruns"