Skip to content

Commit

Permalink
fix(api): during promotion, get build module in parrallel (#6466)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sguiheux
sguiheux committed Feb 28, 2023
1 parent 35cb794 commit 137d7dd
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 79 deletions.
182 changes: 110 additions & 72 deletions contrib/integrations/artifactory/artifactory.go
View file
Expand Up @@ -7,6 +7,7 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"time"

buildinfo "github.com/jfrog/build-info-go/entities"
Expand Down Expand Up @@ -257,100 +258,138 @@ func PrepareBuildInfo(ctx context.Context, artiClient artifact_manager.ArtifactM
return buildInfoRequest, nil
}

func computeBuildInfoModules(ctx context.Context, client artifact_manager.ArtifactManager, execContext executionContext, runResults []sdk.WorkflowRunResult) ([]buildinfo.Module, error) {
func computeBuildInfoModules(ctx context.Context, artiClient artifact_manager.ArtifactManager, execContext executionContext, runResults []sdk.WorkflowRunResult) ([]buildinfo.Module, error) {
ctx, end := telemetry.Span(ctx, "artifactory.computeBuildInfoModules")
defer end()
modules := make([]buildinfo.Module, 0)
for _, r := range runResults {
ctx, endc := telemetry.Span(ctx, "artifactory.PrepareBuildInfo", telemetry.Tag("runResult.Type", r.Type))
if r.Type != sdk.WorkflowRunResultTypeArtifactManager {
endc()
continue
}

var currentMaturity string
if r.DataSync != nil {
latestPromotion := r.DataSync.LatestPromotionOrRelease()
if latestPromotion != nil {
currentMaturity = latestPromotion.ToMaturity
}
}
if currentMaturity == "" {
currentMaturity = execContext.defaultLowMaturitySuffix
}

data, err := r.GetArtifactManager()
if err != nil {
endc()
return nil, err
}
var moduleExists bool
mod := buildinfo.Module{
Id: fmt.Sprintf("%s:%s", data.RepoType, data.Name),
Artifacts: make([]buildinfo.Artifact, 0, len(runResults)),
Dependencies: nil,
}
for _, m := range modules {
if m.Id == mod.Id {
moduleExists = true
endc()
break
}
}
if moduleExists {
endc()
continue
}
switch data.RepoType {
case "docker":
mod.Type = buildinfo.Docker
props := make(map[string]string)
parsedUrl, err := url.Parse(client.GetURL())
var wg sync.WaitGroup
results := make(chan buildinfo.Module, len(runResults))
chanError := make(chan error, len(runResults))

for i := range runResults {
wg.Add(1)
runResult := runResults[i]
goRoutines := &sdk.GoRoutines{}
goRoutines.Exec(ctx, "goroutine-compute-build-info-module-"+runResult.ID, func(ctx context.Context) {
defer wg.Done()
module, err := prepareBuildInfo(ctx, artiClient, runResult, execContext)
if err != nil {
endc()
return nil, sdk.WrapError(err, "unable to parse artifactory url [%s]: %v", client.GetURL())
chanError <- err
}
urlArtifactory := parsedUrl.Host
if parsedUrl.Port() != "" {
urlArtifactory += ":" + parsedUrl.Port()
if module != nil {
results <- *module
}
props["docker.image.tag"] = fmt.Sprintf("%s.%s/%s", data.RepoName, urlArtifactory, data.Name)
mod.Properties = props
}
})
}
wg.Wait()
close(chanError)
close(results)

files, err := retrieveModulesFiles(ctx, client, data.RepoName, data.Path)
if err != nil {
endc()
return nil, err
}
for e := range chanError {
log.ErrorWithStackTrace(ctx, e)
return nil, sdk.WrapError(sdk.ErrUnknownError, "unable to compute build info module")
}

props := utils.NewProperties()
props.AddProperty("build.name", fmt.Sprintf("%s/%s/%s", execContext.buildInfo, execContext.projectKey, execContext.workflowName))
props.AddProperty("build.number", execContext.version)
props.AddProperty("build.timestamp", strconv.FormatInt(time.Now().Unix(), 10))
for b := range results {
modules = append(modules, b)
}
return modules, nil
}

if err := SetPropertiesRecursive(ctx, client, data.RepoName, currentMaturity, files, props); err != nil {
endc()
return nil, err
func prepareBuildInfo(ctx context.Context, client artifact_manager.ArtifactManager, r sdk.WorkflowRunResult, execContext executionContext) (*buildinfo.Module, error) {
ctx, endc := telemetry.Span(ctx, "artifactory.PrepareBuildInfo", telemetry.Tag("runResult.Type", r.Type))
if r.Type != sdk.WorkflowRunResultTypeArtifactManager {
endc()
return nil, nil
}

var currentMaturity string
if r.DataSync != nil {
latestPromotion := r.DataSync.LatestPromotionOrRelease()
if latestPromotion != nil {
currentMaturity = latestPromotion.ToMaturity
}
}
if currentMaturity == "" {
currentMaturity = execContext.defaultLowMaturitySuffix
}

artifacts, err := retrieveModulesArtifacts(ctx, client, files)
data, err := r.GetArtifactManager()
if err != nil {
endc()
return nil, err
}
mod := buildinfo.Module{
Id: fmt.Sprintf("%s:%s", data.RepoType, data.Name),
Artifacts: make([]buildinfo.Artifact, 0),
Dependencies: nil,
}
switch data.RepoType {
case "docker":
mod.Type = buildinfo.Docker
props := make(map[string]string)
parsedUrl, err := url.Parse(client.GetURL())
if err != nil {
endc()
return nil, err
return nil, sdk.WrapError(err, "unable to parse artifactory url [%s]", client.GetURL())
}
mod.Artifacts = artifacts
modules = append(modules, mod)
urlArtifactory := parsedUrl.Host
if parsedUrl.Port() != "" {
urlArtifactory += ":" + parsedUrl.Port()
}
props["docker.image.tag"] = fmt.Sprintf("%s.%s/%s", data.RepoName, urlArtifactory, data.Name)
mod.Properties = props
}

files, err := retrieveModulesFiles(ctx, client, data.RepoName, data.Path)
if err != nil {
endc()
return nil, err
}

return modules, nil
props := utils.NewProperties()
props.AddProperty("build.name", fmt.Sprintf("%s/%s/%s", execContext.buildInfo, execContext.projectKey, execContext.workflowName))
props.AddProperty("build.number", execContext.version)
props.AddProperty("build.timestamp", strconv.FormatInt(time.Now().Unix(), 10))

if err := SetPropertiesRecursive(ctx, client, data.RepoName, currentMaturity, files, props); err != nil {
endc()
return nil, err
}

artifacts, err := retrieveModulesArtifacts(ctx, client, files)
if err != nil {
endc()
return nil, err
}
mod.Artifacts = artifacts
endc()
return &mod, nil
}

func retrieveModulesFiles(ctx context.Context, client artifact_manager.ArtifactManager, repoName string, path string) ([]sdk.FileInfo, error) {
ctx, end := telemetry.Span(ctx, "workflow.retrieveModulesFiles")
defer end()
log.Debug(ctx, "retrieve:ModulesFiles repoName:%s path:%s", repoName, path)
_, endc := telemetry.Span(ctx, "artifactoryClient.GetFileInfo", telemetry.Tag("path", path), telemetry.Tag("repoName", repoName))
fileInfo, err := client.GetFileInfo(repoName, path)
endc()
if err != nil {
return nil, err
}

// If it can be downloaded, it's a file
if fileInfo.DownloadURI != "" {
return []sdk.FileInfo{fileInfo}, nil
}
return retrieveModulesFilesFromFolder(ctx, client, repoName, path)
}

func retrieveModulesFilesFromFolder(ctx context.Context, client artifact_manager.ArtifactManager, repoName string, path string) ([]sdk.FileInfo, error) {
ctx, end := telemetry.Span(ctx, "workflow.retrieveModulesFilesFromFolder")
defer end()
log.Debug(ctx, "retrieve:retrieveModulesFilesFromFolder repoName:%s path:%s", repoName, path)
_, endc := telemetry.Span(ctx, "artifactoryClient.GetFolderInfo", telemetry.Tag("path", path), telemetry.Tag("repoName", repoName))
folderInfo, err := client.GetFolderInfo(repoName, path)
endc()
Expand All @@ -359,10 +398,9 @@ func retrieveModulesFiles(ctx context.Context, client artifact_manager.ArtifactM
}

files := make([]sdk.FileInfo, 0)

for _, c := range folderInfo.Children {
if c.Folder {
childrenFiles, err := retrieveModulesFiles(ctx, client, repoName, fmt.Sprintf("%s%s", path, c.Uri))
childrenFiles, err := retrieveModulesFilesFromFolder(ctx, client, repoName, fmt.Sprintf("%s%s", path, c.Uri))
if err != nil {
return nil, err
}
Expand Down
20 changes: 13 additions & 7 deletions engine/api/workflow/workflow_run_results.go
View file
Expand Up @@ -588,12 +588,6 @@ func SyncRunResultArtifactManagerByRunID(ctx context.Context, db gorpmapper.SqlE
return handleSyncError(sdk.Errorf("unable to find artifact manager %q token", artifactManagerInteg.ProjectIntegration.Name))
}

// Instanciate artifactory client
artifactClient, err := artifact_manager.NewClient(rtName, rtURL, rtToken)
if err != nil {
return err
}

version := fmt.Sprintf("%d", wr.Number)
if wr.Version != nil {
version = *wr.Version
Expand Down Expand Up @@ -625,7 +619,12 @@ func SyncRunResultArtifactManagerByRunID(ctx context.Context, db gorpmapper.SqlE
nodeRunURL := parameters["cds.ui.pipeline.run"][0]
runURL := nodeRunURL[0:strings.Index(nodeRunURL, "/node/")]

buildInfoRequest, err := art.PrepareBuildInfo(ctx, artifactClient, art.BuildInfoRequest{
artiClient, err := artifact_manager.NewClient(rtName, rtURL, rtToken)
if err != nil {
return err
}

buildInfoRequest, err := art.PrepareBuildInfo(ctx, artiClient, art.BuildInfoRequest{
BuildInfoPrefix: buildInfoPrefix,
ProjectKey: wr.Workflow.ProjectKey,
WorkflowName: wr.Workflow.Name,
Expand All @@ -648,6 +647,13 @@ func SyncRunResultArtifactManagerByRunID(ctx context.Context, db gorpmapper.SqlE

log.Debug(ctx, "artifact manager build info request: %+v", buildInfoRequest)
log.Info(ctx, "Creating Artifactory Build %s %s on project %s...\n", buildInfoRequest.Name, buildInfoRequest.Number, artifactoryProjectKey)

// Instanciate artifactory client
artifactClient, err := artifact_manager.NewClient(rtName, rtURL, rtToken)
if err != nil {
return err
}

ctxDelete, endDelete := telemetry.Span(ctx, "artifactClient.DeleteBuild")
if err := artifactClient.DeleteBuild(artifactoryProjectKey, buildInfoRequest.Name, buildInfoRequest.Number); err != nil {
ctx = log.ContextWithStackTrace(ctxDelete, err)
Expand Down

0 comments on commit 137d7dd

Please sign in to comment.