Skip to content

Commit

Permalink
Do not try auto detect Python package if no Python wheel tasks defined (
Browse files Browse the repository at this point in the history
#674)

## Changes
Fixes #673 

It also includes a change for `libraries` from #635 to get the list of
wheel tasks
  • Loading branch information
andrewnester committed Aug 17, 2023
1 parent 35e8ed3 commit 4694832
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 11 deletions.
7 changes: 7 additions & 0 deletions bundle/artifacts/whl/autodetect.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ import (

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/libraries"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/log"
)

type detectPkg struct {
Expand All @@ -25,6 +27,11 @@ func (m *detectPkg) Name() string {
}

func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) error {
wheelTasks := libraries.FindAllWheelTasks(b)
if len(wheelTasks) == 0 {
log.Infof(ctx, "No wheel tasks in databricks.yml config, skipping auto detect")
return nil
}
cmdio.LogString(ctx, "artifacts.whl.AutoDetect: Detecting Python wheel project...")

// checking if there is setup.py in the bundle root
Expand Down
44 changes: 33 additions & 11 deletions bundle/libraries/libraries.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,24 +25,46 @@ func (a *match) Name() string {
}

func (a *match) Apply(ctx context.Context, b *bundle.Bundle) error {
tasks := findAllTasks(b)
for _, task := range tasks {
if isMissingRequiredLibraries(task) {
return fmt.Errorf("task '%s' is missing required libraries. Please include your package code in task libraries block", task.TaskKey)
}
for j := range task.Libraries {
lib := &task.Libraries[j]
err := findArtifactsAndMarkForUpload(ctx, lib, b)
if err != nil {
return err
}
}
}
return nil
}

func findAllTasks(b *bundle.Bundle) []*jobs.Task {
r := b.Config.Resources
result := make([]*jobs.Task, 0)
for k := range b.Config.Resources.Jobs {
tasks := r.Jobs[k].JobSettings.Tasks
for i := range tasks {
task := &tasks[i]
if isMissingRequiredLibraries(task) {
return fmt.Errorf("task '%s' is missing required libraries. Please include your package code in task libraries block", task.TaskKey)
}
for j := range task.Libraries {
lib := &task.Libraries[j]
err := findArtifactsAndMarkForUpload(ctx, lib, b)
if err != nil {
return err
}
}
result = append(result, task)
}
}
return nil

return result
}

func FindAllWheelTasks(b *bundle.Bundle) []*jobs.Task {
tasks := findAllTasks(b)
wheelTasks := make([]*jobs.Task, 0)
for _, task := range tasks {
if task.PythonWheelTask != nil {
wheelTasks = append(wheelTasks, task)
}
}

return wheelTasks
}

func isMissingRequiredLibraries(task *jobs.Task) bool {
Expand Down

0 comments on commit 4694832

Please sign in to comment.