Skip to content

Commit

Permalink
fixup: s/workers/parallelism
Browse files Browse the repository at this point in the history
Signed-off-by: mikcl <mikesmikes400@gmail.com>
  • Loading branch information
Mikcl committed Nov 29, 2022
1 parent 519a998 commit 2929836
Show file tree
Hide file tree
Showing 7 changed files with 28 additions and 28 deletions.
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -534,8 +534,8 @@ file-metadata:
# SYFT_FILE_METADATA_DIGESTS env var
digests: ["sha256"]

# maximum number of workers used to process the list of package catalogers
workers: 1
# maximum number of workers used to process the list of package catalogers in parallel
parallelism: 1

# cataloging secrets is exposed through the power-user subcommand
secrets:
Expand Down
8 changes: 4 additions & 4 deletions cmd/syft/cli/options/packages.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type PackagesOptions struct {
Exclude []string
Catalogers []string
Name string
Workers int
Parallelism int
}

var _ Interface = (*PackagesOptions)(nil)
Expand Down Expand Up @@ -52,8 +52,8 @@ func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error {
cmd.Flags().StringVarP(&o.Name, "name", "", "",
"set the name of the target being analyzed")

cmd.Flags().IntVarP(&o.Workers, "workers", "w", 1,
"number of workers to use to process the catalogers")
cmd.Flags().IntVarP(&o.Parallelism, "parallelism", "", 1,
"number of workers to use to process the catalogers in parallel")

return bindPackageConfigOptions(cmd.Flags(), v)
}
Expand Down Expand Up @@ -93,7 +93,7 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
return err
}

if err := v.BindPFlag("workers", flags.Lookup("workers")); err != nil {
if err := v.BindPFlag("parallelism", flags.Lookup("parallelism")); err != nil {
return err
}

Expand Down
6 changes: 3 additions & 3 deletions internal/config/application.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ type Application struct {
Attest attest `yaml:"attest" json:"attest" mapstructure:"attest"`
Platform string `yaml:"platform" json:"platform" mapstructure:"platform"`
Name string `yaml:"name" json:"name" mapstructure:"name"`
Workers int `yaml:"workers" json:"workers" mapstructure:"workers"` // -w the number of catalog workers to run in parallel
Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // --parallelism the number of catalog workers to run in parallel
}

func (cfg Application) ToCatalogerConfig() cataloger.Config {
Expand All @@ -66,8 +66,8 @@ func (cfg Application) ToCatalogerConfig() cataloger.Config {
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
Scope: cfg.Package.Cataloger.ScopeOpt,
},
Catalogers: cfg.Catalogers,
Workers: cfg.Workers,
Catalogers: cfg.Catalogers,
Parallelism: cfg.Parallelism,
}
}

Expand Down
2 changes: 1 addition & 1 deletion syft/lib.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Catalog, []
}
}

catalog, relationships, err := cataloger.Catalog(resolver, release, cfg.Workers, catalogers...)
catalog, relationships, err := cataloger.Catalog(resolver, release, cfg.Parallelism, catalogers...)
if err != nil {
return nil, nil, nil, err
}
Expand Down
12 changes: 6 additions & 6 deletions syft/pkg/cataloger/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,8 @@ func runCataloger(cataloger pkg.Cataloger, resolver source.FileResolver) (*Catal
// In order to efficiently retrieve contents from a underlying container image the content fetch requests are
// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
// request.
func Catalog(resolver source.FileResolver, release *linux.Release, workers int, catalogers ...pkg.Cataloger) (*pkg.Catalog, []artifact.Relationship, error) {
//nolint:funlen
func Catalog(resolver source.FileResolver, release *linux.Release, parallelism int, catalogers ...pkg.Cataloger) (*pkg.Catalog, []artifact.Relationship, error) {
catalog := pkg.NewCatalog()
var allRelationships []artifact.Relationship
filesProcessed, packagesDiscovered := newMonitor()
Expand All @@ -104,18 +105,17 @@ func Catalog(resolver source.FileResolver, release *linux.Release, workers int,

nCatalogers := len(catalogers)

// we will always need at least 1
// we do not need more workers than there are `catalogers`.
workers = int(math.Max(1.0, math.Min(float64(nCatalogers), float64(workers))))
log.Debugf("Using workers=%d for catalogs=%d", workers, nCatalogers)
// we do not need more parallelism than there are `catalogers`.
parallelism = int(math.Min(float64(nCatalogers), math.Max(1.0, float64(parallelism))))
log.Debugf("Using parallelism=%d for catalogs=%d", parallelism, nCatalogers)

jobs := make(chan pkg.Cataloger, nCatalogers)
results := make(chan *CatalogResult, nCatalogers)
discoveredPackages := make(chan int64, nCatalogers)

waitGroup := sync.WaitGroup{}

for i := 0; i < workers; i++ {
for i := 0; i < parallelism; i++ {
waitGroup.Add(1)

go func() {
Expand Down
10 changes: 5 additions & 5 deletions syft/pkg/cataloger/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@ import (
)

type Config struct {
Search SearchConfig
Catalogers []string
Workers int
Search SearchConfig
Catalogers []string
Parallelism int
}

func DefaultConfig() Config {
return Config{
Search: DefaultSearchConfig(),
Workers: 1,
Search: DefaultSearchConfig(),
Parallelism: 1,
}
}

Expand Down
14 changes: 7 additions & 7 deletions test/cli/packages_cmd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,23 +205,23 @@ func TestPackagesCmdFlags(t *testing.T) {
},
},
{
name: "override-default-workers",
args: []string{"packages", "-vvv", "-o", "json", "--workers", "2", coverageImage},
name: "override-default-parallelism",
args: []string{"packages", "-vvv", "-o", "json", "--parallelism", "2", coverageImage},
assertions: []traitAssertion{
// the application config in the log matches that of what we expect to have been configured.
assertInOutput("workers: 2"),
assertInOutput("Using workers=2 for catalogs"),
assertInOutput("parallelism: 2"),
assertInOutput("Using parallelism=2 for catalogs"),
assertPackageCount(34),
assertSuccessfulReturnCode,
},
},
{
name: "default-workers",
name: "default-parallelism",
args: []string{"packages", "-vvv", "-o", "json", coverageImage},
assertions: []traitAssertion{
// the application config in the log matches that of what we expect to have been configured.
assertInOutput("workers: 1"),
assertInOutput("Using workers=1 for catalogs"),
assertInOutput("parallelism: 1"),
assertInOutput("Using parallelism=1 for catalogs"),
assertPackageCount(34),
assertSuccessfulReturnCode,
},
Expand Down

0 comments on commit 2929836

Please sign in to comment.