use goflags

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/namecheap/go-namecheap-sdk/v2 v2.0.0-20210701132211-e8fe6bd652dd
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/fileutil v0.0.0-20210601061022-8ef4fc6fbfb6
+	github.com/projectdiscovery/goflags v0.0.7
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7

go.sum

@@ -103,6 +103,8 @@ github.com/cloudflare/cloudflare-go v0.17.0/go.mod h1:sPWL/lIC6biLEdyGZwBQ1rGQKF
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -385,6 +387,8 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr
 github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/projectdiscovery/fileutil v0.0.0-20210601061022-8ef4fc6fbfb6 h1:8R7utSGtyYYyya2mfkd0eO0xOV74z+xybR/oK9b+WSQ=
 github.com/projectdiscovery/fileutil v0.0.0-20210601061022-8ef4fc6fbfb6/go.mod h1:+KrA1FQf8IYdgf/ifBw+7GXW+zYuppvqJKb+lmloDVw=
+github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk=
+github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
@@ -395,6 +399,8 @@ github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b/go.mod h1:
 github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
 github.com/projectdiscovery/mapcidr v0.0.6 h1:RRIrqNakUEF/pstIXWTD6yvCMF9N6SnOb9m4ju4xavc=
 github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

internal/runner/options.go

@@ -2,60 +2,89 @@ package runner
 
 import (
 	"errors"
-	"flag"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/user"
-	"path"
+	"path/filepath"
 
 	"github.com/projectdiscovery/cloudlist/pkg/schema"
 	"github.com/projectdiscovery/fileutil"
+	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"gopkg.in/yaml.v2"
 )
 
 // Options contains the configuration options for cloudlist.
 type Options struct {
-	JSON           bool   // JSON returns JSON output
-	Silent         bool   // Silent Display results only
-	Version        bool   // Version returns the version of the tool.
-	Verbose        bool   // Verbose prints verbose output.
-	Hosts          bool   // Hosts specifies to fetch only DNS Names
-	IPAddress      bool   // IPAddress specifes to fetch only IP Addresses
-	Config         string // Config is the location of the config file.
-	Output         string // Output is the file to write found results too.
-	ExcludePrivate bool   // ExcludePrivate excludes private IPs from results
-	Provider       string // Provider specifies what providers to fetch assets for.
+	JSON           bool                          // JSON returns JSON output
+	Silent         bool                          // Silent Display results only
+	Version        bool                          // Version returns the version of the tool.
+	Verbose        bool                          // Verbose prints verbose output.
+	Hosts          bool                          // Hosts specifies to fetch only DNS Names
+	IPAddress      bool                          // IPAddress specifes to fetch only IP Addresses
+	Config         string                        // Config is the location of the config file.
+	Output         string                        // Output is the file to write found results too.
+	ExcludePrivate bool                          // ExcludePrivate excludes private IPs from results
+	Provider       goflags.NormalizedStringSlice // Provider specifies what providers to fetch assets for.
+	Id             goflags.NormalizedStringSlice // Id specifies what id's to fetch assets for.
+	ProviderConfig string                        // ProviderConfig is the location of the provider config file.
 }
 
-var defaultConfigLocation = path.Join(userHomeDir(), "/.config/cloudlist/config.yaml")
+var (
+	defaultConfigLocation         = filepath.Join(userHomeDir(), ".config/cloudlist/config.yaml")
+	defaultProviderConfigLocation = filepath.Join(userHomeDir(), ".config/cloudlist/provider-config.yaml")
+)
 
 // ParseOptions parses the command line flags provided by a user
 func ParseOptions() *Options {
+	showBanner()
+
+	// Migrate config to provider config
+	if fileutil.FileExists(defaultConfigLocation) && !fileutil.FileExists(defaultProviderConfigLocation) {
+		if _, err := readProviderConfig(defaultConfigLocation); err == nil {
+			gologger.Info().Msg("Detected old config.yaml file, trying to rename it to provider-config.yaml\n")
+			if err := os.Rename(defaultConfigLocation, defaultProviderConfigLocation); err != nil {
+				gologger.Fatal().Msgf("Could not rename existing config (config.yaml) to provider config (provider-config.yaml): %s\n", err)
+			} else {
+				gologger.Info().Msg("Renamed config.yaml to provider-config.yaml successfully\n")
+			}
+		}
+	}
+
 	options := &Options{}
+	flagSet := goflags.NewFlagSet()
+	flagSet.SetDescription(`Cloudlist is a tool for listing Assets from multiple cloud providers.`)
+
+	createGroup(flagSet, "config", "Configuration",
+		flagSet.StringVar(&options.Config, "config", defaultConfigLocation, "cloudlist flag configuration file path"),
+		flagSet.StringVarP(&options.ProviderConfig, "provider-config", "pc", defaultProviderConfigLocation, "provider configuration file path"),
+	)
+	createGroup(flagSet, "filter", "Filters",
+		flagSet.NormalizedStringSliceVarP(&options.Provider, "provider", "p", []string{}, "provider to fetch assets from"),
+		flagSet.NormalizedStringSliceVar(&options.Id, "id", []string{}, "id to fetch assets from"),
+		flagSet.BoolVar(&options.Hosts, "host", false, "display only hostnames in cli output"),
+		flagSet.BoolVar(&options.IPAddress, "ip", false, "display only ips in cli output"),
+		flagSet.BoolVarP(&options.ExcludePrivate, "exclude-private", "ep", false, "exclude private ips in cli output"),
+	)
+	createGroup(flagSet, "output", "Output",
+		flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write results"),
+		flagSet.BoolVar(&options.JSON, "json", false, "write output in json format"),
+		flagSet.BoolVar(&options.Version, "version", false, "display version of cloudlist"),
+		flagSet.BoolVar(&options.Verbose, "v", false, "display derbose output"),
+		flagSet.BoolVar(&options.Silent, "silent", false, "display only results in output"),
+	)
 
-	flag.BoolVar(&options.JSON, "json", false, "Show json output")
-	flag.BoolVar(&options.Silent, "silent", false, "Show only results in output")
-	flag.BoolVar(&options.Version, "version", false, "Show version of cloudlist")
-	flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
-	flag.BoolVar(&options.Hosts, "host", false, "Show only hosts in output")
-	flag.BoolVar(&options.IPAddress, "ip", false, "Show only IP addresses in output")
-	flag.StringVar(&options.Config, "config", defaultConfigLocation, "Configuration file to use for enumeration")
-	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
-	flag.StringVar(&options.Provider, "provider", "", "Provider to fetch assets from (optional)")
-	flag.BoolVar(&options.ExcludePrivate, "exclude-private", false, "Exclude private IP addresses from output")
-	flag.Parse()
+	_ = flagSet.Parse()
 
 	options.configureOutput()
-	showBanner()
 
 	if options.Version {
 		gologger.Info().Msgf("Current Version: %s\n", Version)
 		os.Exit(0)
 	}
-	checkAndCreateConfigFile(options)
+	checkAndCreateProviderConfigFile(options)
 	return options
 }
 
@@ -70,8 +99,8 @@ func (options *Options) configureOutput() {
 	}
 }
 
-// readConfig reads the config file from the options
-func readConfig(configFile string) (schema.Options, error) {
+// readProviderConfig reads the provider config file from the options
+func readProviderConfig(configFile string) (schema.Options, error) {
 	file, err := os.Open(configFile)
 	if err != nil {
 		return nil, err
@@ -81,24 +110,24 @@ func readConfig(configFile string) (schema.Options, error) {
 	config := schema.Options{}
 	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
 		if err == io.EOF {
-			return nil, errors.New("invalid configuration file provided")
+			return nil, errors.New("invalid provider configuration file provided")
 		}
 		return nil, err
 	}
 	return config, nil
 }
 
-// checkAndCreateConfigFile checks if a config file exists,
+// checkAndCreateProviderConfigFile checks if a provider config file exists,
 // if not creates a default.
-func checkAndCreateConfigFile(options *Options) {
-	if options.Config == defaultConfigLocation {
-		err := os.MkdirAll(path.Dir(options.Config), os.ModePerm)
+func checkAndCreateProviderConfigFile(options *Options) {
+	if options.ProviderConfig == "" {
+		err := os.MkdirAll(filepath.Dir(options.ProviderConfig), os.ModePerm)
 		if err != nil {
 			gologger.Warning().Msgf("Could not create default config file: %s\n", err)
 		}
-		if !fileutil.FileExists(defaultConfigLocation) {
-			if writeErr := ioutil.WriteFile(defaultConfigLocation, []byte(defaultConfigFile), os.ModePerm); writeErr != nil {
-				gologger.Warning().Msgf("Could not write default output to %s: %s\n", defaultConfigLocation, writeErr)
+		if !fileutil.FileExists(defaultProviderConfigLocation) {
+			if writeErr := ioutil.WriteFile(defaultProviderConfigLocation, []byte(defaultProviderConfigFile), os.ModePerm); writeErr != nil {
+				gologger.Warning().Msgf("Could not write default output to %s: %s\n", defaultProviderConfigLocation, writeErr)
 			}
 		}
 	}
@@ -112,7 +141,14 @@ func userHomeDir() string {
 	return usr.HomeDir
 }
 
-const defaultConfigFile = `#  #Configuration file for cloudlist enumeration agent
+func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
+	flagSet.SetGroup(groupName, description)
+	for _, currentFlag := range flags {
+		currentFlag.Group(groupName)
+	}
+}
+
+const defaultProviderConfigFile = `#  #Provider configuration file for cloudlist enumeration agent
 
 #- # provider is the name of the provider
 #  provider: do

internal/runner/runner.go

@@ -21,7 +21,13 @@ type Runner struct {
 
 // New creates a new runner instance based on configuration options
 func New(options *Options) (*Runner, error) {
-	config, err := readConfig(options.Config)
+
+	if options.ProviderConfig == "" {
+		options.ProviderConfig = defaultProviderConfigLocation
+		gologger.Print().Msgf("Using default provider config: %s\n", options.ProviderConfig)
+	}
+
+	config, err := readProviderConfig(options.ProviderConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -40,13 +46,14 @@ func (r *Runner) Enumerate() {
 			item["id"] = ""
 		}
 		// Validate and only pass the correct items to input
-		if r.options.Provider != "" {
-			if item["provider"] != r.options.Provider {
-				gologger.Verbose().Msgf("WRN: Skipping provider %s due to command line\n", item["provider"])
+		if len(r.options.Provider) != 0 || len(r.options.Id) != 0 {
+			if len(r.options.Provider) != 0 && !Contains(r.options.Provider, item["provider"]) {
 				continue
-			} else {
-				finalConfig = append(finalConfig, item)
 			}
+			if len(r.options.Id) != 0 && !Contains(r.options.Id, item["id"]) {
+				continue
+			}
+			finalConfig = append(finalConfig, item)
 		} else {
 			finalConfig = append(finalConfig, item)
 		}
@@ -68,11 +75,6 @@ func (r *Runner) Enumerate() {
 
 	builder := &bytes.Buffer{}
 	for _, provider := range inventory.Providers {
-		if r.options.Provider != "" {
-			if provider.Name() != r.options.Provider {
-				continue
-			}
-		}
 
 		gologger.Info().Msgf("Listing assets from %s (%s) provider\n", provider.Name(), provider.ID())
 		instances, err := provider.Resources(context.Background())
@@ -183,3 +185,12 @@ func (r *Runner) Enumerate() {
 		}
 	}
 }
+
+func Contains(s []string, e string) bool {
+	for _, a := range s {
+		if strings.EqualFold(a, e) {
+			return true
+		}
+	}
+	return false
+}