Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(count): s3 Ec2 and are counted as stats #17

Closed
wants to merge 1 commit into from
Closed

feat(count): s3 Ec2 and are counted as stats #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmd/aws-snapshot/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ func realMain() error {
manifestFile = flag.String("manifest-file", "", "Manifest filename")
requestTimeout = flag.String("request-timeout", "", "Timeout per request, default is no timeout")
verbosity = flag.Int("v", 0, "Logging verbosity, full debug is 6")
stats = flag.Bool("s", false, "Counts Only")
)

flag.Parse()
Expand Down Expand Up @@ -99,6 +100,7 @@ func realMain() error {
}),
RequestTimeout: timeout,
Logger: &logger,
Stats: *stats,
}

return runner.Run(context.Background())
Expand Down
2 changes: 2 additions & 0 deletions pkg/api/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type Runner struct {
ConcurrentRecorders int
RequestTimeout *time.Duration
Logger *logr.Logger
Stats bool
}

// Pool runs num copies of Recorder.
Expand Down Expand Up @@ -103,6 +104,7 @@ func (r *Runner) Run(ctx context.Context) error {
}

ctx, cancelFunc := context.WithCancel(ctx)
ctx = context.WithValue(ctx , "runner_config", *r)
defer cancelFunc()

var (
Expand Down
35 changes: 29 additions & 6 deletions pkg/service/ec2/addresses.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,17 @@ func (o *DescribeAddressesOutput) Records() (records []*api.Record) {
return
}

type CountInstancesAddressOutput struct {
Count int `json:Count`
}

func (o *CountInstancesAddressOutput) Records() (records []*api.Record) {
records = append(records, &api.Record{
Data: o,
})
return
}

type DescribeAddresses struct {
API
}
Expand All @@ -34,14 +45,26 @@ func (fn *DescribeAddresses) New(name string, config interface{}) ([]api.Request
if err := api.DecodeConfig(config, &input); err != nil {
return nil, err
}

//fmt.Println("Doing EC2 Stats %d", r.Stats )
call := func(ctx context.Context, ch chan<- *api.Record) error {
output, err := fn.DescribeAddressesWithContext(ctx, &input)
if err != nil {
return err
}
r, ok := ctx.Value("runner_config").(api.Runner)

return api.SendRecords(ctx, ch, name, &DescribeAddressesOutput{output})
if !ok {
return nil
} else if ok && r.Stats {
output, err := fn.DescribeAddressesWithContext(ctx, &input)
if err != nil {
return err
}
return api.SendRecords(ctx, ch, name, &CountInstancesAddressOutput{len(output.Addresses)})
} else {
output, err := fn.DescribeAddressesWithContext(ctx, &input)
if err != nil {
return err
}

return api.SendRecords(ctx, ch, name, &DescribeAddressesOutput{output})
}
}

return []api.Request{call}, nil
Expand Down
27 changes: 23 additions & 4 deletions pkg/service/ec2/flowlogs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,18 @@ func (o *DescribeFlowLogsOutput) Records() (records []*api.Record) {
return
}


type CountFlowLogs struct {
Count int `json:Count`
}

func (o *CountFlowLogs) Records() (records []*api.Record) {
records = append(records, &api.Record{
Data: o,
})
return
}

type DescribeFlowLogs struct {
API
}
Expand All @@ -37,15 +49,22 @@ func (fn *DescribeFlowLogs) New(name string, config interface{}) ([]api.Request,

call := func(ctx context.Context, ch chan<- *api.Record) error {
var outerErr, innerErr error

var countFlowLogsOutput int
r, _ := ctx.Value("runner_config").(api.Runner)
outerErr = fn.DescribeFlowLogsPagesWithContext(ctx, &input, func(output *ec2.DescribeFlowLogsOutput, last bool) bool {
if err := api.SendRecords(ctx, ch, name, &DescribeFlowLogsOutput{output}); err != nil {
innerErr = err
return false
if r.Stats {
countFlowLogsOutput += len(output.FlowLogs)
} else {
if innerErr = api.SendRecords(ctx, ch, name, &DescribeFlowLogsOutput{output}); innerErr != nil {
return false
}
}

return true
})
if outerErr == nil && r.Stats {
innerErr = api.SendRecords(ctx, ch, name, &CountFlowLogs{countFlowLogsOutput})
}

return api.FirstError(outerErr, innerErr)
}
Expand Down
42 changes: 34 additions & 8 deletions pkg/service/ec2/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,18 @@ type DescribeInstancesOutput struct {
*ec2.DescribeInstancesOutput
}

// TODO: nested and complicated parts
type CountInstancesOutput struct {
Count int `json:Count`
}

func (o *CountInstancesOutput) Records() (records []*api.Record) {
records = append(records, &api.Record{
Data: o,
})
return
}

func (o *DescribeInstancesOutput) Records() (records []*api.Record) {
type elem struct {
Instance interface{} `json:"instance,omitempty"`
Expand Down Expand Up @@ -52,27 +64,41 @@ type DescribeInstances struct {
API
}

var _ api.RequestBuilder = &DescribeInstances{}

// New implements api.RequestBuilder
func (fn *DescribeInstances) New(name string, config interface{}) ([]api.Request, error) {
var input ec2.DescribeInstancesInput
var instanceCount int

if err := api.DecodeConfig(config, &input); err != nil {
return nil, err
}

call := func(ctx context.Context, ch chan<- *api.Record) error {
var outerErr, innerErr error

r, _ := ctx.Value("runner_config").(api.Runner)
// This fucntion call happens for each page
outerErr = fn.DescribeInstancesPagesWithContext(ctx, &input, func(output *ec2.DescribeInstancesOutput, last bool) bool {
if err := api.SendRecords(ctx, ch, name, &DescribeInstancesOutput{output}); err != nil {
innerErr = err
return false
// Loop through each reservation, instance
if r.Stats {
for _, reservation := range output.Reservations {
for _, instance := range reservation.Instances {
// Is this a valid instance? Spot instances have an InstanceLifecycle of "spot".
// Similarly, Scheduled instances have an InstanceLifecycle of "scheduled".
if instance.InstanceLifecycle == nil {
instanceCount++
}
}
}
} else {
if innerErr = api.SendRecords(ctx, ch, name, &DescribeInstancesOutput{output}); innerErr != nil {
return false
}
}

return true
})

if outerErr == nil && r.Stats {
innerErr = api.SendRecords(ctx, ch, name, &CountInstancesOutput{instanceCount})
}
return api.FirstError(outerErr, innerErr)
}

Expand Down
23 changes: 18 additions & 5 deletions pkg/service/ec2/internetgateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,17 @@ type DescribeInternetGatewaysOutput struct {
*ec2.DescribeInternetGatewaysOutput
}

type CountInternetGatewaysOutput struct {
Count int `json:Count`
}

func (o *CountInternetGatewaysOutput) Records() (records []*api.Record) {
records = append(records, &api.Record{
Data: o,
})
return
}

func (o *DescribeInternetGatewaysOutput) Records() (records []*api.Record) {
for _, s := range o.InternetGateways {
records = append(records, &api.Record{
Expand All @@ -34,19 +45,21 @@ func (fn *DescribeInternetGateways) New(name string, config interface{}) ([]api.
if err := api.DecodeConfig(config, &input); err != nil {
return nil, err
}

call := func(ctx context.Context, ch chan<- *api.Record) error {
var outerErr, innerErr error

r, _:= ctx.Value("runner_config").(api.Runner)
outerErr = fn.DescribeInternetGatewaysPagesWithContext(ctx, &input, func(output *ec2.DescribeInternetGatewaysOutput, last bool) bool {
if err := api.SendRecords(ctx, ch, name, &DescribeInternetGatewaysOutput{output}); err != nil {
if r.Stats {
if err := api.SendRecords(ctx, ch, name, &CountInternetGatewaysOutput{len(output.InternetGateways)}); err != nil {
innerErr = err
return false
}
} else if err := api.SendRecords(ctx, ch, name, &DescribeInternetGatewaysOutput{output}); err != nil {
innerErr = err
return false
}

return true
})

return api.FirstError(outerErr, innerErr)
}

Expand Down
26 changes: 22 additions & 4 deletions pkg/service/ec2/natgateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,17 @@ func (o *DescribeNatGatewaysOutput) Records() (records []*api.Record) {
return
}

type CountNatGatewaysOutput struct {
Count int `json:Count`
}

func (o *CountNatGatewaysOutput) Records() (records []*api.Record) {
records = append(records, &api.Record{
Data: o,
})
return
}

type DescribeNatGateways struct {
API
}
Expand All @@ -37,15 +48,22 @@ func (fn *DescribeNatGateways) New(name string, config interface{}) ([]api.Reque

call := func(ctx context.Context, ch chan<- *api.Record) error {
var outerErr, innerErr error
var gatewayCount int

r, _ := ctx.Value("runner_config").(api.Runner)
outerErr = fn.DescribeNatGatewaysPagesWithContext(ctx, &input, func(output *ec2.DescribeNatGatewaysOutput, last bool) bool {
if err := api.SendRecords(ctx, ch, name, &DescribeNatGatewaysOutput{output}); err != nil {
innerErr = err
return false
if r.Stats {
gatewayCount += len(output.NatGateways)
} else {
if innerErr = api.SendRecords(ctx, ch, name, &DescribeNatGatewaysOutput{output}); innerErr != nil {
return false
}
}

return true
})
if outerErr == nil && r.Stats {
innerErr = api.SendRecords(ctx, ch, name, &CountNatGatewaysOutput{gatewayCount})
}

return api.FirstError(outerErr, innerErr)
}
Expand Down
28 changes: 23 additions & 5 deletions pkg/service/ec2/networkacl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,17 @@ func (o *DescribeNetworkAclsOutput) Records() (records []*api.Record) {
return
}

type CountNetworkACLs struct {
Count int `json:Count`
}

func (o *CountNetworkACLs) Records() (records []*api.Record) {
records = append(records, &api.Record{
Data: o,
})
return
}

type DescribeNetworkAcls struct {
API
}
Expand All @@ -31,21 +42,28 @@ var _ api.RequestBuilder = &DescribeNetworkAcls{}
// New implements api.RequestBuilder
func (fn *DescribeNetworkAcls) New(name string, config interface{}) ([]api.Request, error) {
var input ec2.DescribeNetworkAclsInput
var networkACLCount int
if err := api.DecodeConfig(config, &input); err != nil {
return nil, err
}

call := func(ctx context.Context, ch chan<- *api.Record) error {
var outerErr, innerErr error
r, _ := ctx.Value("runner_config").(api.Runner)

outerErr = fn.DescribeNetworkAclsPagesWithContext(ctx, &input, func(output *ec2.DescribeNetworkAclsOutput, last bool) bool {
if err := api.SendRecords(ctx, ch, name, &DescribeNetworkAclsOutput{output}); err != nil {
innerErr = err
return false
if r.Stats {
networkACLCount += len(output.NetworkAcls)
} else {
if innerErr = api.SendRecords(ctx, ch, name, &DescribeNetworkAclsOutput{output}); innerErr != nil {
return false
}
}

return true
})

if outerErr == nil && r.Stats {
innerErr = api.SendRecords(ctx, ch, name, &CountNetworkACLs{networkACLCount})
}
return api.FirstError(outerErr, innerErr)
}

Expand Down
26 changes: 22 additions & 4 deletions pkg/service/ec2/networkinterfaces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,17 @@ func (o *DescribeNetworkInterfacesOutput) Records() (records []*api.Record) {
return
}

type CountNetworkInterfacesOutput struct {
Count int `json:Count`
}

func (o *CountNetworkInterfacesOutput) Records() (records []*api.Record) {
records = append(records, &api.Record{
Data: o,
})
return
}

type DescribeNetworkInterfaces struct {
API
}
Expand All @@ -37,16 +48,23 @@ func (fn *DescribeNetworkInterfaces) New(name string, config interface{}) ([]api

call := func(ctx context.Context, ch chan<- *api.Record) error {
var outerErr, innerErr error
var countNetworkInterfaces int

r, _ := ctx.Value("runner_config").(api.Runner)
outerErr = fn.DescribeNetworkInterfacesPagesWithContext(ctx, &input, func(output *ec2.DescribeNetworkInterfacesOutput, last bool) bool {
if err := api.SendRecords(ctx, ch, name, &DescribeNetworkInterfacesOutput{output}); err != nil {
innerErr = err
return false
if r.Stats {
countNetworkInterfaces += len(output.NetworkInterfaces)
} else {
if innerErr := api.SendRecords(ctx, ch, name, &DescribeNetworkInterfacesOutput{output}); innerErr != nil {
return false
}
}

return true
})

if outerErr == nil && r.Stats {
innerErr = api.SendRecords(ctx, ch, name, &CountNetworkInterfacesOutput{countNetworkInterfaces})
}
return api.FirstError(outerErr, innerErr)
}

Expand Down
Loading
Loading