Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add dry run #39

Merged
merged 6 commits into from
Sep 22, 2022
Merged

add dry run #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
293e91e
wip
koluku Sep 19, 2022
5af0bc0
add dry-run
koluku Sep 19, 2022
b17cdb9
humanize message
koluku Sep 20, 2022
ccb9480
Merge branch 'v0.4.0' into feature/add-dry-run
koluku Sep 20, 2022
6b51dfa
go mod tidy
koluku Sep 20, 2022
37ff0f4
rename variable
koluku Sep 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions cmd/s3s/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os"
"os/signal"

"github.com/dustin/go-humanize"
"github.com/koluku/s3s"
"github.com/pkg/errors"
"github.com/urfave/cli/v2"
Expand Down Expand Up @@ -42,10 +43,11 @@ var (
maxRetries int
isDelve bool
isDebug bool
isDryRun bool
)

func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill)
defer stop()

app := &cli.App{
Expand Down Expand Up @@ -127,6 +129,13 @@ func main() {
Value: false,
Destination: &isDebug,
},
&cli.BoolFlag{
Name: "dry-run",
Aliases: []string{"dry_run"},
Usage: "pre request for s3 select",
Value: false,
Destination: &isDryRun,
},
},
Action: func(c *cli.Context) error {
if err := cmd(c.Context, c.Args().Slice()); err != nil {
Expand Down Expand Up @@ -195,7 +204,17 @@ func cmd(ctx context.Context, paths []string) error {
default:
queryInfo.FormatType = s3s.FormatTypeJSON
}
app.Run(ctx, paths, queryStr, queryInfo)

if isDryRun {
scanByte, count, err := app.DryRun(ctx, paths, queryStr, queryInfo)
if err != nil {
return errors.WithStack(err)
}
fmt.Printf("all scan byte: %s\n", humanize.Bytes(uint64(scanByte)))
fmt.Printf("file count: %s\n", humanize.Comma(int64(count)))
} else {
app.Run(ctx, paths, queryStr, queryInfo)
}

// Finalize
if isDelve {
Expand Down
13 changes: 7 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ module github.com/koluku/s3s
go 1.17

require (
github.com/aws/aws-sdk-go-v2 v1.16.11
github.com/aws/aws-sdk-go-v2 v1.16.15
github.com/aws/aws-sdk-go-v2/config v1.17.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.5
github.com/dustin/go-humanize v1.0.0
github.com/ktr0731/go-fuzzyfinder v0.6.0
github.com/pkg/errors v0.9.1
github.com/urfave/cli/v2 v2.11.2
Expand All @@ -16,8 +17,8 @@ require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.4 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.14 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.5 // indirect
Expand All @@ -26,7 +27,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 // indirect
github.com/aws/smithy-go v1.12.1 // indirect
github.com/aws/smithy-go v1.13.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/gdamore/tcell/v2 v2.4.0 // indirect
Expand All @@ -36,7 +37,7 @@ require (
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
)
21 changes: 15 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/aws/aws-sdk-go-v2 v1.16.11 h1:xM1ZPSvty3xVmdxiGr7ay/wlqv+MWhH0rMlyLdbC0YQ=
github.com/aws/aws-sdk-go-v2 v1.16.11/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo=
github.com/aws/aws-sdk-go-v2 v1.16.15 h1:2sInOWGE4HV54R90Pj8QgqBBw3Qf1I0husqbqjPZzys=
github.com/aws/aws-sdk-go-v2 v1.16.15/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.4 h1:zfT11pa7ifu/VlLDpmc5OY2W4nYmnKkFDGeMVnmqAI0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.4/go.mod h1:ES0I1GBs+YYgcDS1ek47Erbn4TOL811JKqBXtgzqyZ8=
github.com/aws/aws-sdk-go-v2/config v1.17.1 h1:BWxTjokU/69BZ4DnLrZco6OvBDii6ToEdfBL/y5I1nA=
Expand All @@ -9,10 +10,12 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.14 h1:AtVG/amkjbDBfnPr/tuW2IG18HG
github.com/aws/aws-sdk-go-v2/credentials v1.12.14/go.mod h1:opAndTyq+YN7IpVG57z2CeNuXSQMqTYxGGlYH0m0RMY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 h1:wgJBHO58Pc1V1QAnzdVM3JK3WbE/6eUF0JxCZ+/izz0=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12/go.mod h1:aZ4vZnyUuxedC7eD4JyEHpGnCz+O2sHQEx3VvAwklSE=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 h1:OmiwoVyLKEqqD5GvB683dbSqxiOfvx4U2lDZhG2Esc4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18/go.mod h1:348MLhzV1GSlZSMusdwQpXKbhD7X2gbI/TxwAPKkYZQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12 h1:5mvQDtNWtI6H56+E4LUnLWEmATMB7oEh+Z9RurtIuC0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22 h1:pE27/u2A7JlwICjOvONQDob8PToShRTkuiUE74ymVWg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22/go.mod h1:/vNv5Al0bpiF8YdX2Ov6Xy05VTiXsql94yUqJMYaj0w=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12/go.mod h1:ckaCVTEdGAxO6KwTGzgskxR1xM+iJW4lxMyDFVda2Fc=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16 h1:L5LKGHHXOl4t7+5QZMTl38GIzSAq07XUTRtEquiHGMA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16/go.mod h1:62dsXI0BqTIGomDl8Hpm33dv0OntGaVblri3ZRParVQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19 h1:g5qq9sgtEzt2szMaDqQO6fqKe026T6dHTFJp5NsPzkQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19/go.mod h1:cVHo8KTuHjShb9V8/VjH3S/8+xPu16qx8fdGwmotJhE=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.9 h1:agLpf3vtYX1rtKTrOGpevdP3iC2W0hKDmzmhhxJzL+A=
Expand All @@ -31,11 +34,14 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.17 h1:pXxu9u2z1UqSbjO9YA8kmFJBhFc
github.com/aws/aws-sdk-go-v2/service/sso v1.11.17/go.mod h1:mS5xqLZc/6kc06IpXn5vRxdLaED+jEuaSRv5BxtnsiY=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 h1:dl8T0PJlN92rvEGOEUiD0+YPYdPEaCZK0TqHukvSfII=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.13/go.mod h1:Ru3QVMLygVs/07UQ3YDur1AQZZp2tUNje8wfloFttC0=
github.com/aws/smithy-go v1.12.1 h1:yQRC55aXN/y1W10HgwHle01DRuV9Dpf31iGkotjt3Ag=
github.com/aws/smithy-go v1.12.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA=
github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
github.com/gdamore/tcell/v2 v2.4.0 h1:W6dxJEmaxYvhICFoTY3WrLLEXsQ11SaFnKGVEXW57KM=
Expand Down Expand Up @@ -73,10 +79,13 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc=
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
Expand Down
8 changes: 5 additions & 3 deletions prefix.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,13 @@ func (app *App) GetS3Dir(ctx context.Context, bucket string, prefix string) ([]s
return s3Keys, nil
}

type Path struct {
type ObjectInfo struct {
Bucket string
Key string
Size int64
}

func (app *App) GetS3Keys(ctx context.Context, sender chan<- Path, bucket string, prefix string) error {
func (app *App) GetS3Keys(ctx context.Context, sender chan<- ObjectInfo, bucket string, prefix string) error {
input := &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String(prefix),
Expand All @@ -57,9 +58,10 @@ func (app *App) GetS3Keys(ctx context.Context, sender chan<- Path, bucket string
}

for i := range output.Contents {
sender <- Path{
sender <- ObjectInfo{
Bucket: bucket,
Key: *output.Contents[i].Key,
Size: output.Contents[i].Size,
}
}
}
Expand Down
34 changes: 31 additions & 3 deletions s3s.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func NewApp(ctx context.Context, region string, maxRetries int, threadCount int)
}

func (app *App) Run(ctx context.Context, paths []string, queryStr string, queryInfo *QueryInfo) error {
ch := make(chan Path, app.threadCount)
ch := make(chan ObjectInfo, app.threadCount)
eg, egctx := errgroup.WithContext(ctx)

eg.Go(func() error {
Expand All @@ -61,7 +61,35 @@ func (app *App) Run(ctx context.Context, paths []string, queryStr string, queryI
return nil
}

func (app *App) getBucketKeys(ctx context.Context, ch chan<- Path, paths []string) error {
func (app *App) DryRun(ctx context.Context, paths []string, queryStr string, queryInfo *QueryInfo) (int64, int, error) {
ch := make(chan ObjectInfo, app.threadCount)

var scanByte int64
var count int

eg, egctx := errgroup.WithContext(ctx)
eg.Go(func() error {
if err := app.getBucketKeys(egctx, ch, paths); err != nil {
return errors.WithStack(err)
}
return nil
})
eg.Go(func() error {
for r := range ch {
scanByte += r.Size
count++
}
return nil
})

if err := eg.Wait(); err != nil {
return 0, 0, errors.WithStack(err)
}

return scanByte, count, nil
}

func (app *App) getBucketKeys(ctx context.Context, ch chan<- ObjectInfo, paths []string) error {
eg, egctx := errgroup.WithContext(ctx)
eg.SetLimit(app.threadCount)
for _, path := range paths {
Expand Down Expand Up @@ -91,7 +119,7 @@ func (app *App) getBucketKeys(ctx context.Context, ch chan<- Path, paths []strin
return nil
}

func (app *App) execS3Select(ctx context.Context, reciever <-chan Path, queryStr string, info *QueryInfo) error {
func (app *App) execS3Select(ctx context.Context, reciever <-chan ObjectInfo, queryStr string, info *QueryInfo) error {
var count int
eg, egctx := errgroup.WithContext(ctx)
eg.SetLimit(app.threadCount)
Expand Down